From f5ec44abc8c0787adfcf7b8f6e119fcd24cb910c Mon Sep 17 00:00:00 2001 From: Andres Velasco Date: Fri, 29 Mar 2024 11:12:41 -0500 Subject: [PATCH 1/2] WIP: compress content --- src/app/_whiteboards/hooks/use-whiteboard.ts | 50 ++++++++++++------- src/dtos/whiteboard-dtos.ts | 2 +- src/lib/base64.ts | 32 ++++++++++++ src/lib/compress.ts | 50 +++++++++++++++++++ .../usecases/update-whiteboard-content.ts | 6 ++- 5 files changed, 121 insertions(+), 19 deletions(-) create mode 100644 src/lib/base64.ts create mode 100644 src/lib/compress.ts diff --git a/src/app/_whiteboards/hooks/use-whiteboard.ts b/src/app/_whiteboards/hooks/use-whiteboard.ts index 4a392f5..370234a 100644 --- a/src/app/_whiteboards/hooks/use-whiteboard.ts +++ b/src/app/_whiteboards/hooks/use-whiteboard.ts @@ -1,13 +1,15 @@ 'use client' -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import compare from 'just-compare' import { type ExcalidrawElement } from '@excalidraw/excalidraw/types/element/types' import { type AppState, type BinaryFiles } from '@excalidraw/excalidraw/types/types' import { useDebounceCallback } from '@/app/_shared/hooks/use-debounce-callback' -import * as exportUtils from '@/lib/export-whiteboard' +import { b64encode } from '@/lib/base64' +import { compressStream, JSONtoStream, responseToBuffer } from '@/lib/compress' +import * as exportUtils from '@/lib/export-whiteboard' import { api } from '@/trpc/react' import { type Content } from '../components/whiteboard' @@ -19,24 +21,24 @@ export const useWhiteboard = (id: number) => { const { data: whiteboard, isLoading } = api.whiteboard.findUserWhiteboardById.useQuery({ id, - }, { + }, { enabled: Boolean(id), cacheTime: Infinity, queryKey: ['whiteboard.findUserWhiteboardById', { id }], }) const [currentWhiteboard, setWhiteboard] = useState(whiteboard) + const tmpContent = useRef | null>(null) - const { mutate: updateContent } = api.whiteboard.updateUserWhiteboardContent.useMutation({ onSuccess: () => { void utils.whiteboard.findUserWhiteboardById.invalidate() }, - onMutate: async ({ content }) => { - + onMutate: async () => { + setWhiteboard({ ...currentWhiteboard, - content, + content: tmpContent.current, } as typeof whiteboard) return { currentWhiteboard } @@ -58,14 +60,14 @@ export const useWhiteboard = (id: number) => { const filterdElements = elements .filter(({ isDeleted }) => !isDeleted) - .map((element) =>({ + .map((element) => ({ ...element, customData: element.customData ?? null })) const payload = { - scene: { - elements: filterdElements, + scene: { + elements: filterdElements, appState: { viewBackgroundColor: appState.viewBackgroundColor, currentItemFontFamily: appState.currentItemFontFamily, @@ -73,8 +75,8 @@ export const useWhiteboard = (id: number) => { theme: appState.theme, exportWithDarkMode: appState.exportWithDarkMode, gridSize: appState.gridSize, - }, - scrollToContent: true, + }, + scrollToContent: true, files: filesToUpdate, }, } @@ -82,10 +84,19 @@ export const useWhiteboard = (id: number) => { const areSame = compare(payload, currentWhiteboard.content) - if (areSame){ + if (areSame) { return } - + + const content = { + scene: { + ...payload.scene, + }, + } + + tmpContent.current = content + + const updatedWhitheboard = { id: currentWhiteboard.id, content: { @@ -95,9 +106,14 @@ export const useWhiteboard = (id: number) => { } } - updateContent(updatedWhitheboard) - - void updatePreview(updatedWhitheboard as Whiteboard) + void compressStream(JSONtoStream(content)) + .then(responseToBuffer) + .then(b64encode) + .then((compressedRawContent) => updateContent({ + id: currentWhiteboard.id, + compressedRawContent + })) + .then(() => updatePreview(updatedWhitheboard as Whiteboard)) } const onChangeHandler = (elements: ExcalidrawElement[], appState: AppState, files?: BinaryFiles) => { diff --git a/src/dtos/whiteboard-dtos.ts b/src/dtos/whiteboard-dtos.ts index c13c29c..521f71a 100644 --- a/src/dtos/whiteboard-dtos.ts +++ b/src/dtos/whiteboard-dtos.ts @@ -13,5 +13,5 @@ export const UpdateWhiteboardDto = z.object({ export const UpdateWhiteboardContentDto = z.object({ id: z.number(), - content: z.unknown(), + compressedRawContent: z.string(), }) diff --git a/src/lib/base64.ts b/src/lib/base64.ts new file mode 100644 index 0000000..6f30def --- /dev/null +++ b/src/lib/base64.ts @@ -0,0 +1,32 @@ + +// export const b64encode = (buf: ArrayBuffer) => { +// return btoa(String.fromCharCode(...new Uint8Array(buf))) +// } + +export const b64encode = (buffer: ArrayBuffer) => { + let binary = '' + const bytes = new Uint8Array(buffer) + const len = bytes.byteLength + for (let i = 0; i < len; i++) { + binary += String.fromCharCode(bytes[i]!) + } + + return window.btoa(binary) +} + +// export const b64decode = (str: string) => { +// const binary_string = window.atob(str) +// const len = binary_string.length +// const bytes = new Uint8Array(new ArrayBuffer(len)) +// for (let i = 0; i < len; i++) { +// bytes[i] = binary_string.charCodeAt(i) +// } + +// return bytes +// } + +export const b64decode = (base64: string) => { + const buffer = Buffer.from(base64, 'base64') + + return new Uint8Array(buffer) +} \ No newline at end of file diff --git a/src/lib/compress.ts b/src/lib/compress.ts new file mode 100644 index 0000000..f6ae10f --- /dev/null +++ b/src/lib/compress.ts @@ -0,0 +1,50 @@ +import { b64decode,b64encode } from './base64' + +export const JSONtoStream = (data: object) => { + return new Blob([JSON.stringify(data)], { + type: 'text/plain' + }).stream() +} + +export const b64toStream = (b64: string) => { + return new Blob([b64decode(b64)], { + type: 'text/plain' + }).stream() +} + +export const responseToJSON = async (response: Response): Promise> => { + const blob = await response.blob() + + const textResponse = await blob.text() + + return JSON.parse(textResponse) as Record +} + +export const responseToB64 = async (response: Response) => { + const blob = await response.blob() + const buffer = await blob.arrayBuffer() + + return b64encode(buffer) +} + +export const responseToBuffer = async (response: Response) => { + const blob = await response.blob() + + return blob.arrayBuffer() +} + +export const compressStream = async (stream: ReadableStream) => { + const compressedReadableStream = stream.pipeThrough( + new CompressionStream('gzip') + ) + + return new Response(compressedReadableStream) +} + +export const decompressStream = (stream: ReadableStream) => { + const compressedReadableStream = stream.pipeThrough( + new DecompressionStream('gzip') + ) + + return new Response(compressedReadableStream) +} diff --git a/src/server/api/whiteboard/usecases/update-whiteboard-content.ts b/src/server/api/whiteboard/usecases/update-whiteboard-content.ts index 4136efb..ca1a8cf 100644 --- a/src/server/api/whiteboard/usecases/update-whiteboard-content.ts +++ b/src/server/api/whiteboard/usecases/update-whiteboard-content.ts @@ -3,6 +3,7 @@ import { type PostgresJsDatabase } from 'drizzle-orm/postgres-js' import { type z } from 'zod' import { type UpdateWhiteboardContentDto } from '@/dtos/whiteboard-dtos' +import { b64toStream, decompressStream } from '@/lib/compress' import type * as schema from '@/server/db/schema' import { whiteboards } from '@/server/db/schema' import { NotAuthorized } from '@/server/exceptions/not-authorized' @@ -13,7 +14,7 @@ type Options = z.infer & {userId: string} const updateWhiteboardContent = async (db: PostgresJsDatabase, options: Options) => { - const { id, content, userId } = options + const { id, compressedRawContent, userId } = options const currentWhiteboard = await db.query.whiteboards.findFirst({ where: eq(whiteboards.id, id) @@ -29,6 +30,9 @@ const updateWhiteboardContent = async (db: PostgresJsDatabase, op throw new NotAuthorized('User not related to whiteboard') } + const contentResponse = decompressStream(b64toStream(compressedRawContent)) + const content = await contentResponse.json() as Record + return db.update(whiteboards).set({ content }).where(eq(whiteboards.id, id)) From c03f714d6f143c0b3de5c24faa20b2f1c5ce26b6 Mon Sep 17 00:00:00 2001 From: Andres Velasco Date: Fri, 29 Mar 2024 20:27:45 -0500 Subject: [PATCH 2/2] add compression to public view --- .../_whiteboards/components/whiteboard.tsx | 28 +++++++++- src/app/_whiteboards/hooks/use-whiteboard.ts | 7 +-- src/app/view-whiteboard/[id]/layout.tsx | 4 +- src/app/view-whiteboard/[id]/page.tsx | 19 ++++--- src/dtos/whiteboard-dtos.ts | 7 +++ src/lib/base64.ts | 30 ++--------- src/lib/compress-whiteboard.ts | 21 ++++++++ .../usecases/find-public-whiteboard.ts | 42 --------------- .../usecases/find-whiteboard-content.ts | 51 +++++++++++++++++++ .../usecases/find-whiteboard-info.ts | 37 ++++++++++++++ .../usecases/update-whiteboard-content.ts | 6 +-- 11 files changed, 163 insertions(+), 89 deletions(-) create mode 100644 src/lib/compress-whiteboard.ts delete mode 100644 src/server/api/whiteboard/usecases/find-public-whiteboard.ts create mode 100644 src/server/api/whiteboard/usecases/find-whiteboard-content.ts create mode 100644 src/server/api/whiteboard/usecases/find-whiteboard-info.ts diff --git a/src/app/_whiteboards/components/whiteboard.tsx b/src/app/_whiteboards/components/whiteboard.tsx index 2891d98..3b0f0c2 100644 --- a/src/app/_whiteboards/components/whiteboard.tsx +++ b/src/app/_whiteboards/components/whiteboard.tsx @@ -11,6 +11,8 @@ import { type ExcalidrawInitialDataState } from '@excalidraw/excalidraw/types/types' +import { decompressContent } from '@/lib/compress-whiteboard' + export interface Content { scene: { @@ -31,6 +33,13 @@ interface Props { onChange?: WhiteboardChangeEventHandler } +interface RawContentProps extends Omit { + id: number + viewModeEnabled?: boolean + initialRawCompressed?: string + onChange?: WhiteboardChangeEventHandler +} + const Excalidraw = dynamic( async () => (await import('@excalidraw/excalidraw')).Excalidraw, { @@ -96,4 +105,21 @@ export const Whiteboard = ({ ) } - \ No newline at end of file + + +export const WhiterboardFromCompressed = ({ initialRawCompressed, + ...rest +}: RawContentProps) => { + + const [initialContent, setInitialContent] = useState() + + useEffect(() => { + if (!initialRawCompressed){ + return + } + void decompressContent(initialRawCompressed) + .then((content) => setInitialContent(content as unknown as Content)) + }, [initialRawCompressed]) + + return +} \ No newline at end of file diff --git a/src/app/_whiteboards/hooks/use-whiteboard.ts b/src/app/_whiteboards/hooks/use-whiteboard.ts index 370234a..2921e58 100644 --- a/src/app/_whiteboards/hooks/use-whiteboard.ts +++ b/src/app/_whiteboards/hooks/use-whiteboard.ts @@ -7,8 +7,7 @@ import { type ExcalidrawElement } from '@excalidraw/excalidraw/types/element/typ import { type AppState, type BinaryFiles } from '@excalidraw/excalidraw/types/types' import { useDebounceCallback } from '@/app/_shared/hooks/use-debounce-callback' -import { b64encode } from '@/lib/base64' -import { compressStream, JSONtoStream, responseToBuffer } from '@/lib/compress' +import { compressContent } from '@/lib/compress-whiteboard' import * as exportUtils from '@/lib/export-whiteboard' import { api } from '@/trpc/react' @@ -106,9 +105,7 @@ export const useWhiteboard = (id: number) => { } } - void compressStream(JSONtoStream(content)) - .then(responseToBuffer) - .then(b64encode) + void compressContent(content) .then((compressedRawContent) => updateContent({ id: currentWhiteboard.id, compressedRawContent diff --git a/src/app/view-whiteboard/[id]/layout.tsx b/src/app/view-whiteboard/[id]/layout.tsx index a9a3b8e..658c028 100644 --- a/src/app/view-whiteboard/[id]/layout.tsx +++ b/src/app/view-whiteboard/[id]/layout.tsx @@ -3,14 +3,14 @@ import { type Metadata,type ResolvingMetadata } from 'next' import { redirect } from 'next/navigation' import Loading from '@/app/(protected)/loading' -import findPublicWhiteboardById from '@/server/api/whiteboard/usecases/find-public-whiteboard' +import findWhiteboardInfo from '@/server/api/whiteboard/usecases/find-whiteboard-info' import { db } from '@/server/db' export async function generateMetadata({ params }: {params: {id: string}}, parent: ResolvingMetadata) { const id = Number(params.id) - const whiteboard = await findPublicWhiteboardById(db, { id, omitContent: true }) + const whiteboard = await findWhiteboardInfo(db, { id, isPublic: true }) if (!whiteboard){ diff --git a/src/app/view-whiteboard/[id]/page.tsx b/src/app/view-whiteboard/[id]/page.tsx index ee5a990..43f325d 100644 --- a/src/app/view-whiteboard/[id]/page.tsx +++ b/src/app/view-whiteboard/[id]/page.tsx @@ -3,21 +3,20 @@ import React from 'react' import { redirect } from 'next/navigation' -import { type Content,Whiteboard } from '@/app/_whiteboards/components/whiteboard' +import { WhiterboardFromCompressed } from '@/app/_whiteboards/components/whiteboard' import { WhiteboardHeader } from '@/app/_whiteboards/components/whiteboard-header' -import { type PublicWhiteboard } from '@/app/_whiteboards/interfaces/whiteboard' -import findPublicWhiteboardById from '@/server/api/whiteboard/usecases/find-public-whiteboard' +import findWhiteboardContent from '@/server/api/whiteboard/usecases/find-whiteboard-content' import { db } from '@/server/db' const getWhiteboard = async (id: number) => { - const whiteboard = await findPublicWhiteboardById(db, { id }) + const whiteboard = await findWhiteboardContent(db, { id, isPublic: true }) if (!whiteboard){ redirect('/not-found') } - return whiteboard as unknown as PublicWhiteboard & {content: undefined | Content} + return whiteboard } @@ -25,21 +24,21 @@ const getWhiteboard = async (id: number) => { const WhitebardViewPage = async ({ params }: {params: {id: string}}) => { const whiteboardId = Number(params.id) - const whiteboard: PublicWhiteboard = await getWhiteboard(whiteboardId) + const whiteboard = await getWhiteboard(whiteboardId) return (
-
diff --git a/src/dtos/whiteboard-dtos.ts b/src/dtos/whiteboard-dtos.ts index 521f71a..6627829 100644 --- a/src/dtos/whiteboard-dtos.ts +++ b/src/dtos/whiteboard-dtos.ts @@ -1,5 +1,7 @@ import { z } from 'zod' +import { SearchByIdDto } from './shared-dtos' + export const CreateWhiteboardDto = z.object({ name: z.string().min(1, 'A name is required').max(30, 'The name must be a maximum of 30 characters.'), description: z.string().max(180, 'The description must be a maximum of 180 characters.').optional(), @@ -15,3 +17,8 @@ export const UpdateWhiteboardContentDto = z.object({ id: z.number(), compressedRawContent: z.string(), }) + + +export const SearchWhitheboard = z.object({ + isPublic: z.boolean().optional(), +}).merge(SearchByIdDto) \ No newline at end of file diff --git a/src/lib/base64.ts b/src/lib/base64.ts index 6f30def..ee8a5a1 100644 --- a/src/lib/base64.ts +++ b/src/lib/base64.ts @@ -1,32 +1,12 @@ +export const b64encode = (arrayBuffer: ArrayBuffer) => { + const buffer = Buffer.from(arrayBuffer) + const base64String = buffer.toString('base64') -// export const b64encode = (buf: ArrayBuffer) => { -// return btoa(String.fromCharCode(...new Uint8Array(buf))) -// } - -export const b64encode = (buffer: ArrayBuffer) => { - let binary = '' - const bytes = new Uint8Array(buffer) - const len = bytes.byteLength - for (let i = 0; i < len; i++) { - binary += String.fromCharCode(bytes[i]!) - } - - return window.btoa(binary) + return base64String } - -// export const b64decode = (str: string) => { -// const binary_string = window.atob(str) -// const len = binary_string.length -// const bytes = new Uint8Array(new ArrayBuffer(len)) -// for (let i = 0; i < len; i++) { -// bytes[i] = binary_string.charCodeAt(i) -// } - -// return bytes -// } export const b64decode = (base64: string) => { const buffer = Buffer.from(base64, 'base64') return new Uint8Array(buffer) -} \ No newline at end of file +} \ No newline at end of file diff --git a/src/lib/compress-whiteboard.ts b/src/lib/compress-whiteboard.ts new file mode 100644 index 0000000..a5224d0 --- /dev/null +++ b/src/lib/compress-whiteboard.ts @@ -0,0 +1,21 @@ +import { b64encode } from './base64' +import { + b64toStream, + compressStream, + decompressStream, + JSONtoStream, + responseToBuffer +} from './compress' + +export const compressContent = async (content: Record) => { + const compressedStream = await compressStream(JSONtoStream(content)) + const compressedBuffer = await responseToBuffer(compressedStream) + + return b64encode(compressedBuffer) +} + +export const decompressContent = (base64: string): Promise> => { + const contentResponse = decompressStream(b64toStream(base64)) + + return contentResponse.json() as Promise> +} \ No newline at end of file diff --git a/src/server/api/whiteboard/usecases/find-public-whiteboard.ts b/src/server/api/whiteboard/usecases/find-public-whiteboard.ts deleted file mode 100644 index 41b44f6..0000000 --- a/src/server/api/whiteboard/usecases/find-public-whiteboard.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { type PostgresJsDatabase } from 'drizzle-orm/postgres-js' -import { type z } from 'zod' - -import { type SearchByIdDto } from '@/dtos/shared-dtos' -import type * as schema from '@/server/db/schema' -import { NotFound } from '@/server/exceptions/not-found' - - -type Options = z.infer & {omitContent?: boolean} - -const findPublicWhiteboardById = async (db: PostgresJsDatabase, options: Options) => { - const { id, omitContent = false } = options - - const currentWhiteboard = await db.query.whiteboards.findFirst({ - where: (whiteboards, { eq, and }) => and(eq(whiteboards.id, id), eq(whiteboards.isPublic, true)), - columns: { - id: true, - name: true, - description: true, - content: !omitContent ? true : false, - createdAt: true, - updatedAt: true, - previewUrl: true, - }, - with: { - createdBy: { - columns: { - email: false, - emailVerified: false, - } - }, - } - }) - - if (!currentWhiteboard){ - throw new NotFound('Whiteboard not found') - } - - return currentWhiteboard -} - -export default findPublicWhiteboardById \ No newline at end of file diff --git a/src/server/api/whiteboard/usecases/find-whiteboard-content.ts b/src/server/api/whiteboard/usecases/find-whiteboard-content.ts new file mode 100644 index 0000000..f97030b --- /dev/null +++ b/src/server/api/whiteboard/usecases/find-whiteboard-content.ts @@ -0,0 +1,51 @@ +import { and, eq,type SQL } from 'drizzle-orm' +import { type PostgresJsDatabase } from 'drizzle-orm/postgres-js' +import { type z } from 'zod' + +import { type SearchWhitheboard } from '@/dtos/whiteboard-dtos' +import { compressContent } from '@/lib/compress-whiteboard' +import type * as schema from '@/server/db/schema' +import { whiteboards } from '@/server/db/schema' +import { NotFound } from '@/server/exceptions/not-found' + + +type Options = z.infer + +const findWhiteboardContent = async (db: PostgresJsDatabase, options: Options) => { + const { id, isPublic } = options + + let baseFilter = eq(whiteboards.id, id) + + if (isPublic !== undefined){ + baseFilter = and(baseFilter, eq(whiteboards.isPublic, isPublic)) as SQL + } + + const currentWhiteboard = await db.query.whiteboards.findFirst({ + where: baseFilter, + with: { + createdBy: { + columns: { + email: false, + emailVerified: false, + } + }, + } + }) + + if (!currentWhiteboard){ + throw new NotFound('Whiteboard not found') + } + + const compressedRawContent = await compressContent(currentWhiteboard.content as Record) + + return { + id: currentWhiteboard.id, + name: currentWhiteboard.name, + compressedRawContent, + description: currentWhiteboard.description, + previewUrl: currentWhiteboard.previewUrl, + createdBy: currentWhiteboard.createdBy, + } +} + +export default findWhiteboardContent \ No newline at end of file diff --git a/src/server/api/whiteboard/usecases/find-whiteboard-info.ts b/src/server/api/whiteboard/usecases/find-whiteboard-info.ts new file mode 100644 index 0000000..3b21047 --- /dev/null +++ b/src/server/api/whiteboard/usecases/find-whiteboard-info.ts @@ -0,0 +1,37 @@ +import { and, eq,type SQL } from 'drizzle-orm' +import { type PostgresJsDatabase } from 'drizzle-orm/postgres-js' +import { type z } from 'zod' + +import { type SearchWhitheboard } from '@/dtos/whiteboard-dtos' +import type * as schema from '@/server/db/schema' +import { whiteboards } from '@/server/db/schema' +import { NotFound } from '@/server/exceptions/not-found' + + +type Options = z.infer + +const findWhiteboardInfo = async (db: PostgresJsDatabase, options: Options) => { + const { id, isPublic } = options + + let baseFilter = eq(whiteboards.id, id) + + if (isPublic !== undefined){ + baseFilter = and(baseFilter, eq(whiteboards.isPublic, isPublic)) as SQL + } + + const currentWhiteboard = await db.query.whiteboards.findFirst({ + where: baseFilter, + columns: { + content: false, + } + }) + + if (!currentWhiteboard){ + throw new NotFound('Whiteboard not found') + } + + + return currentWhiteboard +} + +export default findWhiteboardInfo \ No newline at end of file diff --git a/src/server/api/whiteboard/usecases/update-whiteboard-content.ts b/src/server/api/whiteboard/usecases/update-whiteboard-content.ts index ca1a8cf..1fffcf7 100644 --- a/src/server/api/whiteboard/usecases/update-whiteboard-content.ts +++ b/src/server/api/whiteboard/usecases/update-whiteboard-content.ts @@ -3,7 +3,7 @@ import { type PostgresJsDatabase } from 'drizzle-orm/postgres-js' import { type z } from 'zod' import { type UpdateWhiteboardContentDto } from '@/dtos/whiteboard-dtos' -import { b64toStream, decompressStream } from '@/lib/compress' +import { decompressContent } from '@/lib/compress-whiteboard' import type * as schema from '@/server/db/schema' import { whiteboards } from '@/server/db/schema' import { NotAuthorized } from '@/server/exceptions/not-authorized' @@ -29,9 +29,7 @@ const updateWhiteboardContent = async (db: PostgresJsDatabase, op if (!isOwner){ throw new NotAuthorized('User not related to whiteboard') } - - const contentResponse = decompressStream(b64toStream(compressedRawContent)) - const content = await contentResponse.json() as Record + const content = await decompressContent(compressedRawContent) return db.update(whiteboards).set({ content