From 7b245611906e4e418260071cf7bacbc0042783e5 Mon Sep 17 00:00:00 2001 From: pajowu Date: Sun, 3 Sep 2023 15:47:31 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Fix=20document=20import=20for=20?= =?UTF-8?q?documents=20with=20>1000=20changes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/openapi-schema.yml | 7 ------- .../transcribee_backend/routers/document.py | 10 ---------- .../src/editor/automerge_websocket_editor.ts | 6 +++--- frontend/src/openapi-schema.ts | 2 -- frontend/src/pages/document.tsx | 16 ++-------------- frontend/src/pages/new_document.tsx | 11 ++++++++--- frontend/src/utils/auth.ts | 18 +++++++++++++++++- 7 files changed, 30 insertions(+), 40 deletions(-) diff --git a/backend/openapi-schema.yml b/backend/openapi-schema.yml index 183eec40..ebefa040 100644 --- a/backend/openapi-schema.yml +++ b/backend/openapi-schema.yml @@ -133,12 +133,6 @@ components: type: object Body_import_document_api_v1_documents_import__post: properties: - document_updates: - items: - format: binary - type: string - title: Document Updates - type: array media_file: format: binary title: Media File @@ -147,7 +141,6 @@ components: title: Name type: string required: - - document_updates - media_file - name title: Body_import_document_api_v1_documents_import__post diff --git a/backend/transcribee_backend/routers/document.py b/backend/transcribee_backend/routers/document.py index 87326c5a..2073bc01 100644 --- a/backend/transcribee_backend/routers/document.py +++ b/backend/transcribee_backend/routers/document.py @@ -50,7 +50,6 @@ DocumentMediaFile, DocumentMediaTag, DocumentShareToken, - DocumentUpdate, Task, TaskType, UserToken, @@ -360,13 +359,11 @@ async def create_document( @document_router.post("/import/") def import_document( - document_updates: List[UploadFile] = File(), media_file: UploadFile = File(), token: UserToken = Depends(get_user_token), session: Session = Depends(get_session), name: str = Form(), ): - document = Document( name=name, user_id=token.user_id, @@ -399,13 +396,6 @@ def import_document( ) session.add(reencode_task) - for document_update in document_updates: - session.add( - DocumentUpdate( - document_id=document.id, change_bytes=document_update.file.read() - ) - ) - session.commit() return document.as_api_document() diff --git a/frontend/src/editor/automerge_websocket_editor.ts b/frontend/src/editor/automerge_websocket_editor.ts index cc9900b2..dd038d13 100644 --- a/frontend/src/editor/automerge_websocket_editor.ts +++ b/frontend/src/editor/automerge_websocket_editor.ts @@ -17,7 +17,7 @@ enum MessageSyncType { } export function useAutomergeWebsocketEditor( - url: string | URL, + url: string, { onInitialSyncComplete }: { onInitialSyncComplete: (editor?: Editor) => void }, ): [Editor?, Paragraph[]?] { const debug = useDebugMode(); @@ -45,7 +45,7 @@ export function useAutomergeWebsocketEditor( const [_, navigate] = useLocation(); useEffect(() => { - const ws = new ReconnectingWebSocket(url.toString(), [], { debug }); + const ws = new ReconnectingWebSocket(url, [], { debug }); let bytesReceived = 0; console.time('initialSync'); @@ -114,7 +114,7 @@ export function useAutomergeWebsocketEditor( wsRef.current = null; ws.close(); }; - }, [url.toString(), setEditorAndInitialValue]); + }, [url, setEditorAndInitialValue]); return [editorAndInitialValue?.editor, editorAndInitialValue?.initialValue]; } diff --git a/frontend/src/openapi-schema.ts b/frontend/src/openapi-schema.ts index 62ccb5eb..437cda6e 100644 --- a/frontend/src/openapi-schema.ts +++ b/frontend/src/openapi-schema.ts @@ -193,8 +193,6 @@ export interface components { }; /** Body_import_document_api_v1_documents_import__post */ Body_import_document_api_v1_documents_import__post: { - /** Document Updates */ - document_updates: (string)[]; /** * Media File * Format: binary diff --git a/frontend/src/pages/document.tsx b/frontend/src/pages/document.tsx index cc39c40f..f3d76ca3 100644 --- a/frontend/src/pages/document.tsx +++ b/frontend/src/pages/document.tsx @@ -16,8 +16,7 @@ import { BiPencil } from 'react-icons/bi'; import { SubmitHandler, useForm } from 'react-hook-form'; import { Helmet } from 'react-helmet'; import { ShareModal } from '../editor/share'; -import { useAuthData } from '../utils/auth'; -import { getAuthToken, getShareToken } from '../api'; +import { getDocumentWsUrl, useAuthData } from '../utils/auth'; import { ExportModal } from '../editor/export'; const LazyDebugPanel = lazy(() => @@ -114,18 +113,7 @@ export function DocumentPage({ const debugMode = useDebugMode(); const { isLoggedIn } = useAuthData(); - const url = new URL(`/api/v1/documents/sync/${documentId}/`, window.location.href); - url.protocol = url.protocol.replace('http', 'ws'); - - const authToken = getAuthToken(); - if (authToken) { - url.searchParams.append('authorization', `Token ${authToken}`); - } - - const shareToken = getShareToken(); - if (shareToken) { - url.searchParams.append('share_token', shareToken); - } + const url = getDocumentWsUrl(documentId); const [editor, initialValue] = useAutomergeWebsocketEditor(url, { onInitialSyncComplete: () => { diff --git a/frontend/src/pages/new_document.tsx b/frontend/src/pages/new_document.tsx index 31fa0dd9..2ad68cf2 100644 --- a/frontend/src/pages/new_document.tsx +++ b/frontend/src/pages/new_document.tsx @@ -3,6 +3,7 @@ import { SubmitHandler, useForm } from 'react-hook-form'; import clsx from 'clsx'; import { useLocation } from 'wouter'; +import ReconnectingWebSocket from 'reconnecting-websocket'; import { createDocument, importDocument } from '../api/document'; import { Dialog, DialogTitle } from '../components/dialog'; import { FormControl, Input, Select } from '../components/form'; @@ -11,6 +12,7 @@ import { AppCenter } from '../components/app'; import { useGetConfig } from '../api/config'; import { BlobReader, BlobWriter, ZipReader, Entry } from '@zip.js/zip.js'; import * as Automerge from '@automerge/automerge'; +import { getDocumentWsUrl } from '../utils/auth'; type FieldValues = { name: string; @@ -97,7 +99,7 @@ export function NewDocumentPage() { setLoading(true); let response; if (isImport) { - type DocumentImportParamPeters = Parameters[0]; + type DocumentImportParameters = Parameters[0]; const zipReader = new ZipReader(new BlobReader(data.audioFile[0])); const entries = await zipReader.getEntries(); const [automergeFile, mediaFile] = await Promise.all([ @@ -114,13 +116,16 @@ export function NewDocumentPage() { } const doc = Automerge.load(new Uint8Array(await automergeFile.arrayBuffer())); const changes = Automerge.getChanges(Automerge.init(), doc).map((x) => new Blob([x])); - const documentParameters: DocumentImportParamPeters = { + const documentParameters: DocumentImportParameters = { name: data.name, media_file: mediaFile, - document_updates: changes, }; response = await importDocument(documentParameters); + const ws = new ReconnectingWebSocket(getDocumentWsUrl(response.data.id), []); + for (const change of changes) { + ws.send(change); + } } else { type DocumentCreateParameters = Parameters[0]; const documentParameters: DocumentCreateParameters = { diff --git a/frontend/src/utils/auth.ts b/frontend/src/utils/auth.ts index cb1ef76a..4fc80585 100644 --- a/frontend/src/utils/auth.ts +++ b/frontend/src/utils/auth.ts @@ -1,6 +1,22 @@ -import { getShareToken } from '../api'; +import { getAuthToken, getShareToken } from '../api'; import { useGetMe } from '../api/user'; +export function getDocumentWsUrl(documentId: string) { + const url = new URL(`/api/v1/documents/sync/${documentId}/`, window.location.href); + url.protocol = url.protocol.replace('http', 'ws'); + + const authToken = getAuthToken(); + if (authToken) { + url.searchParams.append('authorization', `Token ${authToken}`); + } + + const shareToken = getShareToken(); + if (shareToken) { + url.searchParams.append('share_token', shareToken); + } + return url.toString(); +} + export function useAuthData(): { isLoading: boolean; isLoggedIn: boolean;