From 126f0e83315b81d3b4b6832e70d3336d15d8f8c4 Mon Sep 17 00:00:00 2001 From: Yuhong Sun Date: Fri, 27 Oct 2023 23:31:29 -0700 Subject: [PATCH] comment on correctness --- backend/danswer/document_index/vespa/index.py | 7 +++++++ backend/danswer/server/cc_pair/models.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py index 5e808d9fa3e..6f89d86fc0b 100644 --- a/backend/danswer/document_index/vespa/index.py +++ b/backend/danswer/document_index/vespa/index.py @@ -277,6 +277,13 @@ def _index_vespa_chunks( if chunk_already_existed: already_existing_documents.add(chunk.source_document.id) + # In the logic below, we check if the chunk comes from a doc that has already been + # added to already_existing_document. This works because the chunks are ordered + # and because the Document chunks are not separated into different batches. + # The first chunk is processed first and if it exists, then its entire document + # is marked as already existing, so if the document length increases and new chunks + # are added, they must come last in processing and the doc would already be in + # already existing documents. insertion_records.add( DocumentInsertionRecord( document_id=chunk.source_document.id, diff --git a/backend/danswer/server/cc_pair/models.py b/backend/danswer/server/cc_pair/models.py index 4fd125684fb..cae06a39c5e 100644 --- a/backend/danswer/server/cc_pair/models.py +++ b/backend/danswer/server/cc_pair/models.py @@ -23,7 +23,7 @@ def from_models( cc_pair_model: ConnectorCredentialPair, index_attempt_models: list[IndexAttempt], latest_deletion_attempt: DeletionAttemptSnapshot | None, - num_docs_indexed: int, # not ideal, but this must be computed seperately + num_docs_indexed: int, # not ideal, but this must be computed separately ) -> "CCPairFullInfo": return cls( id=cc_pair_model.id,