comment out the per doc sync hack (#3620)

* comment out the per doc sync hack * fix commented code --------- Co-authored-by: Richard Kuo (Danswer) <[email protected]>
onyx-dot-app · Jan 7, 2025 · 7cd76ec · 7cd76ec
1 parent 5b5c116
commit 7cd76ec
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 23 deletions.
diff --git a/backend/onyx/background/celery/tasks/vespa/tasks.py b/backend/onyx/background/celery/tasks/vespa/tasks.py
@@ -955,12 +955,14 @@ def vespa_metadata_sync_task(
             # the sync might repeat again later
             mark_document_as_synced(document_id, db_session)
 
-            redis_syncing_key = RedisConnectorCredentialPair.make_redis_syncing_key(
-                document_id
-            )
-            r = get_redis_client(tenant_id=tenant_id)
-            r.delete(redis_syncing_key)
-            # r.hdel(RedisConnectorCredentialPair.SYNCING_HASH, document_id)
+            # this code checks for and removes a per document sync key that is
+            # used to block out the same doc from continualy resyncing
+            # a quick hack that is only needed for production issues
+            # redis_syncing_key = RedisConnectorCredentialPair.make_redis_syncing_key(
+            #     document_id
+            # )
+            # r = get_redis_client(tenant_id=tenant_id)
+            # r.delete(redis_syncing_key)
 
             task_logger.info(f"doc={document_id} action=sync chunks={chunks_affected}")
     except SoftTimeLimitExceeded:

diff --git a/backend/onyx/redis/redis_connector_credential_pair.py b/backend/onyx/redis/redis_connector_credential_pair.py
@@ -30,7 +30,6 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
     FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
-    # SYNCING_HASH = PREFIX + ":vespa_syncing"
     SYNCING_PREFIX = PREFIX + ":vespa_syncing"
 
     def __init__(self, tenant_id: str | None, id: int) -> None:
@@ -61,6 +60,7 @@ def set_skip_docs(self, skip_docs: set[str]) -> None:
 
     @staticmethod
     def make_redis_syncing_key(doc_id: str) -> str:
+        """used to create a key in redis to block a doc from syncing"""
         return f"{RedisConnectorCredentialPair.SYNCING_PREFIX}:{doc_id}"
 
     def generate_tasks(
@@ -71,9 +71,6 @@ def generate_tasks(
         lock: RedisLock,
         tenant_id: str | None,
     ) -> tuple[int, int] | None:
-        # an arbitrary number in seconds to prevent the same doc from syncing repeatedly
-        SYNC_EXPIRATION = 24 * 60 * 60
-
         last_lock_time = time.monotonic()
 
         async_results = []
@@ -102,13 +99,14 @@ def generate_tasks(
             if doc.id in self.skip_docs:
                 continue
 
-            # is the document sync already queued?
-            # if redis_client.hexists(doc.id):
-            #     continue
+            # an arbitrary number in seconds to prevent the same doc from syncing repeatedly
+            # SYNC_EXPIRATION = 24 * 60 * 60
 
-            redis_syncing_key = self.make_redis_syncing_key(doc.id)
-            if redis_client.exists(redis_syncing_key):
-                continue
+            # a quick hack that can be uncommented to prevent a doc from resyncing over and over
+            # redis_syncing_key = self.make_redis_syncing_key(doc.id)
+            # if redis_client.exists(redis_syncing_key):
+            #     continue
+            # redis_client.set(redis_syncing_key, custom_task_id, ex=SYNC_EXPIRATION)
 
             # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
             # the key for the result is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
@@ -122,13 +120,6 @@ def generate_tasks(
                 RedisConnectorCredentialPair.get_taskset_key(), custom_task_id
             )
 
-            # track the doc.id in redis so that we don't resubmit it repeatedly
-            # redis_client.hset(
-            #     self.SYNCING_HASH, doc.id, custom_task_id
-            # )
-
-            redis_client.set(redis_syncing_key, custom_task_id, ex=SYNC_EXPIRATION)
-
             # Priority on sync's triggered by new indexing should be medium
             result = celery_app.send_task(
                 OnyxCeleryTask.VESPA_METADATA_SYNC_TASK,