From 243eb7a87046e1b7e8993d9732ae9676a227f05e Mon Sep 17 00:00:00 2001 From: Friedrich Lindenberg Date: Wed, 17 Jul 2024 10:37:48 +0200 Subject: [PATCH] add a flag for disabling delta updates --- yente/data/updater.py | 7 ++++++- yente/settings.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/yente/data/updater.py b/yente/data/updater.py index 4c51658b..4e0013cf 100644 --- a/yente/data/updater.py +++ b/yente/data/updater.py @@ -1,6 +1,7 @@ from typing import Optional, TypedDict, Dict, List, Any from typing import AsyncGenerator, Tuple +from yente import settings from yente.logs import get_logger from yente.data.dataset import Dataset from yente.data.loader import load_json_url, load_json_lines @@ -41,6 +42,8 @@ async def build( if dataset.delta_url is None: log.debug("No delta updates available for: %r" % dataset.name) return obj + if not settings.DELTA_UPDATES: + return obj if obj.base_version is None or obj.target_version <= obj.base_version: return obj @@ -77,7 +80,9 @@ def is_incremental(self) -> bool: """Check if there is sequence of delta entity patches that can be loaded.""" if self.force_full: return False - return self.delta_urls is not None and len(self.delta_urls) > 0 + if not settings.DELTA_UPDATES: + return False + return self.delta_urls is not None def needs_update(self) -> bool: """Confirm that the dataset needs to be loaded.""" diff --git a/yente/settings.py b/yente/settings.py index c1d62e30..bf5a0a65 100644 --- a/yente/settings.py +++ b/yente/settings.py @@ -116,6 +116,7 @@ def random_cron() -> str: CRONTAB = env_str("YENTE_CRONTAB", random_cron()) AUTO_REINDEX = as_bool(env_str("YENTE_AUTO_REINDEX", "true")) STREAM_LOAD = as_bool(env_str("YENTE_STREAM_LOAD", "true")) +DELTA_UPDATES = as_bool(env_str("YENTE_DELTA_UPDATES", "true")) DEFAULT_ALGORITHM = env_str("YENTE_DEFAULT_ALGORITHM", "logic-v1") BEST_ALGORITHM = env_str("YENTE_BEST_ALGORITHM", "logic-v1")