Switch batch to heavy delete

ooni · Sep 6, 2024 · 9c6ba01 · 9c6ba01
1 parent c4d2685
commit 9c6ba01
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 14 deletions.
diff --git a/oonipipeline/src/oonipipeline/cli/commands.py b/oonipipeline/src/oonipipeline/cli/commands.py
@@ -151,8 +151,6 @@ def backfill(
     )
 
     temporal_config = TemporalConfig(
-        prometheus_bind_address=config.prometheus_bind_address,
-        telemetry_endpoint=config.telemetry_endpoint,
         temporal_address=config.temporal_address,
         temporal_namespace=config.temporal_namespace,
         temporal_tls_client_cert_path=config.temporal_tls_client_cert_path,
@@ -172,22 +170,14 @@ def backfill(
 @cli.command()
 @probe_cc_option
 @test_name_option
-@click.option(
-    "--fast-fail",
-    is_flag=True,
-    help="should we fail immediately when we encounter an error?",
-)
 def schedule(
     probe_cc: List[str],
     test_name: List[str],
-    fast_fail: bool,
 ):
     """
     Create schedules for the specified parameters
     """
     temporal_config = TemporalConfig(
-        telemetry_endpoint=config.telemetry_endpoint,
-        prometheus_bind_address=config.prometheus_bind_address,
         temporal_address=config.temporal_address,
         temporal_namespace=config.temporal_namespace,
         temporal_tls_client_cert_path=config.temporal_tls_client_cert_path,
@@ -214,8 +204,6 @@ def reschedule(
     Create schedules for the specified parameters
     """
     temporal_config = TemporalConfig(
-        telemetry_endpoint=config.telemetry_endpoint,
-        prometheus_bind_address=config.prometheus_bind_address,
         temporal_address=config.temporal_address,
         temporal_namespace=config.temporal_namespace,
         temporal_tls_client_cert_path=config.temporal_tls_client_cert_path,

diff --git a/oonipipeline/src/oonipipeline/db/connections.py b/oonipipeline/src/oonipipeline/db/connections.py
@@ -81,7 +81,7 @@ def execute(self, query_str, rows=None, *args, **kwargs):
                 exception_list.append(e)
                 sleep_time = min(self._max_backoff, self._backoff_factor * (2**attempt))
                 row_len = 0
-                if rows:
+                if rows and len(rows) > 0:
                     row_len = len(rows)
                     log.info(f"{query_str} {rows[0]}")
                 log.error(

diff --git a/oonipipeline/src/oonipipeline/temporal/common.py b/oonipipeline/src/oonipipeline/temporal/common.py
@@ -94,7 +94,7 @@ def maybe_delete_prev_range(db: ClickhouseConnection, prev_range: PrevRange) ->
     where = f"{where} AND created_at <= %(max_created_at)s AND created_at >= %(min_created_at)s"
     log.debug(f"runing {where} with {q_args}")
 
-    final_query = f"DELETE FROM {prev_range.table_name} {where}"
+    final_query = f"ALTER TABLE {prev_range.table_name} DELETE {where}"
     db.execute(final_query, q_args)
     return final_query