Skip to content

Commit

Permalink
Fix bug in mutation call
Browse files Browse the repository at this point in the history
Add comments to wait for mutation calls
  • Loading branch information
hellais committed Sep 6, 2024
1 parent 9bd822d commit 4dbf122
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 1 deletion.
4 changes: 3 additions & 1 deletion oonipipeline/src/oonipipeline/temporal/activities/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,9 @@ class OptimizeTablesParams:
def optimize_tables(params: OptimizeTablesParams):
with ClickhouseConnection(params.clickhouse) as db:
for table_name in params.table_names:
wait_for_mutations(db, params.table_names)
# Wait for mutation to complete so that we don't run into out of
# space issues while doing the batch inserts
wait_for_mutations(db, table_name=table_name)
log.info(f"waiting for mutations to finish on {table_name}")
db.execute(f"OPTIMIZE TABLE {table_name}")

Expand Down
1 change: 1 addition & 0 deletions oonipipeline/src/oonipipeline/temporal/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ def maybe_delete_prev_range(db: ClickhouseConnection, prev_range: PrevRange) ->
if not prev_range.max_created_at or not prev_range.min_created_at:
return ""

# Before deleting, we need to wait for all the mutations to be done
wait_for_mutations(db, prev_range.table_name)
where, q_args = prev_range.format_query()

Expand Down

0 comments on commit 4dbf122

Please sign in to comment.