Skip to content

Commit

Permalink
Add debug info to delete previous range queries
Browse files Browse the repository at this point in the history
  • Loading branch information
hellais committed Sep 5, 2024
1 parent ade571e commit 93c7587
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -262,8 +262,10 @@ class DeletePreviousRangeParams:


@activity.defn
def delete_previous_range(params: DeletePreviousRangeParams) -> None:
def delete_previous_range(params: DeletePreviousRangeParams) -> List[str]:
delete_queries = []
with ClickhouseConnection(params.clickhouse) as db:
for pr in params.previous_ranges:
log.info("deleting previous range of {pr}")
maybe_delete_prev_range(db=db, prev_range=pr)
delete_queries.append(maybe_delete_prev_range(db=db, prev_range=pr))
return delete_queries
7 changes: 4 additions & 3 deletions oonipipeline/src/oonipipeline/temporal/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,13 @@ def format_query(self):
return where, q_args


def maybe_delete_prev_range(db: ClickhouseConnection, prev_range: PrevRange):
def maybe_delete_prev_range(db: ClickhouseConnection, prev_range: PrevRange) -> str:
"""
We perform a lightweight delete of all the rows which have been
regenerated, so we don't have any duplicates in the table
"""
if not prev_range.max_created_at or not prev_range.min_created_at:
return
return ""

# Disabled due to: https://github.com/ClickHouse/ClickHouse/issues/40651
# db.execute("SET allow_experimental_lightweight_delete = true;")
Expand All @@ -87,7 +87,8 @@ def maybe_delete_prev_range(db: ClickhouseConnection, prev_range: PrevRange):

q = f"ALTER TABLE {prev_range.table_name} DELETE "
final_query = q + where
return db.execute(final_query, q_args)
db.execute(final_query, q_args)
return final_query


def get_prev_range(
Expand Down

0 comments on commit 93c7587

Please sign in to comment.