From 54d53cfc625a1583fbd5e93b4b6bfff55b9897c6 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 11 Dec 2024 18:24:18 +0530 Subject: [PATCH] fix(ingest/gc): minor tweak gc source (#12093) --- .../src/datahub/ingestion/source/gc/datahub_gc.py | 10 +++++----- .../ingestion/source/gc/soft_deleted_entity_cleanup.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py index 52807ca2a3f026..814f65ecb45cf0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py @@ -153,11 +153,6 @@ def get_workunits_internal( self.truncate_indices() except Exception as e: self.report.failure("While trying to truncate indices ", exc=e) - if self.dataprocess_cleanup: - try: - yield from self.dataprocess_cleanup.get_workunits_internal() - except Exception as e: - self.report.failure("While trying to cleanup data process ", exc=e) if self.soft_deleted_entities_cleanup: try: self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities() @@ -170,6 +165,11 @@ def get_workunits_internal( self.execution_request_cleanup.run() except Exception as e: self.report.failure("While trying to cleanup execution request ", exc=e) + if self.dataprocess_cleanup: + try: + yield from self.dataprocess_cleanup.get_workunits_internal() + except Exception as e: + self.report.failure("While trying to cleanup data process ", exc=e) yield from [] def truncate_indices(self) -> None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py index 3b367cdea58134..bb4ab753543b7b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py @@ -60,7 +60,7 @@ class SoftDeletedEntitiesCleanupConfig(ConfigModel): description="Query to filter entities", ) limit_entities_delete: Optional[int] = Field( - 10000, description="Max number of entities to delete." + 25000, description="Max number of entities to delete." ) runtime_limit_seconds: Optional[int] = Field(