From 0d0fb165b537f2d751aba37140a9a6025bb310f7 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Tue, 24 Dec 2024 11:42:13 -0800 Subject: [PATCH] add type annotation; test report --- metadata-ingestion/src/datahub/ingestion/source/aws/glue.py | 2 +- metadata-ingestion/tests/unit/glue/test_glue_source.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index cdcdd0221934bc..7a5ed154d40bc7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -220,7 +220,7 @@ def platform_validator(cls, v: str) -> str: class GlueSourceReport(StaleEntityRemovalSourceReport): tables_scanned = 0 filtered: List[str] = dataclass_field(default_factory=list) - databases = EntityFilterReport.field(type="database") + databases: EntityFilterReport = EntityFilterReport.field(type="database") num_job_script_location_missing: int = 0 num_job_script_location_invalid: int = 0 diff --git a/metadata-ingestion/tests/unit/glue/test_glue_source.py b/metadata-ingestion/tests/unit/glue/test_glue_source.py index aff169aaf60f36..9e3f260a23f1c8 100644 --- a/metadata-ingestion/tests/unit/glue/test_glue_source.py +++ b/metadata-ingestion/tests/unit/glue/test_glue_source.py @@ -325,9 +325,10 @@ def format_databases(databases): expected = [flights_database, test_database, empty_database] actual = all_catalogs_source.get_all_databases() assert format_databases(actual) == format_databases(expected) + assert all_catalogs_source.report.databases.dropped_entities.as_obj() == [] catalog_id = "123412341234" - single_catalog_source = GlueSource( + single_catalog_source: GlueSource = GlueSource( config=GlueSourceConfig(catalog_id=catalog_id, aws_region="us-west-2"), ctx=PipelineContext(run_id="glue-source-test"), ) @@ -339,6 +340,9 @@ def format_databases(databases): expected = [flights_database, test_database] actual = single_catalog_source.get_all_databases() assert format_databases(actual) == format_databases(expected) + assert single_catalog_source.report.databases.dropped_entities.as_obj() == [ + "empty-database" + ] @freeze_time(FROZEN_TIME)