diff --git a/metaphor/unity_catalog/profile/extractor.py b/metaphor/unity_catalog/profile/extractor.py index f82f4505..b4287c37 100644 --- a/metaphor/unity_catalog/profile/extractor.py +++ b/metaphor/unity_catalog/profile/extractor.py @@ -262,7 +262,7 @@ def get_value_from_row(key: str) -> Optional[float]: None, ) if value: - if value == "NULL": + if value in ["NULL", "Infinity"]: return None return safe_float(value) return value diff --git a/pyproject.toml b/pyproject.toml index 2cf22b49..eb9a0d86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "metaphor-connectors" -version = "0.14.56" +version = "0.14.57" license = "Apache-2.0" description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app." authors = ["Metaphor "] diff --git a/tests/unity_catalog/profile/expected.json b/tests/unity_catalog/profile/expected.json index cdb14a92..746421ca 100644 --- a/tests/unity_catalog/profile/expected.json +++ b/tests/unity_catalog/profile/expected.json @@ -5,8 +5,7 @@ { "distinctValueCount": 1234.0, "fieldPath": "col2", - "maxValue": 9999.0, - "minValue": -9999.0 + "maxValue": 9999.0 } ] }, diff --git a/tests/unity_catalog/profile/test_extractor.py b/tests/unity_catalog/profile/test_extractor.py index 94813454..b28818ee 100644 --- a/tests/unity_catalog/profile/test_extractor.py +++ b/tests/unity_catalog/profile/test_extractor.py @@ -69,7 +69,7 @@ async def test_extractor( mock_col2_stats = [ SimpleNamespace(info_name="distinct_count", info_value="1234"), SimpleNamespace(info_name="max", info_value="9999"), - SimpleNamespace(info_name="min", info_value="-9999"), + SimpleNamespace(info_name="min", info_value="Infinity"), SimpleNamespace(info_name="num_nulls", info_value="NULL"), ] @@ -184,7 +184,7 @@ async def test_should_handle_exception_column_stat( mock_col2_stats = [ SimpleNamespace(info_name="distinct_count", info_value="1234"), SimpleNamespace(info_name="max", info_value="9999"), - SimpleNamespace(info_name="min", info_value="-9999"), + SimpleNamespace(info_name="min", info_value="Infinity"), SimpleNamespace(info_name="num_nulls", info_value="NULL"), ]