diff --git a/metaphor/snowflake/extractor.py b/metaphor/snowflake/extractor.py index 91d2d94a..44062e40 100644 --- a/metaphor/snowflake/extractor.py +++ b/metaphor/snowflake/extractor.py @@ -45,6 +45,7 @@ SourceInfo, SQLSchema, SystemTag, + SystemTags, SystemTagSource, ) from metaphor.snowflake import auth @@ -409,8 +410,11 @@ def _add_system_tag( logical_id = HierarchyLogicalID(path=path) self._hierarchies.setdefault( - hierarchy_key, Hierarchy(logical_id=logical_id, system_tags=[]) - ).system_tags.append( + hierarchy_key, + Hierarchy( + logical_id=logical_id, system_tags=SystemTags(tags=[]) + ), # SystemTags.tags should never be empty + ).system_tags.tags.append( SystemTag( key=tag_key, system_tag_source=SystemTagSource.SNOWFLAKE, diff --git a/poetry.lock b/poetry.lock index bc6a9bbc..71fa38a1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -472,8 +472,8 @@ files = [ jmespath = ">=0.7.1,<2.0.0" python-dateutil = ">=2.1,<3.0.0" urllib3 = [ - {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""}, + {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, ] [package.extras] @@ -881,15 +881,15 @@ files = [ alembic = ">=1.0.11,<2.0.0" lz4 = ">=4.0.2,<5.0.0" numpy = [ - {version = ">=1.16.6", markers = "python_version >= \"3.7\" and python_version < \"3.11\""}, {version = ">=1.23.4", markers = "python_version >= \"3.11\""}, + {version = ">=1.16.6", markers = "python_version >= \"3.7\" and python_version < \"3.11\""}, ] oauthlib = ">=3.1.0,<4.0.0" openpyxl = ">=3.0.10,<4.0.0" pandas = {version = ">=1.2.5,<3.0.0", markers = "python_version >= \"3.8\""} pyarrow = [ - {version = ">=6.0.0", markers = "python_version >= \"3.7\" and python_version < \"3.11\""}, {version = ">=10.0.1", markers = "python_version >= \"3.11\""}, + {version = ">=6.0.0", markers = "python_version >= \"3.7\" and python_version < \"3.11\""}, ] requests = ">=2.18.1,<3.0.0" sqlalchemy = ">=1.3.24,<2.0.0" @@ -934,9 +934,9 @@ openapi-spec-validator = ">=0.2.8,<=0.5.7" packaging = "*" prance = ">=0.18.2" pydantic = [ + {version = ">=1.10.0,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version >= \"3.11\" and python_version < \"4.0\""}, {version = ">=1.5.1,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version < \"3.10\""}, {version = ">=1.9.0,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, - {version = ">=1.10.0,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version >= \"3.11\" and python_version < \"4.0\""}, ] PySnooper = ">=0.4.1,<2.0.0" toml = ">=0.10.0,<1.0.0" @@ -1203,12 +1203,12 @@ files = [ google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" grpcio = [ - {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, ] grpcio-status = [ - {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" @@ -1255,8 +1255,8 @@ files = [ [package.dependencies] google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} proto-plus = [ - {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" @@ -1291,8 +1291,8 @@ google-api-core = {version = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev", extras = google-cloud-core = ">=1.6.0,<3.0.0dev" google-resumable-media = ">=0.6.0,<3.0dev" grpcio = [ - {version = ">=1.47.0,<2.0dev", markers = "python_version < \"3.11\""}, {version = ">=1.49.1,<2.0dev", markers = "python_version >= \"3.11\""}, + {version = ">=1.47.0,<2.0dev", markers = "python_version < \"3.11\""}, ] packaging = ">=20.0.0" proto-plus = ">=1.15.0,<2.0.0dev" @@ -1346,8 +1346,8 @@ google-cloud-audit-log = ">=0.1.0,<1.0.0dev" google-cloud-core = ">=2.0.0,<3.0.0dev" grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" proto-plus = [ - {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" @@ -2210,13 +2210,13 @@ files = [ [[package]] name = "metaphor-models" -version = "0.30.3" +version = "0.30.5" description = "" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "metaphor_models-0.30.3-py3-none-any.whl", hash = "sha256:5e2c630141f17f24a88299ac0f1da92d38e12130608b06e3f69ad3be25c3e01c"}, - {file = "metaphor_models-0.30.3.tar.gz", hash = "sha256:6d94d0be80a73131db781cadc15a24743a33218b80207097969fccc2c8d68565"}, + {file = "metaphor_models-0.30.5-py3-none-any.whl", hash = "sha256:a54b6d16339d6314d6f70e015b7dcba8fcee8306a92c26f07008fe87adaca37c"}, + {file = "metaphor_models-0.30.5.tar.gz", hash = "sha256:6ef54da7ba603bea0c5e48da947c33437e15164a5a7b43852fa6dfce34eb83e4"}, ] [[package]] @@ -2809,9 +2809,9 @@ files = [ [package.dependencies] numpy = [ + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, {version = ">=1.20.3", markers = "python_version < \"3.10\""}, {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -4702,4 +4702,4 @@ unity-catalog = ["databricks-sdk", "databricks-sql-connector"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "d1ab022b20de454a0b5f317f277163c912e9e515c8f8c0a2938a7f4c059479a4" +content-hash = "9dbbf225efd84f9a15438ad0165234a922e811fd75c0a5d8e23e4923151de5e4" diff --git a/pyproject.toml b/pyproject.toml index b2113403..b7fd662d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "metaphor-connectors" -version = "0.13.48" +version = "0.13.49" license = "Apache-2.0" description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app." authors = ["Metaphor "] @@ -30,7 +30,7 @@ google-cloud-logging = { version = "^3.5.0", optional = true } jsonschema = "^4.18.6" lkml = { version = "^1.3.1", optional = true } looker-sdk = { version = "^23.6.0", optional = true } -metaphor-models = "0.30.3" +metaphor-models = "0.30.5" msal = { version = "^1.20.0", optional = true } msgraph-beta-sdk = { version = "1.0.0", optional = true } pycarlo = { version = "^0.8.1", optional = true } diff --git a/tests/snowflake/test_extractor.py b/tests/snowflake/test_extractor.py index d50f8f01..f7f82810 100644 --- a/tests/snowflake/test_extractor.py +++ b/tests/snowflake/test_extractor.py @@ -14,6 +14,7 @@ QueriedDataset, SchemaField, SystemTag, + SystemTags, SystemTagSource, ) from metaphor.snowflake.config import SnowflakeQueryLogConfig, SnowflakeRunConfig @@ -271,12 +272,18 @@ def test_fetch_hierarchy_system_tags(mock_connect: MagicMock): path=[DataPlatform.SNOWFLAKE.value, table_name] ) assert db_hierarchy.system_tags is not None - assert db_hierarchy.system_tags == [ - SystemTag(key="foo", system_tag_source=SystemTagSource.SNOWFLAKE, value="bar"), - SystemTag( - key="grault", system_tag_source=SystemTagSource.SNOWFLAKE, value="garply" - ), - ] + assert db_hierarchy.system_tags == SystemTags( + tags=[ + SystemTag( + key="foo", system_tag_source=SystemTagSource.SNOWFLAKE, value="bar" + ), + SystemTag( + key="grault", + system_tag_source=SystemTagSource.SNOWFLAKE, + value="garply", + ), + ] + ) schema_hierarchy = extractor._hierarchies[ dataset_normalized_name(database, table_name) ] @@ -284,12 +291,16 @@ def test_fetch_hierarchy_system_tags(mock_connect: MagicMock): path=[DataPlatform.SNOWFLAKE.value, database, table_name] ) assert schema_hierarchy.system_tags is not None - assert schema_hierarchy.system_tags == [ - SystemTag(key="baz", system_tag_source=SystemTagSource.SNOWFLAKE, value="qux"), - SystemTag( - key="quux", system_tag_source=SystemTagSource.SNOWFLAKE, value="corge" - ), - ] + assert schema_hierarchy.system_tags == SystemTags( + tags=[ + SystemTag( + key="baz", system_tag_source=SystemTagSource.SNOWFLAKE, value="qux" + ), + SystemTag( + key="quux", system_tag_source=SystemTagSource.SNOWFLAKE, value="corge" + ), + ] + ) @patch("metaphor.snowflake.auth.connect")