From cbd8e14b78f720a3584b0390d272a8a4321109de Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 21 Mar 2023 05:04:22 +0530 Subject: [PATCH] feat(ingest): add auto_materialize_referenced_tags helper (#7626) --- .../ingestion/source/bigquery_v2/bigquery.py | 15 +- .../ingestion/source/dbt/dbt_common.py | 9 +- .../src/datahub/utilities/source_helpers.py | 38 ++++ .../src/datahub/utilities/urns/urn_iter.py | 51 ++++- .../dbt_enabled_with_schemas_mces_golden.json | 201 ++++++++++++++---- .../dbt_test_column_meta_mapping_golden.json | 30 +++ ...th_complex_owner_patterns_mces_golden.json | 149 ++++++++++--- ...th_data_platform_instance_mces_golden.json | 156 ++++++++++---- ...h_non_incremental_lineage_mces_golden.json | 156 ++++++++++---- ..._target_platform_instance_mces_golden.json | 156 ++++++++++---- 10 files changed, 762 insertions(+), 199 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index c57a04ba20926d..a2c07af7df711c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -122,6 +122,7 @@ from datahub.utilities.perf_timer import PerfTimer from datahub.utilities.registries.domain_registry import DomainRegistry from datahub.utilities.source_helpers import ( + auto_materialize_referenced_tags, auto_stale_entity_removal, auto_status_aspect, auto_workunit_reporter, @@ -551,12 +552,14 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: yield from self.generate_lineage(project.id) def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_stale_entity_removal( - self.stale_entity_removal_handler, - auto_workunit_reporter( - self.report, - auto_status_aspect(self.get_workunits_internal()), - ), + return auto_materialize_referenced_tags( + auto_stale_entity_removal( + self.stale_entity_removal_handler, + auto_workunit_reporter( + self.report, + auto_status_aspect(self.get_workunits_internal()), + ), + ) ) def _get_projects(self, conn: bigquery.Client) -> List[BigqueryProject]: diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index 836508a0542184..7e3ce384cd7082 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -108,6 +108,7 @@ from datahub.specific.dataset import DatasetPatchBuilder from datahub.utilities.mapping import Constants, OperationProcessor from datahub.utilities.source_helpers import ( + auto_materialize_referenced_tags, auto_stale_entity_removal, auto_status_aspect, ) @@ -888,9 +889,11 @@ def load_nodes(self) -> Tuple[List[DBTNode], Dict[str, Optional[str]]]: raise NotImplementedError() def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_stale_entity_removal( - self.stale_entity_removal_handler, - auto_status_aspect(self.get_workunits_internal()), + return auto_materialize_referenced_tags( + auto_stale_entity_removal( + self.stale_entity_removal_handler, + auto_status_aspect(self.get_workunits_internal()), + ) ) def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: diff --git a/metadata-ingestion/src/datahub/utilities/source_helpers.py b/metadata-ingestion/src/datahub/utilities/source_helpers.py index b5f4e6f927cf98..7cad501982cfdd 100644 --- a/metadata-ingestion/src/datahub/utilities/source_helpers.py +++ b/metadata-ingestion/src/datahub/utilities/source_helpers.py @@ -10,13 +10,18 @@ MetadataChangeEventClass, MetadataChangeProposalClass, StatusClass, + TagKeyClass, ) +from datahub.utilities.urns.tag_urn import TagUrn from datahub.utilities.urns.urn import guess_entity_type +from datahub.utilities.urns.urn_iter import list_urns def auto_workunit( stream: Iterable[Union[MetadataChangeEventClass, MetadataChangeProposalWrapper]] ) -> Iterable[MetadataWorkUnit]: + """Convert a stream of MCEs and MCPs to a stream of :class:`MetadataWorkUnit`s.""" + for item in stream: if isinstance(item, MetadataChangeEventClass): yield MetadataWorkUnit(id=f"{item.proposedSnapshot.urn}/mce", mce=item) @@ -109,3 +114,36 @@ def auto_workunit_reporter( for wu in stream: report.report_workunit(wu) yield wu + + +def auto_materialize_referenced_tags( + stream: Iterable[MetadataWorkUnit], + active: bool = True, +) -> Iterable[MetadataWorkUnit]: + """For all references to tags, emit a tag key aspect to ensure that the tag exists in our backend.""" + + if not active: + yield from stream + return + + referenced_tags = set() + tags_with_aspects = set() + + for wu in stream: + for urn in list_urns(wu.metadata): + if guess_entity_type(urn) == "tag": + referenced_tags.add(urn) + + urn = wu.get_urn() + if guess_entity_type(urn) == "tag": + tags_with_aspects.add(urn) + + yield wu + + for urn in referenced_tags - tags_with_aspects: + tag_urn = TagUrn.create_from_string(urn) + + yield MetadataChangeProposalWrapper( + entityUrn=urn, + aspect=TagKeyClass(name=tag_urn.get_entity_id()[0]), + ).as_workunit() diff --git a/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py b/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py index d9bbc941ec5e2a..261f95331af612 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py +++ b/metadata-ingestion/src/datahub/utilities/urns/urn_iter.py @@ -2,6 +2,7 @@ from avro.schema import Field, RecordSchema +from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.metadata.schema_classes import DictWrapper from datahub.utilities.urns.dataset_urn import DatasetUrn from datahub.utilities.urns.urn import Urn, guess_entity_type @@ -9,7 +10,15 @@ _Path = List[Union[str, int]] -def list_urns_with_path(model: DictWrapper) -> List[Tuple[str, _Path]]: +def _add_prefix_to_paths( + prefix: _Path, items: List[Tuple[str, _Path]] +) -> List[Tuple[str, _Path]]: + return [(urn, [*prefix, *path]) for urn, path in items] + + +def list_urns_with_path( + model: Union[DictWrapper, MetadataChangeProposalWrapper] +) -> List[Tuple[str, _Path]]: """List urns in the given model with their paths. Args: @@ -19,10 +28,26 @@ def list_urns_with_path(model: DictWrapper) -> List[Tuple[str, _Path]]: A list of tuples of the form (urn, path), where path is a list of keys. """ - schema: RecordSchema = model.RECORD_SCHEMA - urns: List[Tuple[str, _Path]] = [] + if isinstance(model, MetadataChangeProposalWrapper): + if model.entityUrn: + urns.append((model.entityUrn, ["urn"])) + if model.entityKeyAspect: + urns.extend( + _add_prefix_to_paths( + ["entityKeyAspect"], list_urns_with_path(model.entityKeyAspect) + ) + ) + if model.aspect: + urns.extend( + _add_prefix_to_paths(["aspect"], list_urns_with_path(model.aspect)) + ) + + return urns + + schema: RecordSchema = model.RECORD_SCHEMA + for key, value in model.items(): if not value: continue @@ -31,13 +56,13 @@ def list_urns_with_path(model: DictWrapper) -> List[Tuple[str, _Path]]: is_urn = field_schema.get_prop("Urn") is not None if isinstance(value, DictWrapper): - for urn, path in list_urns_with_path(value): - urns.append((urn, [key, *path])) + urns.extend(_add_prefix_to_paths([key], list_urns_with_path(value))) elif isinstance(value, list): for i, item in enumerate(value): if isinstance(item, DictWrapper): - for urn, path in list_urns_with_path(item): - urns.append((urn, [key, i, *path])) + urns.extend( + _add_prefix_to_paths([key, i], list_urns_with_path(item)) + ) elif is_urn: urns.append((item, [key, i])) elif is_urn: @@ -46,6 +71,18 @@ def list_urns_with_path(model: DictWrapper) -> List[Tuple[str, _Path]]: return urns +def list_urns(model: Union[DictWrapper, MetadataChangeProposalWrapper]) -> List[str]: + """List urns in the given model. + + Args: + model: The model to list urns from. + + Returns: A list of URNs contained in the given model. + """ + + return [urn for urn, _ in list_urns_with_path(model)] + + def transform_urns(model: DictWrapper, func: Callable[[str], str]) -> None: """ Rewrites all URNs in the given object according to the given function. diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json index 27244ff889c4f4..ceda22da550310 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json @@ -5,8 +5,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "ephemeral", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -137,7 +141,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -145,7 +149,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" }, { @@ -153,7 +157,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" } ] @@ -180,8 +184,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "table", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -318,7 +326,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "type": "TRANSFORMED" }, { @@ -326,7 +334,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" } ] @@ -353,8 +361,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"view\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "view", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -568,8 +580,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "table", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -699,8 +715,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -864,8 +883,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1047,8 +1069,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1170,8 +1195,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1305,8 +1333,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1448,8 +1479,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1655,8 +1689,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1814,8 +1851,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1994,8 +2034,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2153,8 +2196,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2312,8 +2358,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2471,8 +2520,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2665,5 +2717,80 @@ "lastObserved": 1643871600000, "runId": "dbt-test-with-schemas-dbt-enabled" } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:test_tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:test_tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-schemas-dbt-enabled" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:my_query_tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:my_query_tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-schemas-dbt-enabled" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:column_tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:column_tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-schemas-dbt-enabled" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:int_meta_property", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:int_meta_property" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-schemas-dbt-enabled" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:has_pii_test", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:has_pii_test" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-schemas-dbt-enabled" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json index 7028ed0b6125aa..8b84662dc2acc6 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json @@ -2929,5 +2929,35 @@ "lastObserved": 1643871600000, "runId": "dbt-column-meta-mapping" } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:tag_from_dbt", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:tag_from_dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-column-meta-mapping" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:sensitive", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:sensitive" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-column-meta-mapping" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json index b9615c1f63dd5b..d5a401f36bc29a 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json @@ -5,8 +5,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "ephemeral", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -101,7 +105,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -109,7 +113,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" }, { @@ -117,7 +121,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" } ] @@ -144,8 +148,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "table", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -265,7 +273,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "type": "TRANSFORMED" }, { @@ -273,7 +281,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" } ] @@ -300,8 +308,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"view\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "view", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -515,8 +527,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "table", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -646,8 +662,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -808,8 +827,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -991,8 +1013,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1114,8 +1139,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1249,8 +1277,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1389,8 +1420,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1596,8 +1630,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1755,8 +1792,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1932,8 +1972,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2091,8 +2134,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2250,8 +2296,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2444,5 +2493,35 @@ "lastObserved": 1643871600000, "runId": "dbt-test-with-complex-owner-patterns" } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:test_tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:test_tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-complex-owner-patterns" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:column_tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:column_tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-complex-owner-patterns" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json index b3ab9597fcee8a..3a99cd88de554a 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json @@ -5,8 +5,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "ephemeral", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -102,7 +106,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -110,7 +114,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.city,PROD)", "type": "TRANSFORMED" }, { @@ -118,7 +122,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD)", "type": "TRANSFORMED" } ] @@ -145,8 +149,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "table", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -266,7 +274,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD)", "type": "TRANSFORMED" }, { @@ -274,7 +282,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" } ] @@ -301,8 +309,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"view\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "view", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -516,8 +528,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "table", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -647,8 +663,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -809,8 +828,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -992,8 +1014,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1115,8 +1140,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1250,8 +1278,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1390,8 +1421,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1597,8 +1631,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1756,8 +1793,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1933,8 +1973,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2092,8 +2135,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2251,8 +2297,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2410,8 +2459,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2604,5 +2656,35 @@ "lastObserved": 1643871600000, "runId": "dbt-test-with-data-platform-instance" } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:test_tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:test_tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-data-platform-instance" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:column_tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:column_tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-data-platform-instance" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json index 9f418aba5a40e2..f1a5ce51f9c526 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json @@ -5,8 +5,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "ephemeral", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -102,7 +106,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -110,7 +114,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" }, { @@ -118,7 +122,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" } ] @@ -145,8 +149,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "table", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -266,7 +274,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "type": "TRANSFORMED" }, { @@ -274,7 +282,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" } ] @@ -301,8 +309,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"view\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "view", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -516,8 +528,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "table", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -647,8 +663,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -809,8 +828,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -992,8 +1014,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1115,8 +1140,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1250,8 +1278,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1390,8 +1421,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1597,8 +1631,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1756,8 +1793,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1933,8 +1973,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2092,8 +2135,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2251,8 +2297,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2410,8 +2459,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2643,5 +2695,35 @@ "lastObserved": 1643871600000, "runId": "dbt-test-with-non-incremental-lineage" } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:test_tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:test_tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:column_tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:column_tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-non-incremental-lineage" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json index a07b04dd8dda3e..56f90483b44545 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json @@ -5,8 +5,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "ephemeral", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -102,7 +106,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -110,7 +114,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" }, { @@ -118,7 +122,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" } ] @@ -145,8 +149,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "table", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -266,7 +274,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "type": "TRANSFORMED" }, { @@ -274,7 +282,7 @@ "time": 1643871600000, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" } ] @@ -301,8 +309,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"view\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "view", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -516,8 +528,12 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "table", + "view" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -647,8 +663,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -809,8 +828,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -992,8 +1014,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1115,8 +1140,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1250,8 +1278,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1390,8 +1421,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1597,8 +1631,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1756,8 +1793,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1933,8 +1973,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2092,8 +2135,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2251,8 +2297,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2410,8 +2459,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "source" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -2604,5 +2656,35 @@ "lastObserved": 1643871600000, "runId": "dbt-test-with-target-platform-instance" } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:test_tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:test_tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-target-platform-instance" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:column_tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:column_tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-test-with-target-platform-instance" + } } ] \ No newline at end of file