From 413331933a9fd7f56931faad50587ebc2b0de4cb Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 11 Oct 2024 19:08:25 -0500 Subject: [PATCH 01/40] fix(spark-lineage-legacy): fix check jar script (#11608) --- .../java/spark-lineage-legacy/scripts/check_jar.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh b/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh index 81d6a541d1c2a0..854c4227d08d93 100755 --- a/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh +++ b/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh @@ -40,7 +40,8 @@ jar -tvf $jarFile |\ grep -v "rootdoc.txt" |\ grep -v "VersionInfo.java" |\ grep -v "mime.types" |\ - grep -v "com/ibm/.*" + grep -v "com/ibm/.*" |\ + grep -v "google/" if [ $? -ne 0 ]; then From 38ac1007d6a8e1e22291b9f0a1491734dfd44812 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Mon, 14 Oct 2024 01:25:19 -0700 Subject: [PATCH 02/40] fix(sdk): platform resource api for non existent resources (#11610) --- .../entities/platformresource/platform_resource.py | 7 +++++++ .../platform_resources/test_platform_resource.py | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py b/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py index 2b730ccb86f513..1556a67a9e5555 100644 --- a/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py +++ b/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py @@ -186,10 +186,17 @@ def to_datahub(self, graph_client: DataHubGraph) -> None: def from_datahub( cls, graph_client: DataHubGraph, key: Union[PlatformResourceKey, str] ) -> Optional["PlatformResource"]: + """ + Fetches a PlatformResource from the graph given a key. + Key can be either a PlatformResourceKey object or an urn string. + Returns None if the resource is not found. + """ if isinstance(key, PlatformResourceKey): urn = PlatformResourceUrn(id=key.id) else: urn = PlatformResourceUrn.from_string(key) + if not graph_client.exists(str(urn)): + return None platform_resource = graph_client.get_entity_semityped(str(urn)) return cls( id=urn.id, diff --git a/smoke-test/tests/platform_resources/test_platform_resource.py b/smoke-test/tests/platform_resources/test_platform_resource.py index 09d25031795728..7c53f72d843c93 100644 --- a/smoke-test/tests/platform_resources/test_platform_resource.py +++ b/smoke-test/tests/platform_resources/test_platform_resource.py @@ -99,3 +99,16 @@ def test_platform_resource_search(graph_client, test_id, cleanup_resources): ] assert len(search_results) == 1 assert search_results[0] == platform_resource + + +def test_platform_resource_non_existent(graph_client, test_id): + key = PlatformResourceKey( + platform=f"test_platform_{test_id}", + resource_type=f"test_resource_type_{test_id}", + primary_key=f"test_primary_key_{test_id}", + ) + platform_resource = PlatformResource.from_datahub( + key=key, + graph_client=graph_client, + ) + assert platform_resource is None From 3387110b411d91c7a08baaab76e29caa3bcb3a28 Mon Sep 17 00:00:00 2001 From: skrydal Date: Mon, 14 Oct 2024 11:35:34 +0200 Subject: [PATCH 03/40] fix(ingestion/redshift): Fix for Redshift COPY-based lineage (#11552) --- .../ingestion/source/redshift/lineage_v2.py | 13 ++- .../ingestion/source/redshift/query.py | 89 ++++++------------- .../sql_parsing/sql_parsing_aggregator.py | 4 +- 3 files changed, 41 insertions(+), 65 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py index 4b7f710beed08f..4df64c80bad8a8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py @@ -334,19 +334,26 @@ def _process_view_lineage(self, lineage_row: LineageRow) -> None: ) def _process_copy_command(self, lineage_row: LineageRow) -> None: - source = self._lineage_v1._get_sources( + logger.debug(f"Processing COPY command for lineage row: {lineage_row}") + sources = self._lineage_v1._get_sources( lineage_type=LineageCollectorType.COPY, db_name=self.database, source_schema=None, source_table=None, ddl=None, filename=lineage_row.filename, - )[0] + ) + logger.debug(f"Recognized sources: {sources}") + source = sources[0] if not source: + logger.debug("Ignoring command since couldn't recognize proper source") return s3_urn = source[0].urn - + logger.debug(f"Recognized s3 dataset urn: {s3_urn}") if not lineage_row.target_schema or not lineage_row.target_table: + logger.debug( + f"Didn't find target schema (found: {lineage_row.target_schema}) or target table (found: {lineage_row.target_table})" + ) return target = self._make_filtered_target(lineage_row) if not target: diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py index affbcd00b5107b..39370b93b561c5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py @@ -283,6 +283,34 @@ def alter_table_rename_query( AND SYS.query_text ILIKE '%alter table % rename to %' """ + @staticmethod + def list_copy_commands_sql( + db_name: str, start_time: datetime, end_time: datetime + ) -> str: + return """ + select + distinct + "schema" as target_schema, + "table" as target_table, + c.file_name as filename + from + SYS_QUERY_DETAIL as si + join SYS_LOAD_DETAIL as c on + si.query_id = c.query_id + join SVV_TABLE_INFO sti on + sti.table_id = si.table_id + where + database = '{db_name}' + and si.start_time >= '{start_time}' + and si.start_time < '{end_time}' + order by target_schema, target_table, si.start_time asc + """.format( + # We need the original database name for filtering + db_name=db_name, + start_time=start_time.strftime(redshift_datetime_format), + end_time=end_time.strftime(redshift_datetime_format), + ) + @staticmethod def additional_table_metadata_query() -> str: raise NotImplementedError @@ -317,12 +345,6 @@ def list_insert_create_queries_sql( ) -> str: raise NotImplementedError - @staticmethod - def list_copy_commands_sql( - db_name: str, start_time: datetime, end_time: datetime - ) -> str: - raise NotImplementedError - class RedshiftProvisionedQuery(RedshiftCommonQuery): @staticmethod @@ -536,34 +558,6 @@ def list_insert_create_queries_sql( end_time=end_time.strftime(redshift_datetime_format), ) - @staticmethod - def list_copy_commands_sql( - db_name: str, start_time: datetime, end_time: datetime - ) -> str: - return """ - select - distinct - "schema" as target_schema, - "table" as target_table, - filename - from - stl_insert as si - join stl_load_commits as c on - si.query = c.query - join SVV_TABLE_INFO sti on - sti.table_id = tbl - where - database = '{db_name}' - and si.starttime >= '{start_time}' - and si.starttime < '{end_time}' - order by target_schema, target_table, starttime asc - """.format( - # We need the original database name for filtering - db_name=db_name, - start_time=start_time.strftime(redshift_datetime_format), - end_time=end_time.strftime(redshift_datetime_format), - ) - @staticmethod def temp_table_ddl_query(start_time: datetime, end_time: datetime) -> str: start_time_str: str = start_time.strftime(redshift_datetime_format) @@ -941,33 +935,6 @@ def list_insert_create_queries_sql( # when loading from s3 using prefix with a single file it produces 2 lines (for file and just directory) - also # behaves like this when run in the old way - @staticmethod - def list_copy_commands_sql( - db_name: str, start_time: datetime, end_time: datetime - ) -> str: - return """ - select - distinct - "schema" as target_schema, - "table" as target_table, - c.file_name - from - SYS_QUERY_DETAIL as si - join SYS_LOAD_DETAIL as c on - si.query_id = c.query_id - join SVV_TABLE_INFO sti on - sti.table_id = si.table_id - where - database = '{db_name}' - and si.start_time >= '{start_time}' - and si.start_time < '{end_time}' - order by target_schema, target_table, si.start_time asc - """.format( - # We need the original database name for filtering - db_name=db_name, - start_time=start_time.strftime(redshift_datetime_format), - end_time=end_time.strftime(redshift_datetime_format), - ) # handles "create table IF ..." statements wrong probably - "create command" field contains only "create table if" in such cases # also similar happens if for example table name contains special characters quoted with " i.e. "test-table1" diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py index 52934f9f72a70e..5f2709fe426605 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py @@ -613,7 +613,9 @@ def add_known_lineage_mapping( upstream_urn: The upstream dataset URN. downstream_urn: The downstream dataset URN. """ - + logger.debug( + f"Adding lineage to the map, downstream: {downstream_urn}, upstream: {upstream_urn}" + ) self.report.num_known_mapping_lineage += 1 # We generate a fake "query" object to hold the lineage. From b74ba11d937ef4b902f320e1bdb39a7ece35ffc4 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Mon, 14 Oct 2024 18:21:05 +0530 Subject: [PATCH 04/40] fix(ingest/delta-lake): skip file count if require_files is false (#11611) --- .../src/datahub/ingestion/source/delta_lake/source.py | 5 ++--- metadata-ingestion/tests/unit/test_mlflow_source.py | 6 ++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py index 6a52d8fdd89057..98133ca69011e7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py @@ -223,15 +223,14 @@ def ingest_table( ) customProperties = { - "number_of_files": str(get_file_count(delta_table)), "partition_columns": str(delta_table.metadata().partition_columns), "table_creation_time": str(delta_table.metadata().created_time), "id": str(delta_table.metadata().id), "version": str(delta_table.version()), "location": self.source_config.complete_path, } - if not self.source_config.require_files: - del customProperties["number_of_files"] # always 0 + if self.source_config.require_files: + customProperties["number_of_files"] = str(get_file_count(delta_table)) dataset_properties = DatasetPropertiesClass( description=delta_table.metadata().description, diff --git a/metadata-ingestion/tests/unit/test_mlflow_source.py b/metadata-ingestion/tests/unit/test_mlflow_source.py index ae5a42bad229d2..d213dd92352e62 100644 --- a/metadata-ingestion/tests/unit/test_mlflow_source.py +++ b/metadata-ingestion/tests/unit/test_mlflow_source.py @@ -1,6 +1,6 @@ import datetime from pathlib import Path -from typing import Any, TypeVar, Union +from typing import Any, Union import pytest from mlflow import MlflowClient @@ -11,8 +11,6 @@ from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.source.mlflow import MLflowConfig, MLflowSource -T = TypeVar("T") - @pytest.fixture def tracking_uri(tmp_path: Path) -> str: @@ -46,7 +44,7 @@ def model_version( ) -def dummy_search_func(page_token: Union[None, str], **kwargs: Any) -> PagedList[T]: +def dummy_search_func(page_token: Union[None, str], **kwargs: Any) -> PagedList[str]: dummy_pages = dict( page_1=PagedList(items=["a", "b"], token="page_2"), page_2=PagedList(items=["c", "d"], token="page_3"), From 62c39cdcd3292ed52c6056865a1edb8471139eea Mon Sep 17 00:00:00 2001 From: Semion Sidorenko Date: Mon, 14 Oct 2024 20:57:38 +0200 Subject: [PATCH 05/40] fix(ingest/superset): parse postgres platform correctly (#11540) --- metadata-ingestion/src/datahub/ingestion/source/superset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index 858281f880359a..4e40407fba9086 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -243,6 +243,8 @@ def get_platform_from_database_id(self, database_id): return "athena" if platform_name == "clickhousedb": return "clickhouse" + if platform_name == "postgresql": + return "postgres" return platform_name @lru_cache(maxsize=None) From 09d70bc02c33da5c3f011cd4032fd06b4154e5d7 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:27:36 -0500 Subject: [PATCH 06/40] feat(openapi-v3): support async and createIfNotExists params on aspect (#11609) --- .../metadata/entity/EntityServiceImpl.java | 1 + .../controller/GenericEntitiesController.java | 37 +++++++++++++++---- .../v2/controller/EntityController.java | 27 +++++++++++++- .../openapi/v3/OpenAPIV3Generator.java | 22 +++++++++++ .../v3/controller/EntityController.java | 31 +++++++++++++++- .../metadata/entity/IngestResult.java | 2 + 6 files changed, 110 insertions(+), 10 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index 34c98bba01af4e..00feb547ca3300 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -1357,6 +1357,7 @@ private Stream ingestProposalSync( return IngestResult.builder() .urn(item.getUrn()) .request(item) + .result(result) .publishedMCL(result.getMclFuture() != null) .sqlCommitted(true) .isUpdate(result.getOldValue() != null) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java index 7e7929e7f27d37..7427f293c848f5 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java @@ -170,6 +170,9 @@ protected abstract E buildGenericEntity( @Nonnull UpdateAspectResult updateAspectResult, boolean withSystemMetadata); + protected abstract E buildGenericEntity( + @Nonnull String aspectName, @Nonnull IngestResult ingestResult, boolean withSystemMetadata); + protected abstract AspectsBatch toMCPBatch( @Nonnull OperationContext opContext, String entityArrayList, Actor actor) throws JsonProcessingException, InvalidUrnException; @@ -560,8 +563,11 @@ public ResponseEntity createAspect( @PathVariable("entityName") String entityName, @PathVariable("entityUrn") String entityUrn, @PathVariable("aspectName") String aspectName, + @RequestParam(value = "async", required = false, defaultValue = "false") Boolean async, @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") Boolean withSystemMetadata, + @RequestParam(value = "createIfEntityNotExists", required = false, defaultValue = "false") + Boolean createIfEntityNotExists, @RequestParam(value = "createIfNotExists", required = false, defaultValue = "true") Boolean createIfNotExists, @RequestBody @Nonnull String jsonAspect) @@ -591,24 +597,38 @@ public ResponseEntity createAspect( opContext.getRetrieverContext().get().getAspectRetriever(), urn, aspectSpec, + createIfEntityNotExists, createIfNotExists, jsonAspect, authentication.getActor()); - List results = - entityService.ingestAspects( + Set results = + entityService.ingestProposal( opContext, AspectsBatchImpl.builder() .retrieverContext(opContext.getRetrieverContext().get()) .items(List.of(upsert)) .build(), - true, - true); + async); - return ResponseEntity.of( - results.stream() - .findFirst() - .map(result -> buildGenericEntity(aspectName, result, withSystemMetadata))); + if (!async) { + return ResponseEntity.of( + results.stream() + .filter(item -> aspectName.equals(item.getRequest().getAspectName())) + .findFirst() + .map( + result -> + buildGenericEntity(aspectName, result.getResult(), withSystemMetadata))); + } else { + return results.stream() + .filter(item -> aspectName.equals(item.getRequest().getAspectName())) + .map( + result -> + ResponseEntity.accepted() + .body(buildGenericEntity(aspectName, result, withSystemMetadata))) + .findFirst() + .orElse(ResponseEntity.accepted().build()); + } } @Tag(name = "Generic Aspects") @@ -789,6 +809,7 @@ protected abstract ChangeMCP toUpsertItem( @Nonnull AspectRetriever aspectRetriever, Urn entityUrn, AspectSpec aspectSpec, + Boolean createIfEntityNotExists, Boolean createIfNotExists, String jsonAspect, Actor actor) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java index 28537b849b68ab..7bec052a9fd5d2 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java @@ -232,6 +232,20 @@ protected GenericEntityV2 buildGenericEntity( withSystemMetadata ? updateAspectResult.getNewSystemMetadata() : null))); } + @Override + protected GenericEntityV2 buildGenericEntity( + @Nonnull String aspectName, @Nonnull IngestResult ingestResult, boolean withSystemMetadata) { + return GenericEntityV2.builder() + .urn(ingestResult.getUrn().toString()) + .build( + objectMapper, + Map.of( + aspectName, + Pair.of( + ingestResult.getRequest().getRecordTemplate(), + withSystemMetadata ? ingestResult.getRequest().getSystemMetadata() : null))); + } + private List toRecordTemplates( @Nonnull OperationContext opContext, SearchEntityArray searchEntities, @@ -278,14 +292,25 @@ protected ChangeMCP toUpsertItem( @Nonnull AspectRetriever aspectRetriever, Urn entityUrn, AspectSpec aspectSpec, + Boolean createIfEntityNotExists, Boolean createIfNotExists, String jsonAspect, Actor actor) throws URISyntaxException { + + final ChangeType changeType; + if (Boolean.TRUE.equals(createIfEntityNotExists)) { + changeType = ChangeType.CREATE_ENTITY; + } else if (Boolean.TRUE.equals(createIfNotExists)) { + changeType = ChangeType.CREATE; + } else { + changeType = ChangeType.UPSERT; + } + return ChangeItemImpl.builder() .urn(entityUrn) .aspectName(aspectSpec.getName()) - .changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT) + .changeType(changeType) .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr())) .recordTemplate( GenericRecordUtils.deserializeAspect( diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index e33ad24a6c2486..d179ea8f3a0682 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -1100,6 +1100,28 @@ private static PathItem buildSingleEntityAspectPath( new Operation() .summary(String.format("Create aspect %s on %s ", aspect, upperFirstEntity)) .tags(tags) + .parameters( + List.of( + new Parameter() + .in(NAME_QUERY) + .name("async") + .description("Use async ingestion for high throughput.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), + new Parameter() + .in(NAME_QUERY) + .name(NAME_SYSTEM_METADATA) + .description("Include systemMetadata with response.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), + new Parameter() + .in(NAME_QUERY) + .name("createIfEntityNotExists") + .description("Only create the aspect if the Entity doesn't exist.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), + new Parameter() + .in(NAME_QUERY) + .name("createIfNotExists") + .description("Only create the aspect if the Aspect doesn't exist.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(true)))) .requestBody(requestBody) .responses(new ApiResponses().addApiResponse("201", successPostResponse)); // Patch Operation diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java index c7d8c72f8a1c39..55cf310be3438d 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java @@ -328,6 +328,24 @@ protected GenericEntityV3 buildGenericEntity( .build())); } + @Override + protected GenericEntityV3 buildGenericEntity( + @Nonnull String aspectName, @Nonnull IngestResult ingestResult, boolean withSystemMetadata) { + return GenericEntityV3.builder() + .build( + objectMapper, + ingestResult.getUrn(), + Map.of( + aspectName, + AspectItem.builder() + .aspect(ingestResult.getRequest().getRecordTemplate()) + .systemMetadata( + withSystemMetadata ? ingestResult.getRequest().getSystemMetadata() : null) + .auditStamp( + withSystemMetadata ? ingestResult.getRequest().getAuditStamp() : null) + .build())); + } + private List toRecordTemplates( @Nonnull OperationContext opContext, SearchEntityArray searchEntities, @@ -472,16 +490,27 @@ protected ChangeMCP toUpsertItem( @Nonnull AspectRetriever aspectRetriever, Urn entityUrn, AspectSpec aspectSpec, + Boolean createIfEntityNotExists, Boolean createIfNotExists, String jsonAspect, Actor actor) throws JsonProcessingException { JsonNode jsonNode = objectMapper.readTree(jsonAspect); String aspectJson = jsonNode.get("value").toString(); + + final ChangeType changeType; + if (Boolean.TRUE.equals(createIfEntityNotExists)) { + changeType = ChangeType.CREATE_ENTITY; + } else if (Boolean.TRUE.equals(createIfNotExists)) { + changeType = ChangeType.CREATE; + } else { + changeType = ChangeType.UPSERT; + } + return ChangeItemImpl.builder() .urn(entityUrn) .aspectName(aspectSpec.getName()) - .changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT) + .changeType(changeType) .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr())) .recordTemplate( GenericRecordUtils.deserializeAspect( diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestResult.java index d3f8b507bb14ac..f8b76db110c08f 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestResult.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestResult.java @@ -2,6 +2,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.aspect.batch.BatchItem; +import javax.annotation.Nullable; import lombok.Builder; import lombok.Value; @@ -10,6 +11,7 @@ public class IngestResult { Urn urn; BatchItem request; + @Nullable UpdateAspectResult result; boolean publishedMCL; boolean processedMCL; boolean publishedMCP; From 3078c5a4c982d57ffe308c35acb1d6d03e8c0ad1 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Mon, 14 Oct 2024 13:13:17 -0700 Subject: [PATCH 07/40] fix(ingest/preset): Add skip_on_failure to root_validator decorator (#11615) --- metadata-ingestion/src/datahub/ingestion/source/preset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/preset.py b/metadata-ingestion/src/datahub/ingestion/source/preset.py index e51520898103d6..6f53223e000f1b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/preset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/preset.py @@ -56,7 +56,7 @@ class PresetConfig(SupersetConfig): def remove_trailing_slash(cls, v): return config_clean.remove_trailing_slashes(v) - @root_validator + @root_validator(skip_on_failure=True) def default_display_uri_to_connect_uri(cls, values): base = values.get("display_uri") if base is None: From e0939c7ecd6e9b68a8e1def929dcb50dde1d46df Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Mon, 14 Oct 2024 17:14:55 -0700 Subject: [PATCH 08/40] docs(apis): update OpenAPI disclaimer (#11617) --- docs/api/datahub-apis.md | 132 +++++++++++++++++++-------------------- 1 file changed, 65 insertions(+), 67 deletions(-) diff --git a/docs/api/datahub-apis.md b/docs/api/datahub-apis.md index 6bb793a59a86e8..c46aacde3a0cb5 100644 --- a/docs/api/datahub-apis.md +++ b/docs/api/datahub-apis.md @@ -2,18 +2,16 @@ DataHub has several APIs to manipulate metadata on the platform. Here's the list of APIs and their pros and cons to help you choose the right one for your use case. -| API | Definition | Pros | Cons | -|--------------------------------------------------------------------------------|------------------------------------|------------------------------------------|-------------------------------------------------------------| -| **[Python SDK](/metadata-ingestion/as-a-library.md)** | SDK | Highly flexible, Good for bulk execution | Requires an understanding of the metadata change event | -| **[Java SDK](/metadata-integration/java/as-a-library.md)** | SDK | Highly flexible, Good for bulk execution | Requires an understanding of the metadata change event | -| **[GraphQL API](docs/api/graphql/getting-started.md)** | GraphQL interface | Intuitive; mirrors UI capabilities | Less flexible than SDKs; requires knowledge of GraphQL syntax | -| **[OpenAPI](docs/api/openapi/openapi-usage-guide.md)**
(Not Recommended) | Lower-level API for advanced users | | Generally not recommended for typical use cases | +| API | Definition | Pros | Cons | +| ---------------------------------------------------------- | ---------------------------------- | ---------------------------------------- | ----------------------------------------------------------------------- | +| **[Python SDK](/metadata-ingestion/as-a-library.md)** | SDK | Highly flexible, Good for bulk execution | Requires an understanding of the metadata change event | +| **[Java SDK](/metadata-integration/java/as-a-library.md)** | SDK | Highly flexible, Good for bulk execution | Requires an understanding of the metadata change event | +| **[GraphQL API](docs/api/graphql/getting-started.md)** | GraphQL interface | Intuitive; mirrors UI capabilities | Less flexible than SDKs; requires knowledge of GraphQL syntax | +| **[OpenAPI](docs/api/openapi/openapi-usage-guide.md)** | Lower-level API for advanced users | Most powerful and flexible | Can be hard to use for straightforward use cases; no corresponding SDKs | In general, **Python and Java SDKs** are our most recommended tools for extending and customizing the behavior of your DataHub instance. We don't recommend using the **OpenAPI** directly, as it's more complex and less user-friendly than the other APIs. - - ## Python and Java SDK We offer an SDK for both Python and Java that provide full functionality when it comes to CRUD operations and any complex functionality you may want to build into DataHub. We recommend using the SDKs for most use cases. Here are the examples of how to use the SDKs: @@ -23,22 +21,22 @@ We offer an SDK for both Python and Java that provide full functionality when it - Creating custom metadata entities Learn more about the SDKs: + - **[Python SDK →](/metadata-ingestion/as-a-library.md)** - **[Java SDK →](/metadata-integration/java/as-a-library.md)** - ## GraphQL API The `graphql` API serves as the primary public API for the platform. It can be used to fetch and update metadata programatically in the language of your choice. Intended as a higher-level API that simplifies the most common operations. We recommend using the GraphQL API if you're getting started with DataHub since it's more user-friendly and straighfowrad. Here are some examples of how to use the GraphQL API: + - Search for datasets with conditions - Update a certain field of a dataset Learn more about the GraphQL API: -- **[GraphQL API →](docs/api/graphql/getting-started.md)** - +- **[GraphQL API →](docs/api/graphql/getting-started.md)** ## DataHub API Comparison @@ -47,59 +45,59 @@ Here's an overview of what each API can do. > Last Updated : Feb 16 2024 -| Feature | GraphQL | Python SDK | OpenAPI | -|------------------------------------|------------------------------------------------------------------------------|------------------------------------------------------------------------------|---------| -| Create a Dataset | 🚫 | ✅ [[Guide]](/docs/api/tutorials/datasets.md) | ✅ | -| Delete a Dataset (Soft Delete) | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ | -| Delete a Dataset (Hard Delete) | 🚫 | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ | -| Search a Dataset | ✅ [[Guide]](/docs/how/search.md#graphql) | ✅ | ✅ | -| Read a Dataset Deprecation | ✅ | ✅ | ✅ | -| Read Dataset Entities (V2) | ✅ | ✅ | ✅ | -| Create a Tag | ✅ [[Guide]](/docs/api/tutorials/tags.md#create-tags) | ✅ [[Guide]](/docs/api/tutorials/tags.md#create-tags) | ✅ | -| Read a Tag | ✅ [[Guide]](/docs/api/tutorials/tags.md#read-tags) | ✅ [[Guide]](/docs/api/tutorials/tags.md#read-tags) | ✅ | -| Add Tags to a Dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-dataset) | ✅ | -| Add Tags to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-column-of-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-column-of-a-dataset) | ✅ | -| Remove Tags from a Dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md#remove-tags) | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags#remove-tags) | ✅ | -| Create Glossary Terms | ✅ [[Guide]](/docs/api/tutorials/terms.md#create-terms) | ✅ [[Guide]](/docs/api/tutorials/terms.md#create-terms) | ✅ | -| Read Terms from a Dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md#read-terms) | ✅ [[Guide]](/docs/api/tutorials/terms.md#read-terms) | ✅ | -| Add Terms to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-column-of-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-column-of-a-dataset) | ✅ | -| Add Terms to a Dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-dataset) | ✅ | -| Create Domains | ✅ [[Guide]](/docs/api/tutorials/domains.md#create-domain) | ✅ [[Guide]](/docs/api/tutorials/domains.md#create-domain) | ✅ | -| Read Domains | ✅ [[Guide]](/docs/api/tutorials/domains.md#read-domains) | ✅ [[Guide]](/docs/api/tutorials/domains.md#read-domains) | ✅ | -| Add Domains to a Dataset | ✅ [[Guide]](/docs/api/tutorials/domains.md#add-domains) | ✅ [[Guide]](/docs/api/tutorials/domains.md#add-domains) | ✅ | -| Remove Domains from a Dataset | ✅ [[Guide]](/docs/api/tutorials/domains.md#remove-domains) | ✅ [[Guide]](/docs/api/tutorials/domains.md#remove-domains) | ✅ | -| Create / Upsert Users | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-users) | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-users) | ✅ | -| Create / Upsert Group | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-group) | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-group) | ✅ | -| Read Owners of a Dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md#read-owners) | ✅ [[Guide]](/docs/api/tutorials/owners.md#read-owners) | ✅ | -| Add Owner to a Dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md#add-owners) | ✅ [[Guide]](/docs/api/tutorials/owners.md#add-owners#remove-owners) | ✅ | -| Remove Owner from a Dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md#remove-owners) | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ | -| Add Lineage | ✅ [[Guide]](/docs/api/tutorials/lineage.md) | ✅ [[Guide]](/docs/api/tutorials/lineage.md#add-lineage) | ✅ | -| Add Column Level (Fine Grained) Lineage | 🚫 | ✅ [[Guide]](docs/api/tutorials/lineage.md#add-column-level-lineage) | ✅ | -| Add Documentation (Description) to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅ | -| Add Documentation (Description) to a Dataset | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset) | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset) | ✅ | -| Add / Remove / Replace Custom Properties on a Dataset | 🚫 | ✅ [[Guide]](/docs/api/tutorials/custom-properties.md) | ✅ | -| Add ML Feature to ML Feature Table | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlfeature-to-mlfeaturetable) | ✅ | -| Add ML Feature to MLModel | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlfeature-to-mlmodel) | ✅ | -| Add ML Group to MLFeatureTable | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlgroup-to-mlfeaturetable) | ✅ | -| Create MLFeature | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeature) | ✅ | -| Create MLFeatureTable | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeaturetable) | ✅ | -| Create MLModel | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlmodel) | ✅ | -| Create MLModelGroup | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlmodelgroup) | ✅ | -| Create MLPrimaryKey | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlprimarykey) | ✅ | -| Create MLFeatureTable | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeaturetable)| ✅ | -| Read MLFeature | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeature) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeature) | ✅ | -| Read MLFeatureTable | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeaturetable) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeaturetable) | ✅ | -| Read MLModel | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodel) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodel) | ✅ | -| Read MLModelGroup | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodelgroup) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodelgroup) | ✅ | -| Read MLPrimaryKey | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlprimarykey) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlprimarykey) | ✅ | -| Create Data Product | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/create_dataproduct.py) | ✅ | -| Create Lineage Between Chart and Dashboard | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_chart_dashboard.py) | ✅ | -| Create Lineage Between Dataset and Chart | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_dataset_chart.py) | ✅ | -| Create Lineage Between Dataset and DataJob | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_dataset_job_dataset.py) | ✅ | -| Create Finegrained Lineage as DataJob for Dataset | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_datajob_finegrained.py) | ✅ | -| Create Finegrained Lineage for Dataset | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_dataset_finegrained.py) | ✅ | -| Create Dataset Lineage with Kafka | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_kafka.py) | ✅ | -| Create Dataset Lineage with MCPW & Rest Emitter | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_mcpw_rest.py) | ✅ | -| Create Dataset Lineage with Rest Emitter | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_rest.py) | ✅ | -| Create DataJob with Dataflow | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow.py) [[Simple]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py) [[Verbose]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_verbose.py) | ✅ | -| Create Programmatic Pipeline | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/programatic_pipeline.py) | ✅ | +| Feature | GraphQL | Python SDK | OpenAPI | +| -------------------------------------------------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| Create a Dataset | 🚫 | ✅ [[Guide]](/docs/api/tutorials/datasets.md) | ✅ | +| Delete a Dataset (Soft Delete) | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ | +| Delete a Dataset (Hard Delete) | 🚫 | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ | +| Search a Dataset | ✅ [[Guide]](/docs/how/search.md#graphql) | ✅ | ✅ | +| Read a Dataset Deprecation | ✅ | ✅ | ✅ | +| Read Dataset Entities (V2) | ✅ | ✅ | ✅ | +| Create a Tag | ✅ [[Guide]](/docs/api/tutorials/tags.md#create-tags) | ✅ [[Guide]](/docs/api/tutorials/tags.md#create-tags) | ✅ | +| Read a Tag | ✅ [[Guide]](/docs/api/tutorials/tags.md#read-tags) | ✅ [[Guide]](/docs/api/tutorials/tags.md#read-tags) | ✅ | +| Add Tags to a Dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-dataset) | ✅ | +| Add Tags to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-column-of-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-column-of-a-dataset) | ✅ | +| Remove Tags from a Dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md#remove-tags) | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags#remove-tags) | ✅ | +| Create Glossary Terms | ✅ [[Guide]](/docs/api/tutorials/terms.md#create-terms) | ✅ [[Guide]](/docs/api/tutorials/terms.md#create-terms) | ✅ | +| Read Terms from a Dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md#read-terms) | ✅ [[Guide]](/docs/api/tutorials/terms.md#read-terms) | ✅ | +| Add Terms to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-column-of-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-column-of-a-dataset) | ✅ | +| Add Terms to a Dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-dataset) | ✅ | +| Create Domains | ✅ [[Guide]](/docs/api/tutorials/domains.md#create-domain) | ✅ [[Guide]](/docs/api/tutorials/domains.md#create-domain) | ✅ | +| Read Domains | ✅ [[Guide]](/docs/api/tutorials/domains.md#read-domains) | ✅ [[Guide]](/docs/api/tutorials/domains.md#read-domains) | ✅ | +| Add Domains to a Dataset | ✅ [[Guide]](/docs/api/tutorials/domains.md#add-domains) | ✅ [[Guide]](/docs/api/tutorials/domains.md#add-domains) | ✅ | +| Remove Domains from a Dataset | ✅ [[Guide]](/docs/api/tutorials/domains.md#remove-domains) | ✅ [[Guide]](/docs/api/tutorials/domains.md#remove-domains) | ✅ | +| Create / Upsert Users | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-users) | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-users) | ✅ | +| Create / Upsert Group | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-group) | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-group) | ✅ | +| Read Owners of a Dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md#read-owners) | ✅ [[Guide]](/docs/api/tutorials/owners.md#read-owners) | ✅ | +| Add Owner to a Dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md#add-owners) | ✅ [[Guide]](/docs/api/tutorials/owners.md#add-owners#remove-owners) | ✅ | +| Remove Owner from a Dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md#remove-owners) | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ | +| Add Lineage | ✅ [[Guide]](/docs/api/tutorials/lineage.md) | ✅ [[Guide]](/docs/api/tutorials/lineage.md#add-lineage) | ✅ | +| Add Column Level (Fine Grained) Lineage | 🚫 | ✅ [[Guide]](docs/api/tutorials/lineage.md#add-column-level-lineage) | ✅ | +| Add Documentation (Description) to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅ | +| Add Documentation (Description) to a Dataset | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset) | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset) | ✅ | +| Add / Remove / Replace Custom Properties on a Dataset | 🚫 | ✅ [[Guide]](/docs/api/tutorials/custom-properties.md) | ✅ | +| Add ML Feature to ML Feature Table | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlfeature-to-mlfeaturetable) | ✅ | +| Add ML Feature to MLModel | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlfeature-to-mlmodel) | ✅ | +| Add ML Group to MLFeatureTable | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlgroup-to-mlfeaturetable) | ✅ | +| Create MLFeature | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeature) | ✅ | +| Create MLFeatureTable | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeaturetable) | ✅ | +| Create MLModel | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlmodel) | ✅ | +| Create MLModelGroup | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlmodelgroup) | ✅ | +| Create MLPrimaryKey | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlprimarykey) | ✅ | +| Create MLFeatureTable | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeaturetable) | ✅ | +| Read MLFeature | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeature) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeature) | ✅ | +| Read MLFeatureTable | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeaturetable) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeaturetable) | ✅ | +| Read MLModel | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodel) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodel) | ✅ | +| Read MLModelGroup | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodelgroup) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodelgroup) | ✅ | +| Read MLPrimaryKey | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlprimarykey) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlprimarykey) | ✅ | +| Create Data Product | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/create_dataproduct.py) | ✅ | +| Create Lineage Between Chart and Dashboard | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_chart_dashboard.py) | ✅ | +| Create Lineage Between Dataset and Chart | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_dataset_chart.py) | ✅ | +| Create Lineage Between Dataset and DataJob | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_dataset_job_dataset.py) | ✅ | +| Create Finegrained Lineage as DataJob for Dataset | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_datajob_finegrained.py) | ✅ | +| Create Finegrained Lineage for Dataset | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_dataset_finegrained.py) | ✅ | +| Create Dataset Lineage with Kafka | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_kafka.py) | ✅ | +| Create Dataset Lineage with MCPW & Rest Emitter | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_mcpw_rest.py) | ✅ | +| Create Dataset Lineage with Rest Emitter | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_rest.py) | ✅ | +| Create DataJob with Dataflow | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow.py) [[Simple]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py) [[Verbose]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_verbose.py) | ✅ | +| Create Programmatic Pipeline | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/programatic_pipeline.py) | ✅ | From 0d06a613655fbec2311161c89cb4f333e0d74c31 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Mon, 14 Oct 2024 20:38:23 -0700 Subject: [PATCH 09/40] docs: add docs on term suggestion (#11606) --- docs-website/sidebars.js | 12 ++++ docs/automations/ai-docs.md | 36 ++++++++++ docs/automations/ai-term-suggestion.md | 72 +++++++++++++++++++ docs/automations/snowflake-tag-propagation.md | 33 +++++---- 4 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 docs/automations/ai-docs.md create mode 100644 docs/automations/ai-term-suggestion.md diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 06838f2e686bc0..8f3da2050a9b7f 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -113,6 +113,18 @@ module.exports = { id: "docs/automations/snowflake-tag-propagation", className: "saasOnly", }, + { + label: "AI Classification", + type: "doc", + id: "docs/automations/ai-term-suggestion", + className: "saasOnly", + }, + { + label: "AI Documentation", + type: "doc", + id: "docs/automations/ai-docs", + className: "saasOnly", + }, ], }, { diff --git a/docs/automations/ai-docs.md b/docs/automations/ai-docs.md new file mode 100644 index 00000000000000..bbec33f3bcae64 --- /dev/null +++ b/docs/automations/ai-docs.md @@ -0,0 +1,36 @@ +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# AI Documentation + + + +:::info + +This feature is currently in closed beta. Reach out to your Acryl representative to get access. + +::: + +With AI-powered documentation, you can automatically generate documentation for tables and columns. + +

+ +

+ +## Configuring + +No configuration is required - just hit "Generate" on any table or column in the UI. + +## How it works + +Generating good documentation requires a holistic understanding of the data. Information we take into account includes, but is not limited to: + +- Dataset name and any existing documentation +- Column name, type, description, and sample values +- Lineage relationships to upstream and downstream assets +- Metadata about other related assets + +Data privacy: Your metadata is not sent to any third-party LLMs. We use AWS Bedrock internally, which means all metadata remains within the Acryl AWS account. We do not fine-tune on customer data. + +## Limitations + +- This feature is powered by an LLM, which can produce inaccurate results. While we've taken steps to reduce the likelihood of hallucinations, they can still occur. diff --git a/docs/automations/ai-term-suggestion.md b/docs/automations/ai-term-suggestion.md new file mode 100644 index 00000000000000..27d1716cfc372c --- /dev/null +++ b/docs/automations/ai-term-suggestion.md @@ -0,0 +1,72 @@ +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# AI Glossary Term Suggestions + + + +:::info + +This feature is currently in closed beta. Reach out to your Acryl representative to get access. + +::: + +The AI Glossary Term Suggestion automation uses LLMs to suggest [Glossary Terms](../glossary/business-glossary.md) for tables and columns in your data. + +This is useful for improving coverage of glossary terms across your organization, which is important for compliance and governance efforts. + +This automation can: + +- Automatically suggests glossary terms for tables and columns. +- Goes beyond a predefined set of terms and works with your business glossary. +- Generates [proposals](../managed-datahub/approval-workflows.md) for owners to review, or can automatically add terms to tables/columns. +- Automatically adjusts to human-provided feedback and curation (coming soon). + +## Prerequisites + +- A business glossary with terms defined. Additional metadata, like documentation and existing term assignments, will improve the accuracy of our suggestions. + +## Configuring + +1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar. + +

+ +

+ +2. **Create the Automation**: Click on 'Create' and select 'AI Glossary Term Suggestions'. + +

+ +

+ +3. **Configure the Automation**: Fill in the required fields to configure the automation. + The main fields to configure are (1) what terms to use for suggestions and (2) what entities to generate suggestions for. + +

+ +

+ +4. Once it's enabled, that's it! You'll start to see terms show up in the UI, either on assets or in the proposals page. + +

+ +

+ +## How it works + +The automation will scan through all the datasets matched by the configured filters. For each one, it will generate suggestions. +If new entities are added that match the configured filters, those will also be classified within 24 hours. + +We take into account the following metadata when generating suggestions: + +- Dataset name and description +- Column name, type, description, and sample values +- Glossary term name, documentation, and hierarchy +- Feedback loop: existing assignments and accepted/rejected proposals (coming soon) + +Data privacy: Your metadata is not sent to any third-party LLMs. We use AWS Bedrock internally, which means all metadata remains within the Acryl AWS account. We do not fine-tune on customer data. + +## Limitations + +- A single configured automation can classify at most 10k entities. +- We cannot do partial reclassification. If you add a new column to an existing table, we won't regenerate suggestions for that table. diff --git a/docs/automations/snowflake-tag-propagation.md b/docs/automations/snowflake-tag-propagation.md index bdc80376dfb484..c708e40cbdd81e 100644 --- a/docs/automations/snowflake-tag-propagation.md +++ b/docs/automations/snowflake-tag-propagation.md @@ -1,4 +1,3 @@ - import FeatureAvailability from '@site/src/components/FeatureAvailability'; # Snowflake Tag Propagation Automation @@ -20,22 +19,22 @@ both columns and tables back to Snowflake. This automation is available in DataH 1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar. -

- +

+

2. **Create An Automation**: Click on 'Create' and select 'Snowflake Tag Propagation'. -

- +

+

-3. **Configure Automation**: Fill in the required fields to connect to Snowflake, along with the name, description, and category. -Note that you can limit propagation based on specific Tags and Glossary Terms. If none are selected, then ALL Tags or Glossary Terms will be automatically -propagated to Snowflake tables and columns. Finally, click 'Save and Run' to start the automation +3. **Configure Automation**: Fill in the required fields to connect to Snowflake, along with the name, description, and category. + Note that you can limit propagation based on specific Tags and Glossary Terms. If none are selected, then ALL Tags or Glossary Terms will be automatically + propagated to Snowflake tables and columns. Finally, click 'Save and Run' to start the automation -

- +

+

## Propagating for Existing Assets @@ -46,13 +45,13 @@ Note that it may take some time to complete the initial back-filling process, de To do so, navigate to the Automation you created in Step 3 above, click the 3-dot "More" menu

- +

and then click "Initialize".

- +

This one-time step will kick off the back-filling process for existing descriptions. If you only want to begin propagating @@ -68,21 +67,21 @@ that you no longer want propagated descriptions to be visible. To do this, navigate to the Automation you created in Step 3 above, click the 3-dot "More" menu

- +

and then click "Rollback".

- +

This one-time step will remove all propagated tags and glossary terms from Snowflake. To simply stop propagating new tags, you can disable the automation. ## Viewing Propagated Tags -You can view propagated Tags (and corresponding DataHub URNs) inside the Snowflake UI to confirm the automation is working as expected. +You can view propagated Tags (and corresponding DataHub URNs) inside the Snowflake UI to confirm the automation is working as expected. -

- +

+

From be1b8806a772e459016a225f3ace103bb692b070 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 15 Oct 2024 01:39:41 -0700 Subject: [PATCH 10/40] fix(airflow): fix lint related to dag_run field (#11616) --- .../src/datahub_airflow_plugin/datahub_listener.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py index d1c7e996dd03dc..c1d5b306f187dc 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py @@ -383,13 +383,11 @@ def on_task_instance_running( return logger.debug( - f"DataHub listener got notification about task instance start for {task_instance.task_id} of dag {task_instance.dag_run.dag_id}" + f"DataHub listener got notification about task instance start for {task_instance.task_id} of dag {task_instance.dag_id}" ) - if not self.config.dag_filter_pattern.allowed(task_instance.dag_run.dag_id): - logger.debug( - f"DAG {task_instance.dag_run.dag_id} is not allowed by the pattern" - ) + if not self.config.dag_filter_pattern.allowed(task_instance.dag_id): + logger.debug(f"DAG {task_instance.dag_id} is not allowed by the pattern") return if self.config.render_templates: From 1eec2c42383b1b808ea157f9e2acebe9f9ec4fbe Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 15 Oct 2024 14:53:45 +0530 Subject: [PATCH 11/40] fix(ingest): drop empty fields (#11613) --- .../src/datahub/ingestion/api/source.py | 2 + .../datahub/ingestion/api/source_helpers.py | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py index 85ae17ddf65291..586b1c610dc756 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source.py @@ -37,6 +37,7 @@ from datahub.ingestion.api.source_helpers import ( auto_browse_path_v2, auto_fix_duplicate_schema_field_paths, + auto_fix_empty_field_paths, auto_lowercase_urns, auto_materialize_referenced_tags_terms, auto_status_aspect, @@ -444,6 +445,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: partial( auto_fix_duplicate_schema_field_paths, platform=self._infer_platform() ), + partial(auto_fix_empty_field_paths, platform=self._infer_platform()), browse_path_processor, partial(auto_workunit_reporter, self.get_report()), auto_patch_last_modified, diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py index 372aef707f232f..748d8a8e52a793 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py @@ -394,6 +394,50 @@ def auto_fix_duplicate_schema_field_paths( ) +def auto_fix_empty_field_paths( + stream: Iterable[MetadataWorkUnit], + *, + platform: Optional[str] = None, +) -> Iterable[MetadataWorkUnit]: + """Count schema metadata aspects with empty field paths and emit telemetry.""" + + total_schema_aspects = 0 + schemas_with_empty_fields = 0 + empty_field_paths = 0 + + for wu in stream: + schema_metadata = wu.get_aspect_of_type(SchemaMetadataClass) + if schema_metadata: + total_schema_aspects += 1 + + updated_fields: List[SchemaFieldClass] = [] + for field in schema_metadata.fields: + if field.fieldPath: + updated_fields.append(field) + else: + empty_field_paths += 1 + + if empty_field_paths > 0: + logger.info( + f"Fixing empty field paths in schema aspect for {wu.get_urn()} by dropping empty fields" + ) + schema_metadata.fields = updated_fields + schemas_with_empty_fields += 1 + + yield wu + + if schemas_with_empty_fields > 0: + properties = { + "platform": platform, + "total_schema_aspects": total_schema_aspects, + "schemas_with_empty_fields": schemas_with_empty_fields, + "empty_field_paths": empty_field_paths, + } + telemetry.telemetry_instance.ping( + "ingestion_empty_schema_field_paths", properties + ) + + def auto_empty_dataset_usage_statistics( stream: Iterable[MetadataWorkUnit], *, From d8ede691f095aa7dc27f90692a067555584bd33e Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:34:30 -0500 Subject: [PATCH 12/40] fix(ci): ensure py 3.10 (#11626) --- .github/workflows/docker-unified.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index ef5770ccb167a8..a1ae7cee1736fa 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -760,14 +760,18 @@ jobs: needs: [setup, datahub_ingestion_base_slim_build] if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} steps: + - name: Check out the repo + uses: acryldata/sane-checkout-action@v3 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" - name: Set up JDK 17 uses: actions/setup-java@v4 with: distribution: "zulu" java-version: 17 - uses: gradle/actions/setup-gradle@v3 - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - name: Build codegen if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish =='true' }} run: ./gradlew :metadata-ingestion:codegen @@ -852,14 +856,18 @@ jobs: needs: [setup, datahub_ingestion_base_full_build] if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} steps: + - name: Check out the repo + uses: acryldata/sane-checkout-action@v3 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" - name: Set up JDK 17 uses: actions/setup-java@v4 with: distribution: "zulu" java-version: 17 - uses: gradle/actions/setup-gradle@v3 - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - name: Build codegen if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} run: ./gradlew :metadata-ingestion:codegen @@ -983,16 +991,16 @@ jobs: run: df -h . && docker images - name: Check out the repo uses: acryldata/sane-checkout-action@v3 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" - name: Set up JDK 17 uses: actions/setup-java@v4 with: distribution: "zulu" java-version: 17 - uses: gradle/actions/setup-gradle@v3 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: "pip" - name: Login to DockerHub uses: docker/login-action@v3 if: ${{ needs.setup.outputs.docker-login == 'true' }} From c834cdb0ab9f63dedbc8b69ccc729f776d160337 Mon Sep 17 00:00:00 2001 From: Jay <159848059+jayacryl@users.noreply.github.com> Date: Tue, 15 Oct 2024 10:55:23 -0400 Subject: [PATCH 13/40] feat(docs-site) brought back announcement banner (#11618) --- docs-website/docusaurus.config.js | 8 ++++++++ docs-website/src/pages/index.js | 2 +- docs-website/src/styles/global.scss | 5 +++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js index b0cf5bfa35ecae..b016f9518ea6c1 100644 --- a/docs-website/docusaurus.config.js +++ b/docs-website/docusaurus.config.js @@ -65,6 +65,14 @@ module.exports = { // isCloseable: false, // }, // }), + announcementBar: { + id: "announcement-2", + content: + '
NEW

Join us at Metadata & AI Summit, Oct. 29 & 30!

Register →
', + backgroundColor: "#111", + textColor: "#ffffff", + isCloseable: false, + }, colorMode: { // Only support light mode. defaultMode: 'light', diff --git a/docs-website/src/pages/index.js b/docs-website/src/pages/index.js index 1c36b81a2da95d..e1c94780715d35 100644 --- a/docs-website/src/pages/index.js +++ b/docs-website/src/pages/index.js @@ -44,7 +44,7 @@ function Home() { return !siteConfig.customFields.isSaas ? ( {isTourModalVisible ? (
diff --git a/docs-website/src/styles/global.scss b/docs-website/src/styles/global.scss index e256c752b4a0b4..96ca07d45d0c28 100644 --- a/docs-website/src/styles/global.scss +++ b/docs-website/src/styles/global.scss @@ -31,7 +31,7 @@ --ifm-navbar-item-padding-horizontal: 1rem; /* Announcement Bar */ - --docusaurus-announcement-bar-height: 60px !important; + --docusaurus-announcement-bar-height: 48px !important; /* Rule */ --ifm-hr-border-width: 1px 0 0 0; @@ -141,8 +141,9 @@ div[class^="announcementBar"] { } a { - color: var(--ifm-button-color); + color: #EFB300; text-decoration: none; + font-size: 1rem } } } From b4b91421e817ccbb4ed0012b55b0793bdbed7b2c Mon Sep 17 00:00:00 2001 From: deepgarg-visa <149145061+deepgarg-visa@users.noreply.github.com> Date: Tue, 15 Oct 2024 20:46:56 +0530 Subject: [PATCH 14/40] fix(search): make graphql query autoCompleteForMultiple to show exact matches first (#11586) --- .../elasticsearch/query/ESSearchDAO.java | 2 +- .../request/AutocompleteRequestHandler.java | 95 ++++++++++------- .../fixtures/SampleDataFixtureTestBase.java | 22 +++- .../AutocompleteRequestHandlerTest.java | 98 +++++++++++++----- .../sample_data/containerindex_v2.json.gz | Bin 295 -> 335 bytes 5 files changed, 154 insertions(+), 63 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index f09a81c0c8b891..2d7db075e676ff 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -370,7 +370,7 @@ public AutoCompleteResult autoComplete( IndexConvention indexConvention = opContext.getSearchContext().getIndexConvention(); AutocompleteRequestHandler builder = AutocompleteRequestHandler.getBuilder( - entitySpec, customSearchConfiguration, queryFilterRewriteChain); + entitySpec, customSearchConfiguration, queryFilterRewriteChain, searchConfiguration); SearchRequest req = builder.getSearchRequest( opContext, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java index 294efb069a9046..45359285b4a046 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java @@ -1,6 +1,5 @@ package com.linkedin.metadata.search.elasticsearch.query.request; -import static com.linkedin.metadata.models.SearchableFieldSpecExtractor.PRIMARY_URN_SEARCH_PROPERTIES; import static com.linkedin.metadata.search.utils.ESAccessControlUtil.restrictUrn; import static com.linkedin.metadata.search.utils.ESUtils.applyDefaultSearchFilters; @@ -8,6 +7,7 @@ import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.StringArray; +import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.AutocompleteConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.config.search.custom.QueryConfiguration; @@ -35,6 +35,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.index.query.*; @@ -46,7 +47,7 @@ @Slf4j public class AutocompleteRequestHandler { - private final List _defaultAutocompleteFields; + private final List _defaultAutocompleteFields; private final Map> searchableFieldTypes; private static final Map @@ -56,11 +57,13 @@ public class AutocompleteRequestHandler { private final EntitySpec entitySpec; private final QueryFilterRewriteChain queryFilterRewriteChain; + private final SearchConfiguration searchConfiguration; public AutocompleteRequestHandler( @Nonnull EntitySpec entitySpec, @Nullable CustomSearchConfiguration customSearchConfiguration, - @Nonnull QueryFilterRewriteChain queryFilterRewriteChain) { + @Nonnull QueryFilterRewriteChain queryFilterRewriteChain, + @Nonnull SearchConfiguration searchConfiguration) { this.entitySpec = entitySpec; List fieldSpecs = entitySpec.getSearchableFieldSpecs(); this.customizedQueryHandler = CustomizedQueryHandler.builder(customSearchConfiguration).build(); @@ -69,8 +72,12 @@ public AutocompleteRequestHandler( fieldSpecs.stream() .map(SearchableFieldSpec::getSearchableAnnotation) .filter(SearchableAnnotation::isEnableAutocomplete) - .map(SearchableAnnotation::getFieldName), - Stream.of("urn")) + .map( + searchableAnnotation -> + Pair.of( + searchableAnnotation.getFieldName(), + Double.toString(searchableAnnotation.getBoostScore()))), + Stream.of(Pair.of("urn", "1.0"))) .collect(Collectors.toList()); searchableFieldTypes = fieldSpecs.stream() @@ -87,17 +94,22 @@ public AutocompleteRequestHandler( return set1; })); this.queryFilterRewriteChain = queryFilterRewriteChain; + this.searchConfiguration = searchConfiguration; } public static AutocompleteRequestHandler getBuilder( @Nonnull EntitySpec entitySpec, @Nullable CustomSearchConfiguration customSearchConfiguration, - @Nonnull QueryFilterRewriteChain queryFilterRewriteChain) { + @Nonnull QueryFilterRewriteChain queryFilterRewriteChain, + @Nonnull SearchConfiguration searchConfiguration) { return AUTOCOMPLETE_QUERY_BUILDER_BY_ENTITY_NAME.computeIfAbsent( entitySpec, k -> new AutocompleteRequestHandler( - entitySpec, customSearchConfiguration, queryFilterRewriteChain)); + entitySpec, + customSearchConfiguration, + queryFilterRewriteChain, + searchConfiguration)); } public SearchRequest getSearchRequest( @@ -169,7 +181,7 @@ private BoolQueryBuilder getQuery( public BoolQueryBuilder getQuery( @Nonnull ObjectMapper objectMapper, @Nullable AutocompleteConfiguration customAutocompleteConfig, - List autocompleteFields, + List autocompleteFields, @Nonnull String query) { BoolQueryBuilder finalQuery = @@ -189,7 +201,7 @@ public BoolQueryBuilder getQuery( private Optional getAutocompleteQuery( @Nullable AutocompleteConfiguration customConfig, - List autocompleteFields, + List autocompleteFields, @Nonnull String query) { Optional result = Optional.empty(); @@ -200,33 +212,39 @@ private Optional getAutocompleteQuery( return result; } - private static BoolQueryBuilder defaultQuery( - List autocompleteFields, @Nonnull String query) { + private BoolQueryBuilder defaultQuery(List autocompleteFields, @Nonnull String query) { BoolQueryBuilder finalQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); // Search for exact matches with higher boost and ngram matches - MultiMatchQueryBuilder autocompleteQueryBuilder = + MultiMatchQueryBuilder multiMatchQueryBuilder = QueryBuilders.multiMatchQuery(query).type(MultiMatchQueryBuilder.Type.BOOL_PREFIX); - final float urnBoost = - Float.parseFloat((String) PRIMARY_URN_SEARCH_PROPERTIES.get("boostScore")); autocompleteFields.forEach( - fieldName -> { - if ("urn".equals(fieldName)) { - autocompleteQueryBuilder.field(fieldName + ".ngram", urnBoost); - autocompleteQueryBuilder.field(fieldName + ".ngram._2gram", urnBoost); - autocompleteQueryBuilder.field(fieldName + ".ngram._3gram", urnBoost); - autocompleteQueryBuilder.field(fieldName + ".ngram._4gram", urnBoost); - } else { - autocompleteQueryBuilder.field(fieldName + ".ngram"); - autocompleteQueryBuilder.field(fieldName + ".ngram._2gram"); - autocompleteQueryBuilder.field(fieldName + ".ngram._3gram"); - autocompleteQueryBuilder.field(fieldName + ".ngram._4gram"); + pair -> { + final String fieldName = (String) pair.getLeft(); + final float boostScore = Float.parseFloat((String) pair.getRight()); + multiMatchQueryBuilder.field(fieldName + ".ngram"); + multiMatchQueryBuilder.field(fieldName + ".ngram._2gram"); + multiMatchQueryBuilder.field(fieldName + ".ngram._3gram"); + multiMatchQueryBuilder.field(fieldName + ".ngram._4gram"); + multiMatchQueryBuilder.field(fieldName + ".delimited"); + if (!fieldName.equalsIgnoreCase("urn")) { + multiMatchQueryBuilder.field(fieldName + ".ngram", boostScore); + multiMatchQueryBuilder.field( + fieldName + ".ngram._2gram", + boostScore * (searchConfiguration.getWordGram().getTwoGramFactor())); + multiMatchQueryBuilder.field( + fieldName + ".ngram._3gram", + boostScore * (searchConfiguration.getWordGram().getThreeGramFactor())); + multiMatchQueryBuilder.field( + fieldName + ".ngram._4gram", + boostScore * (searchConfiguration.getWordGram().getFourGramFactor())); + finalQuery.should( + QueryBuilders.matchQuery(fieldName + ".keyword", query).boost(boostScore)); } - autocompleteQueryBuilder.field(fieldName + ".delimited"); finalQuery.should(QueryBuilders.matchPhrasePrefixQuery(fieldName + ".delimited", query)); }); - finalQuery.should(autocompleteQueryBuilder); + finalQuery.should(multiMatchQueryBuilder); return finalQuery; } @@ -241,12 +259,17 @@ private HighlightBuilder getHighlights(@Nullable String field) { // Check for each field name and any subfields getAutocompleteFields(field) .forEach( - fieldName -> - highlightBuilder - .field(fieldName) - .field(fieldName + ".*") - .field(fieldName + ".ngram") - .field(fieldName + ".delimited")); + pair -> { + final String fieldName = (String) pair.getLeft(); + highlightBuilder + .field(fieldName) + .field(fieldName + ".*") + .field(fieldName + ".ngram") + .field(fieldName + ".delimited"); + if (!fieldName.equalsIgnoreCase("urn")) { + highlightBuilder.field(fieldName + ".keyword"); + } + }); // set field match req false for ngram highlightBuilder.fields().stream() @@ -256,9 +279,9 @@ private HighlightBuilder getHighlights(@Nullable String field) { return highlightBuilder; } - private List getAutocompleteFields(@Nullable String field) { - if (field != null && !field.isEmpty()) { - return ImmutableList.of(field); + private List getAutocompleteFields(@Nullable String field) { + if (field != null && !field.isEmpty() && !field.equalsIgnoreCase("urn")) { + return ImmutableList.of(Pair.of(field, "10.0")); } return _defaultAutocompleteFields; } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java index bc3c892e07b1bb..504eb5f5fc13db 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java @@ -283,7 +283,7 @@ public void testFixtureInitialization() { Map.of( "dataset", 13, "chart", 0, - "container", 1, + "container", 2, "dashboard", 0, "tag", 0, "mlmodel", 0); @@ -903,6 +903,26 @@ public void testContainerAutoComplete() { }); } + @Test + public void testContainerAutoComplete_with_exactMatch_onTop() { + List.of("container") + .forEach( + query -> { + try { + AutoCompleteResults result = + autocomplete( + getOperationContext(), new ContainerType(getEntityClient()), query); + assertTrue( + result.getSuggestions().get(0).equals("container"), + String.format( + "Expected query:`%s` on top of suggestions, found %s", + query, result.getSuggestions().get(0))); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + @Test public void testGroupAutoComplete() { List.of("T", "Te", "Tes", "Test ", "Test G", "Test Gro", "Test Group ") diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java index 572d79ebf2f0ce..c5205906e9d373 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java @@ -5,6 +5,10 @@ import static org.testng.Assert.assertTrue; import com.linkedin.metadata.TestEntitySpecBuilder; +import com.linkedin.metadata.config.search.ExactMatchConfiguration; +import com.linkedin.metadata.config.search.PartialConfiguration; +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.WordGramConfiguration; import com.linkedin.metadata.config.search.custom.AutocompleteConfiguration; import com.linkedin.metadata.config.search.custom.BoolQueryConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; @@ -32,14 +36,44 @@ import org.testng.annotations.Test; public class AutocompleteRequestHandlerTest { - private AutocompleteRequestHandler handler = - AutocompleteRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), - CustomSearchConfiguration.builder().build(), - QueryFilterRewriteChain.EMPTY); + private static SearchConfiguration testQueryConfig; + private static AutocompleteRequestHandler handler; private OperationContext mockOpContext = TestOperationContexts.systemContextNoSearchAuthorization(mock(EntityRegistry.class)); + static { + testQueryConfig = new SearchConfiguration(); + testQueryConfig.setMaxTermBucketSize(20); + + ExactMatchConfiguration exactMatchConfiguration = new ExactMatchConfiguration(); + exactMatchConfiguration.setExclusive(false); + exactMatchConfiguration.setExactFactor(10.0f); + exactMatchConfiguration.setWithPrefix(true); + exactMatchConfiguration.setPrefixFactor(6.0f); + exactMatchConfiguration.setCaseSensitivityFactor(0.7f); + exactMatchConfiguration.setEnableStructured(true); + + WordGramConfiguration wordGramConfiguration = new WordGramConfiguration(); + wordGramConfiguration.setTwoGramFactor(1.2f); + wordGramConfiguration.setThreeGramFactor(1.5f); + wordGramConfiguration.setFourGramFactor(1.8f); + + PartialConfiguration partialConfiguration = new PartialConfiguration(); + partialConfiguration.setFactor(0.4f); + partialConfiguration.setUrnFactor(0.7f); + + testQueryConfig.setExactMatch(exactMatchConfiguration); + testQueryConfig.setWordGram(wordGramConfiguration); + testQueryConfig.setPartial(partialConfiguration); + + handler = + AutocompleteRequestHandler.getBuilder( + TestEntitySpecBuilder.getSpec(), + CustomSearchConfiguration.builder().build(), + QueryFilterRewriteChain.EMPTY, + testQueryConfig); + } + private static final QueryConfiguration TEST_QUERY_CONFIG = QueryConfiguration.builder() .queryRegex(".*") @@ -88,9 +122,12 @@ public void testDefaultAutocompleteRequest() { BoolQueryBuilder wrapper = (BoolQueryBuilder) ((FunctionScoreQueryBuilder) sourceBuilder.query()).query(); BoolQueryBuilder query = (BoolQueryBuilder) extractNestedQuery(wrapper); - assertEquals(query.should().size(), 3); + assertEquals(query.should().size(), 4); - MultiMatchQueryBuilder autocompleteQuery = (MultiMatchQueryBuilder) query.should().get(2); + MatchQueryBuilder matchQueryBuilder = (MatchQueryBuilder) query.should().get(0); + assertEquals("keyPart1.keyword", matchQueryBuilder.fieldName()); + + MultiMatchQueryBuilder autocompleteQuery = (MultiMatchQueryBuilder) query.should().get(3); Map queryFields = autocompleteQuery.fields(); assertTrue(queryFields.containsKey("keyPart1.ngram")); assertTrue(queryFields.containsKey("keyPart1.ngram._2gram")); @@ -99,7 +136,7 @@ public void testDefaultAutocompleteRequest() { assertEquals(autocompleteQuery.type(), MultiMatchQueryBuilder.Type.BOOL_PREFIX); MatchPhrasePrefixQueryBuilder prefixQuery = - (MatchPhrasePrefixQueryBuilder) query.should().get(0); + (MatchPhrasePrefixQueryBuilder) query.should().get(1); assertEquals("keyPart1.delimited", prefixQuery.fieldName()); assertEquals(wrapper.mustNot().size(), 1); @@ -108,15 +145,16 @@ public void testDefaultAutocompleteRequest() { assertEquals(removedFilter.value(), true); HighlightBuilder highlightBuilder = sourceBuilder.highlighter(); List highlightedFields = highlightBuilder.fields(); - assertEquals(highlightedFields.size(), 8); + assertEquals(highlightedFields.size(), 9); assertEquals(highlightedFields.get(0).name(), "keyPart1"); assertEquals(highlightedFields.get(1).name(), "keyPart1.*"); assertEquals(highlightedFields.get(2).name(), "keyPart1.ngram"); assertEquals(highlightedFields.get(3).name(), "keyPart1.delimited"); - assertEquals(highlightedFields.get(4).name(), "urn"); - assertEquals(highlightedFields.get(5).name(), "urn.*"); - assertEquals(highlightedFields.get(6).name(), "urn.ngram"); - assertEquals(highlightedFields.get(7).name(), "urn.delimited"); + assertEquals(highlightedFields.get(4).name(), "keyPart1.keyword"); + assertEquals(highlightedFields.get(5).name(), "urn"); + assertEquals(highlightedFields.get(6).name(), "urn.*"); + assertEquals(highlightedFields.get(7).name(), "urn.ngram"); + assertEquals(highlightedFields.get(8).name(), "urn.delimited"); } @Test @@ -130,9 +168,12 @@ public void testAutocompleteRequestWithField() { (BoolQueryBuilder) ((FunctionScoreQueryBuilder) sourceBuilder.query()).query(); assertEquals(wrapper.should().size(), 1); BoolQueryBuilder query = (BoolQueryBuilder) extractNestedQuery(wrapper); - assertEquals(query.should().size(), 2); + assertEquals(query.should().size(), 3); - MultiMatchQueryBuilder autocompleteQuery = (MultiMatchQueryBuilder) query.should().get(1); + MatchQueryBuilder matchQueryBuilder = (MatchQueryBuilder) query.should().get(0); + assertEquals("field.keyword", matchQueryBuilder.fieldName()); + + MultiMatchQueryBuilder autocompleteQuery = (MultiMatchQueryBuilder) query.should().get(2); Map queryFields = autocompleteQuery.fields(); assertTrue(queryFields.containsKey("field.ngram")); assertTrue(queryFields.containsKey("field.ngram._2gram")); @@ -141,7 +182,7 @@ public void testAutocompleteRequestWithField() { assertEquals(autocompleteQuery.type(), MultiMatchQueryBuilder.Type.BOOL_PREFIX); MatchPhrasePrefixQueryBuilder prefixQuery = - (MatchPhrasePrefixQueryBuilder) query.should().get(0); + (MatchPhrasePrefixQueryBuilder) query.should().get(1); assertEquals("field.delimited", prefixQuery.fieldName()); MatchQueryBuilder removedFilter = (MatchQueryBuilder) wrapper.mustNot().get(0); @@ -149,11 +190,12 @@ public void testAutocompleteRequestWithField() { assertEquals(removedFilter.value(), true); HighlightBuilder highlightBuilder = sourceBuilder.highlighter(); List highlightedFields = highlightBuilder.fields(); - assertEquals(highlightedFields.size(), 4); + assertEquals(highlightedFields.size(), 5); assertEquals(highlightedFields.get(0).name(), "field"); assertEquals(highlightedFields.get(1).name(), "field.*"); assertEquals(highlightedFields.get(2).name(), "field.ngram"); assertEquals(highlightedFields.get(3).name(), "field.delimited"); + assertEquals(highlightedFields.get(4).name(), "field.keyword"); } @Test @@ -174,7 +216,8 @@ public void testCustomConfigWithDefault() { .build()) .build())) .build(), - QueryFilterRewriteChain.EMPTY); + QueryFilterRewriteChain.EMPTY, + testQueryConfig); SearchRequest autocompleteRequest = withoutDefaultQuery.getSearchRequest(mockOpContext, "input", null, null, 10); @@ -200,7 +243,8 @@ public void testCustomConfigWithDefault() { .build()) .build())) .build(), - QueryFilterRewriteChain.EMPTY); + QueryFilterRewriteChain.EMPTY, + testQueryConfig); autocompleteRequest = withDefaultQuery.getSearchRequest(mockOpContext, "input", null, null, 10); sourceBuilder = autocompleteRequest.source(); @@ -215,7 +259,7 @@ public void testCustomConfigWithDefault() { BoolQueryBuilder defaultQuery = (BoolQueryBuilder) shouldQueries.stream().filter(qb -> qb instanceof BoolQueryBuilder).findFirst().get(); - assertEquals(defaultQuery.should().size(), 3); + assertEquals(defaultQuery.should().size(), 4); // Custom customQuery = @@ -243,7 +287,8 @@ public void testCustomConfigWithInheritedQueryFunctionScores() { .build()) .build())) .build(), - QueryFilterRewriteChain.EMPTY); + QueryFilterRewriteChain.EMPTY, + testQueryConfig); SearchRequest autocompleteRequest = withInherit.getSearchRequest(mockOpContext, "input", null, null, 10); @@ -282,7 +327,8 @@ public void testCustomConfigWithInheritedQueryFunctionScores() { .build()) .build())) .build(), - QueryFilterRewriteChain.EMPTY); + QueryFilterRewriteChain.EMPTY, + testQueryConfig); autocompleteRequest = noQueryCustomization.getSearchRequest(mockOpContext, "input", null, null, 10); @@ -345,7 +391,8 @@ public void testCustomConfigWithFunctionScores() { "deprecated", Map.of("value", false))))))) .build())) .build(), - QueryFilterRewriteChain.EMPTY); + QueryFilterRewriteChain.EMPTY, + testQueryConfig); SearchRequest autocompleteRequest = explicitNoInherit.getSearchRequest(mockOpContext, "input", null, null, 10); @@ -398,7 +445,8 @@ public void testCustomConfigWithFunctionScores() { "deprecated", Map.of("value", false))))))) .build())) .build(), - QueryFilterRewriteChain.EMPTY); + QueryFilterRewriteChain.EMPTY, + testQueryConfig); autocompleteRequest = explicit.getSearchRequest(mockOpContext, "input", null, null, 10); sourceBuilder = autocompleteRequest.source(); @@ -411,7 +459,7 @@ public void testCustomConfigWithFunctionScores() { assertEquals(customQuery, QueryBuilders.matchAllQuery()); // standard query still present - assertEquals(((BoolQueryBuilder) query.should().get(1)).should().size(), 3); + assertEquals(((BoolQueryBuilder) query.should().get(1)).should().size(), 4); // custom functions included assertEquals(wrapper.filterFunctionBuilders(), expectedCustomScoreFunctions); diff --git a/metadata-io/src/test/resources/elasticsearch/sample_data/containerindex_v2.json.gz b/metadata-io/src/test/resources/elasticsearch/sample_data/containerindex_v2.json.gz index 2fa49c810abfa1af8cef95ac79136377d4b1ce4c..bd36747255f8604c37bc50fceb13293218a46330 100644 GIT binary patch literal 335 zcmV-V0kHlbiwFqPZ4PGu17mM)bYW?3WpZh5WMz0?b}}w%b8l_{?T|lf!!QuV_kIk` zHpGe(Qb9H^U0PGRl@QL-nHa1;mCh|8^t)Gf6K64$ln#Xigoekx_wIfN)_KE3|BJQ} z(i(!spcgz-IiKa2&DEm1U*&SK0tqT#u28B>0!;xL)=8Ry$9i!bH#ZG8EX zZ9eY~zo14K=nQSNq|lL-R=e~#@c$u>yT5#I{4>na(ck#jP_rqjLlL2NH=>sl17mM)bYW?3WpZh5WMz0?b}}w%b8l_{&5$u`!!Qtq_x=pc zF~o`!T!A+)U0PGRl@QL-1q&?6mCh|8^uJef9eXN;PKD!!$Gz|F-j$8sGLgFIwUEvd zXp3I(Tors Date: Wed, 16 Oct 2024 01:12:58 +0900 Subject: [PATCH 15/40] feat: add contributor pr open comment action (#11487) Co-authored-by: Harshal Sheth --- .../workflows/contributor-open-pr-comment.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/contributor-open-pr-comment.yml diff --git a/.github/workflows/contributor-open-pr-comment.yml b/.github/workflows/contributor-open-pr-comment.yml new file mode 100644 index 00000000000000..b3da16ca994baa --- /dev/null +++ b/.github/workflows/contributor-open-pr-comment.yml @@ -0,0 +1,39 @@ +name: PR Comment + +on: + pull_request: + types: [opened] + +jobs: + post-pr-opened-comment: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Get and Format Username (PR only) + if: github.event_name == 'pull_request' + run: | + formatted_username=$(echo "${{ github.event.pull_request.user.login }}" | tr '[:upper:]' '[:lower:]' | sed 's/ /-/g') + echo "FORMATTED_USERNAME=$formatted_username" >> $GITHUB_ENV + + - name: Create Comment (PR only) + if: github.event_name == 'pull_request' + uses: actions/github-script@v6 + with: + script: | + if (context.payload.pull_request) { + const prUser = process.env.FORMATTED_USERNAME; + const url = `https://contributors.datahubproject.io/${prUser}`; + const body = `Hello @${prUser} :smile: \n\n Thank you so much for opening a pull request!\n\n![Image](https://contributors.datahubproject.io/api/og?userId=${{ github.event.pull_request.user.login }})\nYou can check out your contributor card and see all your past stats [here](${url})!`; + + // Create a comment on the PR + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body: body + }); + } else { + console.log('Not a pull request event.'); + } From fdae71d25f68842609bd7e2ec837744b56531029 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:44:55 -0500 Subject: [PATCH 16/40] docs(ingestion): add architecture diagrams (#11628) --- docs/advanced/mcp-mcl.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/advanced/mcp-mcl.md b/docs/advanced/mcp-mcl.md index 9efb9b794954da..333891ba1a95d3 100644 --- a/docs/advanced/mcp-mcl.md +++ b/docs/advanced/mcp-mcl.md @@ -14,6 +14,18 @@ To mitigate these downsides, we are committed to providing cross-language client Ultimately, we intend to realize a state in which the Entities and Aspect schemas can be altered without requiring generated code and without maintaining a single mega-model schema (looking at you, Snapshot.pdl). The intention is that changes to the metadata model become even easier than they are today. +### Synchronous Ingestion Architecture + +

+ +

+ +### Asynchronous Ingestion Architecture + +

+ +

+ ## Modeling A Metadata Change Proposal is defined (in PDL) as follows From 5d3e464c21f384d5eb25a0291a77067a9605a9fc Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:35:41 -0500 Subject: [PATCH 17/40] feat(validations): Ingest and metadata schema validators (#11619) Co-authored-by: Pedro Silva --- docs/how/updating-datahub.md | 2 + .../aspect/validation/FieldPathValidator.java | 116 +++++++++ .../validators/FieldPathValidatorTest.java | 233 ++++++++++++++++++ .../java/com/linkedin/metadata/Constants.java | 7 + .../datahub/ingestion/run/pipeline_config.py | 17 +- .../datahub/testing/compare_metadata_json.py | 4 + .../ExecutionRequestResultValidator.java | 70 ++++++ .../ExecutionRequestResultValidatorTest.java | 166 +++++++++++++ .../src/main/resources/entity-registry.yml | 6 + .../SpringStandardPluginConfiguration.java | 46 ++++ 10 files changed, 663 insertions(+), 4 deletions(-) create mode 100644 entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/FieldPathValidator.java create mode 100644 entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/FieldPathValidatorTest.java create mode 100644 metadata-io/src/main/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidator.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidatorTest.java diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 00e020bd2a3875..dbcc7da8467035 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -24,6 +24,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #11484 - Metadata service authentication enabled by default - #11484 - Rest API authorization enabled by default - #10472 - `SANDBOX` added as a FabricType. No rollbacks allowed once metadata with this fabric type is added without manual cleanups in databases. +- #11619 - schema field/column paths can no longer be empty strings +- #11619 - schema field/column paths can no longer be duplicated within the schema ### Potential Downtime diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/FieldPathValidator.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/FieldPathValidator.java new file mode 100644 index 00000000000000..7c279254e1bc33 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/FieldPathValidator.java @@ -0,0 +1,116 @@ +package com.linkedin.metadata.aspect.validation; + +import static com.linkedin.metadata.Constants.*; + +import com.linkedin.metadata.aspect.RetrieverContext; +import com.linkedin.metadata.aspect.batch.BatchItem; +import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator; +import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; +import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection; +import com.linkedin.schema.EditableSchemaFieldInfo; +import com.linkedin.schema.EditableSchemaMetadata; +import com.linkedin.schema.SchemaField; +import com.linkedin.schema.SchemaMetadata; +import java.util.Collection; +import java.util.Optional; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +/** + * 1. Validates the Schema Field Path specification, specifically that all field IDs must be unique + * across all fields within a schema. 2. Validates that the field path id is not empty. + * + * @see Field + * Path V2 docs + */ +@Setter +@Getter +@Accessors(chain = true) +public class FieldPathValidator extends AspectPayloadValidator { + @Nonnull private AspectPluginConfig config; + + /** Prevent any MCP for SchemaMetadata where field ids are duplicated. */ + @Override + protected Stream validateProposedAspects( + @Nonnull Collection mcpItems, + @Nonnull RetrieverContext retrieverContext) { + + ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection(); + + mcpItems.forEach( + i -> { + if (i.getAspectName().equals(SCHEMA_METADATA_ASPECT_NAME)) { + processSchemaMetadataAspect(i, exceptions); + } else { + processEditableSchemaMetadataAspect(i, exceptions); + } + }); + + return exceptions.streamAllExceptions(); + } + + @Override + protected Stream validatePreCommitAspects( + @Nonnull Collection changeMCPs, @Nonnull RetrieverContext retrieverContext) { + return Stream.of(); + } + + private static void processEditableSchemaMetadataAspect( + BatchItem i, ValidationExceptionCollection exceptions) { + final EditableSchemaMetadata schemaMetadata = i.getAspect(EditableSchemaMetadata.class); + final long uniquePaths = + validateAndCount( + i, + schemaMetadata.getEditableSchemaFieldInfo().stream() + .map(EditableSchemaFieldInfo::getFieldPath), + exceptions); + + if (uniquePaths != schemaMetadata.getEditableSchemaFieldInfo().size()) { + exceptions.addException( + i, + String.format( + "Cannot perform %s action on proposal. EditableSchemaMetadata aspect has duplicated field paths", + i.getChangeType())); + } + } + + private static void processSchemaMetadataAspect( + BatchItem i, ValidationExceptionCollection exceptions) { + final SchemaMetadata schemaMetadata = i.getAspect(SchemaMetadata.class); + final long uniquePaths = + validateAndCount( + i, schemaMetadata.getFields().stream().map(SchemaField::getFieldPath), exceptions); + + if (uniquePaths != schemaMetadata.getFields().size()) { + exceptions.addException( + i, + String.format( + "Cannot perform %s action on proposal. SchemaMetadata aspect has duplicated field paths", + i.getChangeType())); + } + } + + private static long validateAndCount( + BatchItem i, Stream fieldPaths, ValidationExceptionCollection exceptions) { + return fieldPaths + .distinct() + // inspect the stream of fieldPath validation errors since we're already iterating + .peek( + fieldPath -> + validateFieldPath(fieldPath) + .ifPresent(message -> exceptions.addException(i, message))) + .count(); + } + + private static Optional validateFieldPath(String fieldPath) { + if (fieldPath == null || fieldPath.isEmpty()) { + return Optional.of("SchemaMetadata aspect has empty field path."); + } + return Optional.empty(); + } +} diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/FieldPathValidatorTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/FieldPathValidatorTest.java new file mode 100644 index 00000000000000..bd5912764edce3 --- /dev/null +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/FieldPathValidatorTest.java @@ -0,0 +1,233 @@ +package com.linkedin.metadata.aspect.validators; + +import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; + +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.aspect.GraphRetriever; +import com.linkedin.metadata.aspect.RetrieverContext; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; +import com.linkedin.metadata.aspect.validation.CreateIfNotExistsValidator; +import com.linkedin.metadata.aspect.validation.FieldPathValidator; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.schema.EditableSchemaFieldInfo; +import com.linkedin.schema.EditableSchemaFieldInfoArray; +import com.linkedin.schema.EditableSchemaMetadata; +import com.linkedin.schema.SchemaField; +import com.linkedin.schema.SchemaFieldArray; +import com.linkedin.schema.SchemaFieldDataType; +import com.linkedin.schema.SchemaMetadata; +import com.linkedin.schema.StringType; +import com.linkedin.test.metadata.aspect.TestEntityRegistry; +import com.linkedin.test.metadata.aspect.batch.TestMCP; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +public class FieldPathValidatorTest { + + private static final AspectPluginConfig validatorConfig = + AspectPluginConfig.builder() + .supportedOperations( + Arrays.stream(ChangeType.values()) + .map(Objects::toString) + .collect(Collectors.toList())) + .className(CreateIfNotExistsValidator.class.getName()) + .supportedEntityAspectNames(List.of(AspectPluginConfig.EntityAspectName.ALL)) + .enabled(true) + .build(); + private EntityRegistry entityRegistry; + private RetrieverContext mockRetrieverContext; + private static final DatasetUrn TEST_DATASET_URN; + private final FieldPathValidator test = new FieldPathValidator().setConfig(validatorConfig); + + static { + try { + TEST_DATASET_URN = + DatasetUrn.createFromUrn( + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,test,PROD)")); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + @BeforeTest + public void init() { + entityRegistry = new TestEntityRegistry(); + AspectRetriever mockAspectRetriever = mock(AspectRetriever.class); + when(mockAspectRetriever.getEntityRegistry()).thenReturn(entityRegistry); + GraphRetriever mockGraphRetriever = mock(GraphRetriever.class); + mockRetrieverContext = mock(RetrieverContext.class); + when(mockRetrieverContext.getAspectRetriever()).thenReturn(mockAspectRetriever); + when(mockRetrieverContext.getGraphRetriever()).thenReturn(mockGraphRetriever); + } + + @Test + public void testValidateNonDuplicatedSchemaFieldPath() { + final SchemaMetadata schema = getMockSchemaMetadataAspect(false); + assertEquals( + test.validateProposed( + Set.of( + TestMCP.builder() + .changeType(ChangeType.UPSERT) + .urn(TEST_DATASET_URN) + .entitySpec(entityRegistry.getEntitySpec(TEST_DATASET_URN.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(TEST_DATASET_URN.getEntityType()) + .getAspectSpec(SCHEMA_METADATA_ASPECT_NAME)) + .recordTemplate(schema) + .build()), + mockRetrieverContext) + .count(), + 0); + } + + @Test + public void testValidateDuplicatedSchemaFieldPath() { + final SchemaMetadata schema = getMockSchemaMetadataAspect(true); + + assertEquals( + test.validateProposed( + Set.of( + TestMCP.builder() + .changeType(ChangeType.UPSERT) + .urn(TEST_DATASET_URN) + .entitySpec(entityRegistry.getEntitySpec(TEST_DATASET_URN.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(TEST_DATASET_URN.getEntityType()) + .getAspectSpec(SCHEMA_METADATA_ASPECT_NAME)) + .recordTemplate(schema) + .build()), + mockRetrieverContext) + .count(), + 1); + } + + @Test + public void testValidateNonDuplicatedEditableSchemaFieldPath() { + final EditableSchemaMetadata schema = getMockEditableSchemaMetadataAspect(false); + assertEquals( + test.validateProposed( + Set.of( + TestMCP.builder() + .changeType(ChangeType.UPSERT) + .urn(TEST_DATASET_URN) + .entitySpec(entityRegistry.getEntitySpec(TEST_DATASET_URN.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(TEST_DATASET_URN.getEntityType()) + .getAspectSpec(EDITABLE_SCHEMA_METADATA_ASPECT_NAME)) + .recordTemplate(schema) + .build()), + mockRetrieverContext) + .count(), + 0); + } + + @Test + public void testValidateDuplicatedEditableSchemaFieldPath() { + final EditableSchemaMetadata schema = getMockEditableSchemaMetadataAspect(true); + + assertEquals( + test.validateProposed( + Set.of( + TestMCP.builder() + .changeType(ChangeType.UPSERT) + .urn(TEST_DATASET_URN) + .entitySpec(entityRegistry.getEntitySpec(TEST_DATASET_URN.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(TEST_DATASET_URN.getEntityType()) + .getAspectSpec(EDITABLE_SCHEMA_METADATA_ASPECT_NAME)) + .recordTemplate(schema) + .build()), + mockRetrieverContext) + .count(), + 1); + } + + @Test + public void testEmptySchemaFieldPath() { + final SchemaMetadata schema = getMockSchemaMetadataAspect(false, ""); + TestMCP testItem = + TestMCP.builder() + .changeType(ChangeType.UPSERT) + .urn(TEST_DATASET_URN) + .entitySpec(entityRegistry.getEntitySpec(TEST_DATASET_URN.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(TEST_DATASET_URN.getEntityType()) + .getAspectSpec(SCHEMA_METADATA_ASPECT_NAME)) + .recordTemplate(schema) + .build(); + + Set exceptions = + test.validateProposed(Set.of(testItem), mockRetrieverContext).collect(Collectors.toSet()); + + assertEquals( + exceptions, + Set.of( + AspectValidationException.forItem( + testItem, "SchemaMetadata aspect has empty field path."))); + } + + private static SchemaMetadata getMockSchemaMetadataAspect(boolean duplicateFields) { + return getMockSchemaMetadataAspect(duplicateFields, null); + } + + private static SchemaMetadata getMockSchemaMetadataAspect( + boolean duplicateFields, @Nullable String fieldPath) { + List fields = new ArrayList<>(); + fields.add( + new SchemaField() + .setType( + new SchemaFieldDataType() + .setType(SchemaFieldDataType.Type.create(new StringType()))) + .setNullable(false) + .setNativeDataType("string") + .setFieldPath(fieldPath == null ? "test" : fieldPath)); + + if (duplicateFields) { + fields.add( + new SchemaField() + .setType( + new SchemaFieldDataType() + .setType(SchemaFieldDataType.Type.create(new StringType()))) + .setNullable(false) + .setNativeDataType("string") + .setFieldPath(fieldPath == null ? "test" : fieldPath)); + } + + return new SchemaMetadata() + .setPlatform(TEST_DATASET_URN.getPlatformEntity()) + .setFields(new SchemaFieldArray(fields)); + } + + private static EditableSchemaMetadata getMockEditableSchemaMetadataAspect( + boolean duplicateFields) { + + List fields = new ArrayList<>(); + fields.add(new EditableSchemaFieldInfo().setFieldPath("test")); + + if (duplicateFields) { + fields.add(new EditableSchemaFieldInfo().setFieldPath("test")); + } + + return new EditableSchemaMetadata() + .setEditableSchemaFieldInfo(new EditableSchemaFieldInfoArray(fields)); + } +} diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index e085a5876a42b0..8961677b568788 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -319,6 +319,13 @@ public class Constants { public static final String EXECUTION_REQUEST_INPUT_ASPECT_NAME = "dataHubExecutionRequestInput"; public static final String EXECUTION_REQUEST_SIGNAL_ASPECT_NAME = "dataHubExecutionRequestSignal"; public static final String EXECUTION_REQUEST_RESULT_ASPECT_NAME = "dataHubExecutionRequestResult"; + public static final String EXECUTION_REQUEST_STATUS_RUNNING = "RUNNING"; + public static final String EXECUTION_REQUEST_STATUS_FAILURE = "FAILURE"; + public static final String EXECUTION_REQUEST_STATUS_SUCCESS = "SUCCESS"; + public static final String EXECUTION_REQUEST_STATUS_TIMEOUT = "TIMEOUT"; + public static final String EXECUTION_REQUEST_STATUS_CANCELLED = "CANCELLED"; + public static final String EXECUTION_REQUEST_STATUS_ABORTED = "ABORTED"; + public static final String EXECUTION_REQUEST_STATUS_DUPLICATE = "DUPLICATE"; // DataHub Access Token public static final String ACCESS_TOKEN_KEY_ASPECT_NAME = "dataHubAccessTokenKey"; diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py index 98629ba030695a..2b2f992249f1e8 100644 --- a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py +++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py @@ -1,6 +1,7 @@ import datetime import logging -import uuid +import random +import string from typing import Any, Dict, List, Optional from pydantic import Field, validator @@ -71,6 +72,15 @@ class FlagsConfig(ConfigModel): ) +def _generate_run_id(source_type: Optional[str] = None) -> str: + current_time = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S") + random_suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) + + if source_type is None: + source_type = "ingestion" + return f"{source_type}-{current_time}-{random_suffix}" + + class PipelineConfig(ConfigModel): source: SourceConfig sink: Optional[DynamicTypedConfig] = None @@ -91,12 +101,11 @@ def run_id_should_be_semantic( cls, v: Optional[str], values: Dict[str, Any], **kwargs: Any ) -> str: if v == DEFAULT_RUN_ID: + source_type = None if "source" in values and hasattr(values["source"], "type"): source_type = values["source"].type - current_time = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S") - return f"{source_type}-{current_time}" - return str(uuid.uuid1()) # default run_id if we cannot infer a source type + return _generate_run_id(source_type) else: assert v is not None return v diff --git a/metadata-ingestion/src/datahub/testing/compare_metadata_json.py b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py index 61b222f8d2dd50..155773f9898b47 100644 --- a/metadata-ingestion/src/datahub/testing/compare_metadata_json.py +++ b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py @@ -27,6 +27,8 @@ r"root\[\d+\]\['aspect'\]\['json'\]\['lastUpdatedTimestamp'\]", r"root\[\d+\]\['aspect'\]\['json'\]\['created'\]", r"root\[\d+\]\['aspect'\]\['json'\]\['lastModified'\]", + r"root\[\d+\].*?\['systemMetadata'\]\['runId'\]", + r"root\[\d+\].*?\['systemMetadata'\]\['lastRunId'\]", ] @@ -82,6 +84,8 @@ def assert_metadata_files_equal( json_path = f"root[{i}]['aspect']['json'][{j}]['value']" ignore_paths = (*ignore_paths, re.escape(json_path)) + ignore_paths = (*ignore_paths, *default_exclude_paths) + diff = diff_metadata_json(output, golden, ignore_paths, ignore_order=ignore_order) if diff and update_golden: if isinstance(diff, MCPDiff) and diff.is_delta_valid: diff --git a/metadata-io/src/main/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidator.java b/metadata-io/src/main/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidator.java new file mode 100644 index 00000000000000..b77d3b48d5bd58 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidator.java @@ -0,0 +1,70 @@ +package com.linkedin.metadata.aspect.validation; + +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_ABORTED; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_CANCELLED; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_DUPLICATE; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_SUCCESS; + +import com.linkedin.execution.ExecutionRequestResult; +import com.linkedin.metadata.aspect.RetrieverContext; +import com.linkedin.metadata.aspect.batch.BatchItem; +import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator; +import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; +import java.util.Collection; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; + +/** A Validator for StructuredProperties Aspect that is attached to entities like Datasets, etc. */ +@Setter +@Getter +@Slf4j +@Accessors(chain = true) +public class ExecutionRequestResultValidator extends AspectPayloadValidator { + private static final Set IMMUTABLE_STATUS = + Set.of( + EXECUTION_REQUEST_STATUS_ABORTED, + EXECUTION_REQUEST_STATUS_CANCELLED, + EXECUTION_REQUEST_STATUS_SUCCESS, + EXECUTION_REQUEST_STATUS_DUPLICATE); + + @Nonnull private AspectPluginConfig config; + + @Override + protected Stream validateProposedAspects( + @Nonnull Collection mcpItems, + @Nonnull RetrieverContext retrieverContext) { + return Stream.of(); + } + + @Override + protected Stream validatePreCommitAspects( + @Nonnull Collection changeMCPs, @Nonnull RetrieverContext retrieverContext) { + return changeMCPs.stream() + .filter(item -> item.getPreviousRecordTemplate() != null) + .map( + item -> { + ExecutionRequestResult existingResult = + item.getPreviousAspect(ExecutionRequestResult.class); + + if (IMMUTABLE_STATUS.contains(existingResult.getStatus())) { + ExecutionRequestResult currentResult = item.getAspect(ExecutionRequestResult.class); + return AspectValidationException.forItem( + item, + String.format( + "Invalid update to immutable state for aspect dataHubExecutionRequestResult. Execution urn: %s previous status: %s. Denied status update: %s", + item.getUrn(), existingResult.getStatus(), currentResult.getStatus())); + } + + return null; + }) + .filter(Objects::nonNull); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidatorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidatorTest.java new file mode 100644 index 00000000000000..f46772ca7b350d --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/aspect/validation/ExecutionRequestResultValidatorTest.java @@ -0,0 +1,166 @@ +package com.linkedin.metadata.aspect.validation; + +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_ENTITY_NAME; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_ABORTED; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_CANCELLED; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_DUPLICATE; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_FAILURE; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_RUNNING; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_SUCCESS; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_STATUS_TIMEOUT; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.execution.ExecutionRequestResult; +import com.linkedin.metadata.aspect.RetrieverContext; +import com.linkedin.metadata.aspect.SystemAspect; +import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; +import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; +import com.linkedin.metadata.utils.AuditStampUtils; +import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.test.metadata.context.TestOperationContexts; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.testng.annotations.Test; + +public class ExecutionRequestResultValidatorTest { + private static final OperationContext TEST_CONTEXT = + TestOperationContexts.systemContextNoSearchAuthorization(); + private static final AspectPluginConfig TEST_PLUGIN_CONFIG = + AspectPluginConfig.builder() + .className(ExecutionRequestResultValidator.class.getName()) + .enabled(true) + .supportedOperations(List.of("UPSERT")) + .supportedEntityAspectNames( + List.of( + AspectPluginConfig.EntityAspectName.builder() + .entityName(EXECUTION_REQUEST_ENTITY_NAME) + .aspectName(EXECUTION_REQUEST_RESULT_ASPECT_NAME) + .build())) + .build(); + private static final Urn TEST_URN = UrnUtils.getUrn("urn:li:dataHubExecutionRequest:xyz"); + + @Test + public void testAllowed() { + ExecutionRequestResultValidator test = new ExecutionRequestResultValidator(); + test.setConfig(TEST_PLUGIN_CONFIG); + + Set allowedUpdateStates = + Set.of( + EXECUTION_REQUEST_STATUS_RUNNING, + EXECUTION_REQUEST_STATUS_FAILURE, + EXECUTION_REQUEST_STATUS_TIMEOUT); + Set destinationStates = new HashSet<>(allowedUpdateStates); + destinationStates.addAll( + Set.of( + EXECUTION_REQUEST_STATUS_ABORTED, + EXECUTION_REQUEST_STATUS_CANCELLED, + EXECUTION_REQUEST_STATUS_SUCCESS, + EXECUTION_REQUEST_STATUS_DUPLICATE)); + + List testItems = + new ArrayList<>( + // Tests with previous state + allowedUpdateStates.stream() + .flatMap( + prevState -> + destinationStates.stream() + .map( + destState -> { + SystemAspect prevData = mock(SystemAspect.class); + when(prevData.getRecordTemplate()) + .thenReturn( + new ExecutionRequestResult().setStatus(prevState)); + return ChangeItemImpl.builder() + .changeType(ChangeType.UPSERT) + .urn(TEST_URN) + .aspectName(EXECUTION_REQUEST_RESULT_ASPECT_NAME) + .recordTemplate( + new ExecutionRequestResult().setStatus(destState)) + .previousSystemAspect(prevData) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(TEST_CONTEXT.getAspectRetriever()); + })) + .toList()); + // Tests with no previous + testItems.addAll( + destinationStates.stream() + .map( + destState -> + ChangeItemImpl.builder() + .changeType(ChangeType.UPSERT) + .urn(TEST_URN) + .aspectName(EXECUTION_REQUEST_RESULT_ASPECT_NAME) + .recordTemplate(new ExecutionRequestResult().setStatus(destState)) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(TEST_CONTEXT.getAspectRetriever())) + .toList()); + + List result = + test.validatePreCommitAspects(testItems, mock(RetrieverContext.class)).toList(); + + assertTrue(result.isEmpty(), "Did not expect any validation errors."); + } + + @Test + public void testDenied() { + ExecutionRequestResultValidator test = new ExecutionRequestResultValidator(); + test.setConfig(TEST_PLUGIN_CONFIG); + + Set deniedUpdateStates = + Set.of( + EXECUTION_REQUEST_STATUS_ABORTED, + EXECUTION_REQUEST_STATUS_CANCELLED, + EXECUTION_REQUEST_STATUS_SUCCESS, + EXECUTION_REQUEST_STATUS_DUPLICATE); + Set destinationStates = new HashSet<>(deniedUpdateStates); + destinationStates.addAll( + Set.of( + EXECUTION_REQUEST_STATUS_RUNNING, + EXECUTION_REQUEST_STATUS_FAILURE, + EXECUTION_REQUEST_STATUS_TIMEOUT)); + + List testItems = + new ArrayList<>( + // Tests with previous state + deniedUpdateStates.stream() + .flatMap( + prevState -> + destinationStates.stream() + .map( + destState -> { + SystemAspect prevData = mock(SystemAspect.class); + when(prevData.getRecordTemplate()) + .thenReturn( + new ExecutionRequestResult().setStatus(prevState)); + return ChangeItemImpl.builder() + .changeType(ChangeType.UPSERT) + .urn(TEST_URN) + .aspectName(EXECUTION_REQUEST_RESULT_ASPECT_NAME) + .recordTemplate( + new ExecutionRequestResult().setStatus(destState)) + .previousSystemAspect(prevData) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(TEST_CONTEXT.getAspectRetriever()); + })) + .toList()); + + List result = + test.validatePreCommitAspects(testItems, mock(RetrieverContext.class)).toList(); + + assertEquals( + result.size(), + deniedUpdateStates.size() * destinationStates.size(), + "Expected ALL items to be denied."); + } +} diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 9b692b51dc2b5f..ec9c3fee1c404c 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -673,6 +673,12 @@ plugins: supportedEntityAspectNames: - entityName: '*' aspectName: '*' + - className: 'com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator' + enabled: true + spring: + enabled: true + packageScan: + - com.linkedin.gms.factory.plugins mcpSideEffects: - className: 'com.linkedin.metadata.structuredproperties.hooks.PropertyDefinitionDeleteSideEffect' packageScan: diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java index 4a2095685abe1f..943b1c7184a60d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java @@ -1,5 +1,8 @@ package com.linkedin.gms.factory.plugins; +import static com.linkedin.metadata.Constants.EDITABLE_SCHEMA_METADATA_ASPECT_NAME; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_ENTITY_NAME; +import static com.linkedin.metadata.Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME; import static com.linkedin.metadata.Constants.SCHEMA_METADATA_ASPECT_NAME; import com.linkedin.metadata.Constants; @@ -7,6 +10,9 @@ import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; import com.linkedin.metadata.aspect.plugins.hooks.MCPSideEffect; import com.linkedin.metadata.aspect.plugins.hooks.MutationHook; +import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator; +import com.linkedin.metadata.aspect.validation.ExecutionRequestResultValidator; +import com.linkedin.metadata.aspect.validation.FieldPathValidator; import com.linkedin.metadata.dataproducts.sideeffects.DataProductUnsetSideEffect; import com.linkedin.metadata.schemafields.sideeffects.SchemaFieldSideEffect; import com.linkedin.metadata.timeline.eventgenerator.EntityChangeEventGeneratorRegistry; @@ -21,6 +27,7 @@ @Configuration @Slf4j public class SpringStandardPluginConfiguration { + private static final String ALL = "*"; @Value("${metadataChangeProposal.validation.ignoreUnknown}") private boolean ignoreUnknownEnabled; @@ -104,4 +111,43 @@ public MCPSideEffect dataProductUnsetSideEffect() { log.info("Initialized {}", SchemaFieldSideEffect.class.getName()); return new DataProductUnsetSideEffect().setConfig(config); } + + @Bean + public AspectPayloadValidator fieldPathValidator() { + return new FieldPathValidator() + .setConfig( + AspectPluginConfig.builder() + .className(FieldPathValidator.class.getName()) + .enabled(true) + .supportedOperations( + List.of("CREATE", "CREATE_ENTITY", "UPSERT", "UPDATE", "RESTATE")) + .supportedEntityAspectNames( + List.of( + AspectPluginConfig.EntityAspectName.builder() + .entityName(ALL) + .aspectName(SCHEMA_METADATA_ASPECT_NAME) + .build(), + AspectPluginConfig.EntityAspectName.builder() + .entityName(ALL) + .aspectName(EDITABLE_SCHEMA_METADATA_ASPECT_NAME) + .build())) + .build()); + } + + @Bean + public AspectPayloadValidator dataHubExecutionRequestResultValidator() { + return new ExecutionRequestResultValidator() + .setConfig( + AspectPluginConfig.builder() + .className(ExecutionRequestResultValidator.class.getName()) + .enabled(true) + .supportedOperations(List.of("UPSERT", "UPDATE")) + .supportedEntityAspectNames( + List.of( + AspectPluginConfig.EntityAspectName.builder() + .entityName(EXECUTION_REQUEST_ENTITY_NAME) + .aspectName(EXECUTION_REQUEST_RESULT_ASPECT_NAME) + .build())) + .build()); + } } From 555f391c24fe245c96fdbdb7462ab80e9a87acaa Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:14:21 -0500 Subject: [PATCH 18/40] fix(ci): Update contributor-open-pr-comment.yml (#11631) --- .github/workflows/contributor-open-pr-comment.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/contributor-open-pr-comment.yml b/.github/workflows/contributor-open-pr-comment.yml index b3da16ca994baa..2f700290ee0f28 100644 --- a/.github/workflows/contributor-open-pr-comment.yml +++ b/.github/workflows/contributor-open-pr-comment.yml @@ -4,6 +4,9 @@ on: pull_request: types: [opened] +permissions: + pull-requests: write + jobs: post-pr-opened-comment: runs-on: ubuntu-latest From 14a22bfeaf8bedcb873da6dbcb0a40833fde96a3 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:14:46 -0500 Subject: [PATCH 19/40] fix(ci): add runtime limit (#11630) --- .github/workflows/metadata-ingestion.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index c718febca398a9..92dcb3d8ac2890 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -26,6 +26,7 @@ concurrency: jobs: metadata-ingestion: runs-on: ubuntu-latest + timeout-minutes: 40 env: SPARK_VERSION: 3.3.2 DATAHUB_TELEMETRY_ENABLED: false From 2bc96e9e69f4523cd5eb27bf0b60e309694b7b17 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:43:36 -0500 Subject: [PATCH 20/40] fix(ci): metadata-io req python (#11632) --- .github/workflows/metadata-io.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index 7018b42949e892..5ee2223d71b039 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -57,17 +57,16 @@ jobs: - name: Disk Check run: df -h . && docker images - uses: acryldata/sane-checkout-action@v3 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" - name: Set up JDK 17 uses: actions/setup-java@v4 with: distribution: "zulu" java-version: 17 - uses: gradle/actions/setup-gradle@v3 - - uses: actions/setup-python@v5 - if: ${{ needs.setup.outputs.ingestion_change == 'true' }} - with: - python-version: "3.10" - cache: "pip" - name: Gradle build (and test) run: | ./gradlew :metadata-io:test From 2f2a7af58428563a3acb2963c8278445513f8e85 Mon Sep 17 00:00:00 2001 From: Hyejin Yoon <0327jane@gmail.com> Date: Wed, 16 Oct 2024 15:49:59 +0900 Subject: [PATCH 21/40] feat: add quickstart post (#11623) --- .../examples/mce_files/bootstrap_mce.json | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/metadata-ingestion/examples/mce_files/bootstrap_mce.json b/metadata-ingestion/examples/mce_files/bootstrap_mce.json index bc218e5e8c2d53..f0c4e7ff996ed3 100644 --- a/metadata-ingestion/examples/mce_files/bootstrap_mce.json +++ b/metadata-ingestion/examples/mce_files/bootstrap_mce.json @@ -3613,6 +3613,33 @@ }, "systemMetadata": null }, + { + "entityType": "post", + "entityUrn": "urn:li:post:f3a68539-f7e4-4c41-a4fd-9e57c085d8de", + "changeType": "UPSERT", + "aspectName": "postInfo", + "aspect": { + "json": { + "type": "HOME_PAGE_ANNOUNCEMENT", + "content": { + "title": "Join Metadata & AI Summit 2024", + "type": "LINK", + "link": "http://www.acryldata.io/conference?utm_source=datahub_quickstart&utm_medium=metadata_ai_2024&utm_campaign=pinned_announcement", + "media": { + "type": "IMAGE", + "location": "https://formulatedby.com/wp-content/uploads/2024/07/0193320a6d93e7508d1598f7b24662f75a87e92f-352x456-1.svg" + } + }, + "created": 1712547125049, + "lastModified": 1712547125049 + } + }, + "systemMetadata": { + "lastObserved": 1712548844816, + "runId": "datahub-2024_04_08-13_00_44", + "lastRunId": "no-run-id-provided" + } + }, { "entityType": "post", "entityUrn": "urn:li:post:f3a68539-f7e4-4c41-a4fd-9e57c085d8dd", From e76647dd7a094907bdfa22682b79b35965e5537c Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Wed, 16 Oct 2024 10:04:01 +0200 Subject: [PATCH 22/40] feat(ingest/bigquery): Generate platform resource entities for BigQuery labels (#11602) Co-authored-by: Shirshanka Das --- .../ingestion/source/bigquery_v2/bigquery.py | 1 + .../bigquery_platform_resource_helper.py | 144 +++++++++++++ .../source/bigquery_v2/bigquery_schema_gen.py | 114 +++++++++-- .../bigquery_v2/bigquery_mcp_golden.json | 192 ++++++++++++++++++ .../integration/bigquery_v2/test_bigquery.py | 41 +++- .../tests/unit/test_bigquery_source.py | 74 ++++--- 6 files changed, 517 insertions(+), 49 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index 4cc3ec50bacd46..c30dade921d257 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -188,6 +188,7 @@ def __init__(self, ctx: PipelineContext, config: BigQueryV2Config): self.sql_parser_schema_resolver, self.profiler, self.identifiers, + self.ctx.graph, ) self.add_config_to_report() diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py new file mode 100644 index 00000000000000..d2da895be985dc --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py @@ -0,0 +1,144 @@ +import logging +from dataclasses import dataclass +from typing import Optional + +import cachetools +from pydantic import BaseModel, ValidationError + +from datahub.api.entities.platformresource.platform_resource import ( + PlatformResource, + PlatformResourceKey, +) +from datahub.ingestion.graph.client import DataHubGraph +from datahub.metadata.urns import TagUrn + +logger: logging.Logger = logging.getLogger(__name__) + + +@dataclass +class BigQueryLabel: + key: str + value: Optional[str] + + def primary_key(self) -> str: + return f"{self.key}/{self.value}" if self.value else f"{self.key}" + + +class BigQueryLabelInfo(BaseModel): + datahub_urn: str + managed_by_datahub: bool + key: str + value: str + + +@dataclass() +class BigQueryLabelPlatformResource: + datahub_urn: str + project: Optional[str] + managed_by_datahub: bool + label: BigQueryLabel + + def platform_resource_key(self) -> PlatformResourceKey: + return PlatformResourceKey( + platform="bigquery", + resource_type="BigQueryLabelInfo", + platform_instance=self.project, + primary_key=self.label.primary_key(), + ) + + def platform_resource_info(self) -> BigQueryLabelInfo: + bq_label_info = BigQueryLabelInfo( + datahub_urn=self.datahub_urn, + managed_by_datahub=self.managed_by_datahub, + key=self.label.key, + value=self.label.value, + ) + return bq_label_info + + def platform_resource(self) -> PlatformResource: + return PlatformResource.create( + key=self.platform_resource_key(), + secondary_keys=[self.datahub_urn], + value=self.platform_resource_info(), + ) + + +class BigQueryPlatformResourceHelper: + def __init__( + self, + bq_project: Optional[str], + graph: Optional[DataHubGraph], + ): + self.bq_project = bq_project + self.graph = graph + + platform_resource_cache: cachetools.LRUCache = cachetools.LRUCache(maxsize=500) + + def get_platform_resource( + self, platform_resource_key: PlatformResourceKey + ) -> Optional[PlatformResource]: + # if graph is not available we always create a new PlatformResource + if not self.graph: + return None + if self.platform_resource_cache.get(platform_resource_key.primary_key): + return self.platform_resource_cache.get(platform_resource_key.primary_key) + + platform_resource = PlatformResource.from_datahub( + key=platform_resource_key, graph_client=self.graph + ) + if platform_resource: + self.platform_resource_cache[ + platform_resource_key.primary_key + ] = platform_resource + return platform_resource + return None + + def generate_label_platform_resource( + self, + bigquery_label: BigQueryLabel, + tag_urn: TagUrn, + managed_by_datahub: bool = True, + ) -> PlatformResource: + new_platform_resource = BigQueryLabelPlatformResource( + datahub_urn=tag_urn.urn(), + project=self.bq_project, + managed_by_datahub=managed_by_datahub, + label=bigquery_label, + ) + + platform_resource = self.get_platform_resource( + new_platform_resource.platform_resource_key() + ) + if platform_resource: + if ( + platform_resource.resource_info + and platform_resource.resource_info.value + ): + try: + existing_info: Optional[BigQueryLabelInfo] = platform_resource.resource_info.value.as_pydantic_object(BigQueryLabelInfo) # type: ignore + except ValidationError as e: + logger.error( + f"Error converting existing value to BigQueryLabelInfo: {e}. Creating new one. Maybe this is because of a non backward compatible schema change." + ) + existing_info = None + + if existing_info: + if ( + new_platform_resource.platform_resource_info() == existing_info + or existing_info.managed_by_datahub + ): + return platform_resource + else: + raise ValueError( + f"Datahub URN mismatch for platform resources. Old (existing) platform resource: {platform_resource} and new platform resource: {new_platform_resource}" + ) + + logger.info(f"Created platform resource {new_platform_resource}") + + self.platform_resource_cache.update( + { + new_platform_resource.platform_resource_key().primary_key: new_platform_resource.platform_resource() + } + ) + + return new_platform_resource.platform_resource() diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py index 11d06771d4e4f4..1235f638f68ff7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py @@ -6,6 +6,7 @@ from google.cloud.bigquery.table import TableListItem +from datahub.api.entities.platformresource.platform_resource import PlatformResource from datahub.configuration.pattern_utils import is_schema_allowed, is_tag_allowed from datahub.emitter.mce_builder import make_tag_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper @@ -16,6 +17,7 @@ ClassificationHandler, classification_workunit_processor, ) +from datahub.ingestion.graph.client import DataHubGraph from datahub.ingestion.source.bigquery_v2.bigquery_audit import ( BigqueryTableIdentifier, BigQueryTableRef, @@ -25,6 +27,11 @@ from datahub.ingestion.source.bigquery_v2.bigquery_helper import ( unquote_and_decode_unicode_escape_seq, ) +from datahub.ingestion.source.bigquery_v2.bigquery_platform_resource_helper import ( + BigQueryLabel, + BigQueryLabelInfo, + BigQueryPlatformResourceHelper, +) from datahub.ingestion.source.bigquery_v2.bigquery_report import BigQueryV2Report from datahub.ingestion.source.bigquery_v2.bigquery_schema import ( BigqueryColumn, @@ -84,6 +91,7 @@ GlobalTagsClass, TagAssociationClass, ) +from datahub.metadata.urns import TagUrn from datahub.sql_parsing.schema_resolver import SchemaResolver from datahub.utilities.file_backed_collections import FileBackedDict from datahub.utilities.hive_schema_to_avro import ( @@ -160,6 +168,7 @@ def __init__( sql_parser_schema_resolver: SchemaResolver, profiler: BigqueryProfiler, identifiers: BigQueryIdentifierBuilder, + graph: Optional[DataHubGraph] = None, ): self.config = config self.report = report @@ -168,6 +177,7 @@ def __init__( self.sql_parser_schema_resolver = sql_parser_schema_resolver self.profiler = profiler self.identifiers = identifiers + self.graph = graph self.classification_handler = ClassificationHandler(self.config, self.report) self.data_reader: Optional[BigQueryDataReader] = None @@ -188,6 +198,21 @@ def __init__( # Maps snapshot ref -> Snapshot self.snapshots_by_ref: FileBackedDict[BigqueryTableSnapshot] = FileBackedDict() + bq_project = ( + self.config.project_on_behalf + if self.config.project_on_behalf + else self.config.credential.project_id + if self.config.credential + else None + ) + + self.platform_resource_helper: BigQueryPlatformResourceHelper = ( + BigQueryPlatformResourceHelper( + bq_project, + self.graph, + ) + ) + @property def store_table_refs(self): return ( @@ -264,13 +289,28 @@ def gen_dataset_containers( ) -> Iterable[MetadataWorkUnit]: schema_container_key = self.gen_dataset_key(project_id, dataset) - tags_joined: Optional[List[str]] = None + tags_joined: List[str] = [] if tags and self.config.capture_dataset_label_as_tag: - tags_joined = [ - self.make_tag_from_label(k, v) - for k, v in tags.items() - if is_tag_allowed(self.config.capture_dataset_label_as_tag, k) - ] + for k, v in tags.items(): + if is_tag_allowed(self.config.capture_dataset_label_as_tag, k): + tag_urn = TagUrn.from_string(self.make_tag_urn_from_label(k, v)) + label = BigQueryLabel(key=k, value=v) + try: + platform_resource: PlatformResource = self.platform_resource_helper.generate_label_platform_resource( + label, tag_urn, managed_by_datahub=False + ) + label_info: BigQueryLabelInfo = platform_resource.resource_info.value.as_pydantic_object( # type: ignore + BigQueryLabelInfo + ) + tag_urn = TagUrn.from_string(label_info.datahub_urn) + + for mcpw in platform_resource.to_mcps(): + yield mcpw.as_workunit() + except ValueError as e: + logger.warning( + f"Failed to generate platform resource for label {k}:{v}: {e}" + ) + tags_joined.append(tag_urn.urn()) database_container_key = self.gen_project_id_key(database=project_id) @@ -676,10 +716,11 @@ def _process_snapshot( dataset_name=dataset_name, ) - def make_tag_from_label(self, key: str, value: str) -> str: - if not value.startswith(ENCODED_TAG_PREFIX): + def make_tag_urn_from_label(self, key: str, value: str) -> str: + if value: return make_tag_urn(f"""{key}:{value}""") - return self.modified_base32decode(value) + else: + return make_tag_urn(key) def gen_table_dataset_workunits( self, @@ -724,13 +765,26 @@ def gen_table_dataset_workunits( tags_to_add = None if table.labels and self.config.capture_table_label_as_tag: tags_to_add = [] - tags_to_add.extend( - [ - self.make_tag_from_label(k, v) - for k, v in table.labels.items() - if is_tag_allowed(self.config.capture_table_label_as_tag, k) - ] - ) + for k, v in table.labels.items(): + if is_tag_allowed(self.config.capture_table_label_as_tag, k): + tag_urn = TagUrn.from_string(self.make_tag_urn_from_label(k, v)) + try: + label = BigQueryLabel(key=k, value=v) + platform_resource: PlatformResource = self.platform_resource_helper.generate_label_platform_resource( + label, tag_urn, managed_by_datahub=False + ) + label_info: BigQueryLabelInfo = platform_resource.resource_info.value.as_pydantic_object( # type: ignore + BigQueryLabelInfo + ) + tag_urn = TagUrn.from_string(label_info.datahub_urn) + + for mcpw in platform_resource.to_mcps(): + yield mcpw.as_workunit() + except ValueError as e: + logger.warning( + f"Failed to generate platform resource for label {k}:{v}: {e}" + ) + tags_to_add.append(tag_urn.urn()) yield from self.gen_dataset_workunits( table=table, @@ -749,13 +803,29 @@ def gen_view_dataset_workunits( project_id: str, dataset_name: str, ) -> Iterable[MetadataWorkUnit]: - tags_to_add = None + tags_to_add = [] if table.labels and self.config.capture_view_label_as_tag: - tags_to_add = [ - self.make_tag_from_label(k, v) - for k, v in table.labels.items() - if is_tag_allowed(self.config.capture_view_label_as_tag, k) - ] + for k, v in table.labels.items(): + if is_tag_allowed(self.config.capture_view_label_as_tag, k): + tag_urn = TagUrn.from_string(self.make_tag_urn_from_label(k, v)) + try: + label = BigQueryLabel(key=k, value=v) + platform_resource: PlatformResource = self.platform_resource_helper.generate_label_platform_resource( + label, tag_urn, managed_by_datahub=False + ) + label_info: BigQueryLabelInfo = platform_resource.resource_info.value.as_pydantic_object( # type: ignore + BigQueryLabelInfo + ) + tag_urn = TagUrn.from_string(label_info.datahub_urn) + + for mcpw in platform_resource.to_mcps(): + yield mcpw.as_workunit() + except ValueError as e: + logger.warning( + f"Failed to generate platform resource for label {k}:{v}: {e}" + ) + + tags_to_add.append(tag_urn.urn()) yield from self.gen_dataset_workunits( table=table, columns=columns, diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json index 02660f0fae08ed..b268926f155b74 100644 --- a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json +++ b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json @@ -199,6 +199,49 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "platformResource", + "entityUrn": "urn:li:platformResource:79d443a7956814fdab2168e11392bbf2", + "changeType": "UPSERT", + "aspectName": "platformResourceInfo", + "aspect": { + "json": { + "resourceType": "BigQueryLabelInfo", + "primaryKey": "priority/high", + "secondaryKeys": [ + "urn:li:tag:priority:high" + ], + "value": { + "blob": "{\"datahub_urn\": \"urn:li:tag:priority:high\", \"managed_by_datahub\": false, \"key\": \"priority\", \"value\": \"high\"}", + "contentType": "JSON", + "schemaType": "JSON", + "schemaRef": "BigQueryLabelInfo" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "platformResource", + "entityUrn": "urn:li:platformResource:79d443a7956814fdab2168e11392bbf2", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:bigquery", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:bigquery,project-id-1)" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", @@ -215,6 +258,49 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "platformResource", + "entityUrn": "urn:li:platformResource:0a8c87e84bd90486c4fd57bbae6557e3", + "changeType": "UPSERT", + "aspectName": "platformResourceInfo", + "aspect": { + "json": { + "resourceType": "BigQueryLabelInfo", + "primaryKey": "purchase", + "secondaryKeys": [ + "urn:li:tag:purchase" + ], + "value": { + "blob": "{\"datahub_urn\": \"urn:li:tag:purchase\", \"managed_by_datahub\": false, \"key\": \"purchase\", \"value\": \"\"}", + "contentType": "JSON", + "schemaType": "JSON", + "schemaRef": "BigQueryLabelInfo" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "platformResource", + "entityUrn": "urn:li:platformResource:0a8c87e84bd90486c4fd57bbae6557e3", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:bigquery", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:bigquery,project-id-1)" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", @@ -309,6 +395,38 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "platformResource", + "entityUrn": "urn:li:platformResource:79d443a7956814fdab2168e11392bbf2", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "platformResource", + "entityUrn": "urn:li:platformResource:0a8c87e84bd90486c4fd57bbae6557e3", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", @@ -330,6 +448,45 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "platformResource", + "entityUrn": "urn:li:platformResource:7da6409504c5c6444b4ce60b0239b759", + "changeType": "UPSERT", + "aspectName": "platformResourceInfo", + "aspect": { + "json": { + "resourceType": "BigQueryLabelInfo", + "primaryKey": "mixedcasetag", + "value": { + "blob": "{\"datahub_urn\": \"urn:li:tag:MixedCaseTag\", \"managed_by_datahub\": true, \"key\": \"mixedcasetag\", \"value\": \"\"}", + "contentType": "JSON", + "schemaType": "JSON", + "schemaRef": "BigQueryLabelInfo" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "platformResource", + "entityUrn": "urn:li:platformResource:7da6409504c5c6444b4ce60b0239b759", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:bigquery" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", @@ -343,6 +500,9 @@ }, { "tag": "urn:li:tag:purchase" + }, + { + "tag": "urn:li:tag:MixedCaseTag" } ] } @@ -353,6 +513,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "platformResource", + "entityUrn": "urn:li:platformResource:7da6409504c5c6444b4ce60b0239b759", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", @@ -1082,5 +1258,21 @@ "runId": "bigquery-2022_02_03-07_00_00", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:MixedCaseTag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "MixedCaseTag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py index 0ac4e94a5a24f3..39cefcb42f360b 100644 --- a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py +++ b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py @@ -1,12 +1,16 @@ import random import string from datetime import datetime, timezone -from typing import Any, Dict +from typing import Any, Dict, Optional from unittest.mock import MagicMock, patch from freezegun import freeze_time from google.cloud.bigquery.table import TableListItem +from datahub.api.entities.platformresource.platform_resource import ( + PlatformResource, + PlatformResourceKey, +) from datahub.ingestion.glossary.classifier import ( ClassificationConfig, DynamicTypedClassifierConfig, @@ -14,6 +18,10 @@ from datahub.ingestion.glossary.datahub_classifier import DataHubClassifierConfig from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier from datahub.ingestion.source.bigquery_v2.bigquery_data_reader import BigQueryDataReader +from datahub.ingestion.source.bigquery_v2.bigquery_platform_resource_helper import ( + BigQueryLabelInfo, + BigQueryPlatformResourceHelper, +) from datahub.ingestion.source.bigquery_v2.bigquery_schema import ( BigqueryColumn, BigqueryDataset, @@ -51,6 +59,13 @@ def recipe(mcp_output_path: str, source_config_override: dict = {}) -> dict: "type": "bigquery", "config": { "project_ids": ["project-id-1"], + "credential": { + "project_id": "project-id-1", + "private_key_id": "private_key_id", + "private_key": "private_key", + "client_email": "client_email", + "client_id": "client_id", + }, "include_usage_statistics": False, "include_table_lineage": True, "include_data_platform_instance": True, @@ -82,6 +97,7 @@ def recipe(mcp_output_path: str, source_config_override: dict = {}) -> dict: @patch.object(BigQuerySchemaApi, "get_datasets_for_project_id") @patch.object(BigQuerySchemaApi, "get_columns_for_dataset") @patch.object(BigQueryDataReader, "get_sample_data_for_table") +@patch.object(BigQueryPlatformResourceHelper, "get_platform_resource") @patch("google.cloud.bigquery.Client") @patch("google.cloud.datacatalog_v1.PolicyTagManagerClient") @patch("google.cloud.resourcemanager_v3.ProjectsClient") @@ -89,6 +105,7 @@ def test_bigquery_v2_ingest( client, policy_tag_manager_client, projects_client, + get_platform_resource, get_sample_data_for_table, get_columns_for_dataset, get_datasets_for_project_id, @@ -104,6 +121,25 @@ def test_bigquery_v2_ingest( mcp_output_path = "{}/{}".format(tmp_path, "bigquery_mcp_output.json") dataset_name = "bigquery-dataset-1" + + def side_effect(*args: Any) -> Optional[PlatformResource]: + if args[0].primary_key == "mixedcasetag": + return PlatformResource.create( + key=PlatformResourceKey( + primary_key="mixedcasetag", + resource_type="BigQueryLabelInfo", + platform="bigquery", + ), + value=BigQueryLabelInfo( + datahub_urn="urn:li:tag:MixedCaseTag", + managed_by_datahub=True, + key="mixedcasetag", + value="", + ), + ) + return None + + get_platform_resource.side_effect = side_effect get_datasets_for_project_id.return_value = [ BigqueryDataset(name=dataset_name, location="US") ] @@ -158,7 +194,8 @@ def test_bigquery_v2_ingest( rows_count=None, labels={ "priority": "high", - "purchase": "urn_li_encoded_tag_ovzg4otmne5hiylhhjyhk4tdnbqxgzi_", + "purchase": "", + "mixedcasetag": "", }, ) get_tables_for_dataset.return_value = iter([bigquery_table]) diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/test_bigquery_source.py index 38239d150dd6b5..b605e9b3f8a3e6 100644 --- a/metadata-ingestion/tests/unit/test_bigquery_source.py +++ b/metadata-ingestion/tests/unit/test_bigquery_source.py @@ -489,30 +489,45 @@ def test_gen_table_dataset_workunits( gen = schema_gen.gen_table_dataset_workunits( bigquery_table, [], project_id, dataset_name ) - mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) - assert mcp.aspect == StatusClass(removed=False) + mcps = list(gen) + + # Helper function to find MCP by aspect type + def find_mcp_by_aspect(aspect_type): + return next( + mcp # type: ignore + for mcp in mcps + if isinstance(mcp.metadata.aspect, aspect_type) # type: ignore + ) - mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) - assert isinstance(mcp.aspect, SchemaMetadataClass) - assert mcp.aspect.schemaName == f"{project_id}.{dataset_name}.{bigquery_table.name}" - assert mcp.aspect.fields == [] + # Assert StatusClass + status_mcp = find_mcp_by_aspect(StatusClass) + assert status_mcp.metadata.aspect.removed is False - mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) - assert isinstance(mcp.aspect, DatasetPropertiesClass) - assert mcp.aspect.name == bigquery_table.name + # Assert SchemaMetadataClass + schema_mcp = find_mcp_by_aspect(SchemaMetadataClass) + assert ( + schema_mcp.metadata.aspect.schemaName + == f"{project_id}.{dataset_name}.{bigquery_table.name}" + ) + assert schema_mcp.metadata.aspect.fields == [] + + # Assert DatasetPropertiesClass + dataset_props_mcp = find_mcp_by_aspect(DatasetPropertiesClass) + assert dataset_props_mcp.metadata.aspect.name == bigquery_table.name assert ( - mcp.aspect.qualifiedName == f"{project_id}.{dataset_name}.{bigquery_table.name}" + dataset_props_mcp.metadata.aspect.qualifiedName + == f"{project_id}.{dataset_name}.{bigquery_table.name}" ) - assert mcp.aspect.description == bigquery_table.comment - assert mcp.aspect.created == TimeStampClass( + assert dataset_props_mcp.metadata.aspect.description == bigquery_table.comment + assert dataset_props_mcp.metadata.aspect.created == TimeStampClass( time=int(bigquery_table.created.timestamp() * 1000) ) - assert mcp.aspect.lastModified == TimeStampClass( + assert dataset_props_mcp.metadata.aspect.lastModified == TimeStampClass( time=int(bigquery_table.last_altered.timestamp() * 1000) ) - assert mcp.aspect.tags == [] + assert dataset_props_mcp.metadata.aspect.tags == [] - assert mcp.aspect.customProperties == { + expected_custom_properties = { "expiration_date": str(bigquery_table.expires), "size_in_bytes": str(bigquery_table.size_in_bytes), "billable_bytes_active": str(bigquery_table.active_billable_bytes), @@ -523,24 +538,33 @@ def test_gen_table_dataset_workunits( "max_shard_id": str(bigquery_table.max_shard_id), "is_sharded": "True", } + assert ( + dataset_props_mcp.metadata.aspect.customProperties == expected_custom_properties + ) - mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) - assert isinstance(mcp.aspect, GlobalTagsClass) - assert mcp.aspect.tags == [ + # Assert GlobalTagsClass + global_tags_mcp = find_mcp_by_aspect(GlobalTagsClass) + assert global_tags_mcp.metadata.aspect.tags == [ TagAssociationClass( "urn:li:tag:data_producer_owner_email:games_team-nytimes_com" ) ] - mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) - assert isinstance(mcp.aspect, ContainerClass) + # Assert ContainerClass + container_mcp = find_mcp_by_aspect(ContainerClass) + assert container_mcp is not None - mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) - assert isinstance(mcp.aspect, DataPlatformInstanceClass) + # Assert DataPlatformInstanceClass + data_platform_instance_mcp = find_mcp_by_aspect(DataPlatformInstanceClass) + assert data_platform_instance_mcp is not None - mcp = cast(MetadataChangeProposalClass, next(iter(gen)).metadata) - assert isinstance(mcp.aspect, SubTypesClass) - assert mcp.aspect.typeNames[1] == DatasetSubTypes.TABLE + # Assert SubTypesClass + sub_types_mcp = find_mcp_by_aspect(SubTypesClass) + assert sub_types_mcp.metadata.aspect.typeNames[1] == DatasetSubTypes.TABLE + + # Ensure all MCPs were checked + # TODO: Test for PlatformResource MCPs as well + assert len(mcps) >= 7 @patch.object(BigQueryV2Config, "get_bigquery_client") From 58f5b4a0cb6bff1f1f0f80d09d980d5a0f438d5f Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 16 Oct 2024 04:53:01 -0700 Subject: [PATCH 23/40] fix(ingest): add preset deps (#11637) --- metadata-ingestion/setup.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index f14c080df644a1..bfec2c00cb8647 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -321,6 +321,13 @@ "Authlib", } +superset_common = { + "requests", + "sqlalchemy", + "great_expectations", + "greenlet", +} + # Note: for all of these, framework_common will be added. plugins: Dict[str, Set[str]] = { # Sink plugins. @@ -462,12 +469,8 @@ "sqlalchemy": sql_common, "sql-queries": usage_common | sqlglot_lib, "slack": slack, - "superset": { - "requests", - "sqlalchemy", - "great_expectations", - "greenlet", - }, + "superset": superset_common, + "preset": superset_common, # FIXME: I don't think tableau uses sqllineage anymore so we should be able # to remove that dependency. "tableau": {"tableauserverclient>=0.24.0"} | sqllineage_lib | sqlglot_lib, From 06698308df21f34a752f7bd18ac8b3d3f8b104e6 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 16 Oct 2024 10:22:59 -0500 Subject: [PATCH 24/40] feat(docker-profiles): allow version override for quickstartDebug (#11643) --- docker/build.gradle | 3 +++ docker/profiles/docker-compose.frontend.yml | 2 +- docker/profiles/docker-compose.gms.yml | 8 ++++---- docker/profiles/docker-compose.prerequisites.yml | 10 +++++----- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docker/build.gradle b/docker/build.gradle index 47f52079e67e09..cdf2d1271d6e85 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -128,6 +128,9 @@ dockerCompose { isRequiredBy(tasks.named('quickstartDebug')) composeAdditionalArgs = ['--profile', 'debug'] + if (System.getenv().containsKey("DATAHUB_VERSION")) { + environment.put 'DATAHUB_VERSION', System.getenv("DATAHUB_VERSION") + } environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally useComposeFiles = ['profiles/docker-compose.yml'] diff --git a/docker/profiles/docker-compose.frontend.yml b/docker/profiles/docker-compose.frontend.yml index b5b2d50143927f..c6b15a7016670d 100644 --- a/docker/profiles/docker-compose.frontend.yml +++ b/docker/profiles/docker-compose.frontend.yml @@ -16,7 +16,7 @@ x-datahub-frontend-service: &datahub-frontend-service x-datahub-frontend-service-dev: &datahub-frontend-service-dev <<: *datahub-frontend-service - image: ${DATAHUB_FRONTEND_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-frontend-react}:debug + image: ${DATAHUB_FRONTEND_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-frontend-react}:${DATAHUB_VERSION:-debug} ports: - ${DATAHUB_MAPPED_FRONTEND_DEBUG_PORT:-5002}:5002 - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 diff --git a/docker/profiles/docker-compose.gms.yml b/docker/profiles/docker-compose.gms.yml index 6e3e5780506ac0..2683734c2d5e56 100644 --- a/docker/profiles/docker-compose.gms.yml +++ b/docker/profiles/docker-compose.gms.yml @@ -73,7 +73,7 @@ x-datahub-system-update-service: &datahub-system-update-service x-datahub-system-update-service-dev: &datahub-system-update-service-dev <<: *datahub-system-update-service - image: ${DATAHUB_UPGRADE_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-upgrade}:debug + image: ${DATAHUB_UPGRADE_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-upgrade}:${DATAHUB_VERSION:-debug} ports: - ${DATAHUB_MAPPED_UPGRADE_DEBUG_PORT:-5003}:5003 environment: &datahub-system-update-dev-env @@ -115,7 +115,7 @@ x-datahub-gms-service: &datahub-gms-service x-datahub-gms-service-dev: &datahub-gms-service-dev <<: *datahub-gms-service - image: ${DATAHUB_GMS_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-gms}:debug + image: ${DATAHUB_GMS_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-gms}:${DATAHUB_VERSION:-debug} ports: - ${DATAHUB_MAPPED_GMS_DEBUG_PORT:-5001}:5001 - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 @@ -159,7 +159,7 @@ x-datahub-mae-consumer-service: &datahub-mae-consumer-service x-datahub-mae-consumer-service-dev: &datahub-mae-consumer-service-dev <<: *datahub-mae-consumer-service - image: ${DATAHUB_MAE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mae-consumer}:debug + image: ${DATAHUB_MAE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mae-consumer}:${DATAHUB_VERSION:-debug} environment: <<: [*datahub-dev-telemetry-env, *datahub-mae-consumer-env] volumes: @@ -185,7 +185,7 @@ x-datahub-mce-consumer-service: &datahub-mce-consumer-service x-datahub-mce-consumer-service-dev: &datahub-mce-consumer-service-dev <<: *datahub-mce-consumer-service - image: ${DATAHUB_MCE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mce-consumer}:debug + image: ${DATAHUB_MCE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mce-consumer}:${DATAHUB_VERSION:-debug} environment: <<: [*datahub-dev-telemetry-env, *datahub-mce-consumer-env] volumes: diff --git a/docker/profiles/docker-compose.prerequisites.yml b/docker/profiles/docker-compose.prerequisites.yml index eed23a749628fe..c9cb444d57c98f 100644 --- a/docker/profiles/docker-compose.prerequisites.yml +++ b/docker/profiles/docker-compose.prerequisites.yml @@ -135,7 +135,7 @@ services: mysql-setup-dev: <<: *mysql-setup profiles: *mysql-profiles-dev - image: ${DATAHUB_MYSQL_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mysql-setup}:debug + image: ${DATAHUB_MYSQL_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mysql-setup}:${DATAHUB_VERSION:-debug} postgres: profiles: *postgres-profiles hostname: postgres @@ -166,7 +166,7 @@ services: postgres-setup-dev: <<: *postgres-setup profiles: *postgres-profiles-dev - image: ${DATAHUB_POSTGRES_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-postgres-setup}:debug + image: ${DATAHUB_POSTGRES_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-postgres-setup}:${DATAHUB_VERSION:-debug} cassandra: profiles: *cassandra-profiles hostname: cassandra @@ -272,7 +272,7 @@ services: environment: <<: *kafka-setup-env DATAHUB_PRECREATE_TOPICS: ${DATAHUB_PRECREATE_TOPICS:-true} - image: ${DATAHUB_KAFKA_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-kafka-setup}:debug + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-kafka-setup}:${DATAHUB_VERSION:-debug} elasticsearch: profiles: *elasticsearch-profiles hostname: search @@ -296,7 +296,7 @@ services: volumes: - esdata:/usr/share/elasticsearch/data elasticsearch-setup-dev: &elasticsearch-setup-dev - image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-elasticsearch-setup}:debug + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-debug} profiles: *elasticsearch-profiles hostname: elasticsearch-setup env_file: elasticsearch-setup/env/docker.env @@ -347,7 +347,7 @@ services: <<: *opensearch-setup profiles: *opensearch-profiles-dev hostname: opensearch-setup-dev - image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-elasticsearch-setup}:debug + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-debug} environment: <<: *search-datastore-environment USE_AWS_ELASTICSEARCH: ${USE_AWS_ELASTICSEARCH:-true} From ce909c8b8b2509e696f71939d9354dc58d5ffd0d Mon Sep 17 00:00:00 2001 From: sid-acryl <155424659+sid-acryl@users.noreply.github.com> Date: Wed, 16 Oct 2024 20:58:05 +0530 Subject: [PATCH 25/40] feat(ingest/powerbi): link cross workspace dataset into assets (#11560) --- docs/how/updating-datahub.md | 15 + .../ingestion/source/powerbi/config.py | 11 +- .../powerbi/rest_api_wrapper/data_classes.py | 11 +- .../powerbi/rest_api_wrapper/data_resolver.py | 33 +- .../powerbi/rest_api_wrapper/powerbi_api.py | 69 +- .../integration/powerbi/golden_test_cll.json | 1114 ++++++++--------- .../golden_test_cross_workspace_dataset.json | 668 ++++++++++ .../golden_test_disabled_ownership.json | 720 +++++------ .../powerbi/golden_test_endorsement.json | 846 ++++++------- .../powerbi/golden_test_ingest.json | 722 +++++------ .../golden_test_ingest_patch_disabled.json | 715 +++++------ .../powerbi/golden_test_lineage.json | 1014 +++++++-------- .../golden_test_lower_case_urn_ingest.json | 766 ++++++------ ..._config_and_modified_since_admin_only.json | 624 ++++----- .../golden_test_platform_instance_ingest.json | 702 +++++------ .../powerbi/golden_test_profiling.json | 68 +- .../golden_test_scan_all_workspaces.json | 818 ++++++------ ...lden_test_server_to_platform_instance.json | 968 +++++++------- .../cross_workspace_mock_response.json | 220 ++++ .../mock_data/default_mock_response.json | 558 +++++++++ .../mock_data/workspace_type_filter.json | 76 ++ .../tests/integration/powerbi/test_powerbi.py | 797 ++---------- 22 files changed, 6305 insertions(+), 5230 deletions(-) create mode 100644 metadata-ingestion/tests/integration/powerbi/golden_test_cross_workspace_dataset.json create mode 100644 metadata-ingestion/tests/integration/powerbi/mock_data/cross_workspace_mock_response.json create mode 100644 metadata-ingestion/tests/integration/powerbi/mock_data/default_mock_response.json create mode 100644 metadata-ingestion/tests/integration/powerbi/mock_data/workspace_type_filter.json diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index dbcc7da8467035..5b4769ed30e3ed 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -18,6 +18,21 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ## Next +- #11560 - The PowerBI ingestion source configuration option include_workspace_name_in_dataset_urn determines whether the workspace name is included in the PowerBI dataset's URN.
PowerBI allows to have identical name of semantic model and their tables across the workspace, It will overwrite the semantic model in-case of multi-workspace ingestion.
+ Entity urn with `include_workspace_name_in_dataset_urn: false` + ``` + urn:li:dataset:(urn:li:dataPlatform:powerbi,[.].,) + ``` + + Entity urn with `include_workspace_name_in_dataset_urn: true` + ``` + urn:li:dataset:(urn:li:dataPlatform:powerbi,[.]...,) + ``` + + The config `include_workspace_name_in_dataset_urn` is default to `false` for backward compatiblity, However, we recommend enabling this flag after performing the necessary cleanup. + If stateful ingestion is enabled, running ingestion with the latest CLI version will handle the cleanup automatically. Otherwise, we recommend soft deleting all powerbi data via the DataHub CLI: + `datahub delete --platform powerbi --soft` and then re-ingest with the latest CLI version, ensuring the `include_workspace_name_in_dataset_urn` configuration is set to true. + ### Breaking Changes - #11486 - Deprecated Criterion filters using `value`. Use `values` instead. This also deprecates the ability to use comma delimited string to represent multiple values using `value`. diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py index 522639a160781e..0716a658b61c6f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py @@ -331,8 +331,8 @@ class PowerBiDashboardSourceConfig( ) workspace_id_as_urn_part: bool = pydantic.Field( default=False, - description="Highly recommend changing this to True, as you can have the same workspace name" - "To maintain backward compatability, this is set to False which uses workspace name", + description="It is recommended to set this to True only if you have legacy workspaces based on Office 365 groups, as those workspaces can have identical names." + "To maintain backward compatibility, this is set to False which uses workspace name", ) # Enable/Disable extracting ownership information of Dashboard extract_ownership: bool = pydantic.Field( @@ -466,6 +466,13 @@ class PowerBiDashboardSourceConfig( " Note: This field works in conjunction with 'workspace_id_pattern'. Both must be matched for a workspace to be processed.", ) + include_workspace_name_in_dataset_urn: bool = pydantic.Field( + default=False, + description="It is recommended to set this to true, as it helps prevent the overwriting of datasets." + "Read section #11560 at https://datahubproject.io/docs/how/updating-datahub/ before enabling this option." + "To maintain backward compatibility, this is set to False.", + ) + @root_validator(skip_on_failure=True) def validate_extract_column_level_lineage(cls, values: Dict) -> Dict: flags = [ diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py index fb0959ac604c4c..d54b4a42b742e0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py @@ -142,6 +142,7 @@ class PowerBIDataset: description: str webUrl: Optional[str] workspace_id: str + workspace_name: str parameters: Dict[str, str] # Table in datasets @@ -225,7 +226,10 @@ class Report: webUrl: Optional[str] embedUrl: str description: str - dataset: Optional["PowerBIDataset"] + dataset_id: Optional[str] # dataset_id is coming from REST API response + dataset: Optional[ + "PowerBIDataset" + ] # This the dataclass later initialise by powerbi_api.py pages: List["Page"] users: List["User"] tags: List[str] @@ -283,7 +287,7 @@ def __hash__(self): return hash(self.__members()) -def new_powerbi_dataset(workspace_id: str, raw_instance: dict) -> PowerBIDataset: +def new_powerbi_dataset(workspace: Workspace, raw_instance: dict) -> PowerBIDataset: return PowerBIDataset( id=raw_instance["id"], name=raw_instance.get("name"), @@ -293,7 +297,8 @@ def new_powerbi_dataset(workspace_id: str, raw_instance: dict) -> PowerBIDataset if raw_instance.get("webUrl") is not None else None ), - workspace_id=workspace_id, + workspace_id=workspace.id, + workspace_name=workspace.name, parameters={}, tables=[], tags=[], diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py index d89b9662d12edb..8849e19ea86228 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py @@ -129,7 +129,7 @@ def profile_dataset( @abstractmethod def get_dataset( - self, workspace_id: str, dataset_id: str + self, workspace: Workspace, dataset_id: str ) -> Optional[PowerBIDataset]: pass @@ -279,9 +279,10 @@ def fetch_reports(): pages=self._get_pages_by_report( workspace=workspace, report_id=raw_instance[Constant.ID] ), + dataset_id=raw_instance.get(Constant.DATASET_ID), users=[], # It will be fetched using Admin Fetcher based on condition tags=[], # It will be fetched using Admin Fetcher based on condition - dataset=workspace.datasets.get(raw_instance.get(Constant.DATASET_ID)), + dataset=None, # It will come from dataset_registry defined in powerbi_api.py ) for raw_instance in fetch_reports() if Constant.APP_ID @@ -317,11 +318,6 @@ def new_dataset_or_report(tile_instance: Any) -> dict: Find out which is the data source for tile. It is either REPORT or DATASET """ report_fields = { - Constant.DATASET: ( - workspace.datasets.get(tile_instance.get(Constant.DATASET_ID)) - if tile_instance.get("datasetId") is not None - else None - ), Constant.REPORT: ( self.get_report( workspace=workspace, @@ -372,6 +368,7 @@ def new_dataset_or_report(tile_instance: Any) -> dict: title=instance.get(Constant.TITLE), embedUrl=instance.get(Constant.EMBED_URL), dataset_id=instance.get(Constant.DATASET_ID), + dataset=None, **new_dataset_or_report(instance), ) for instance in tile_dict @@ -430,14 +427,14 @@ class RegularAPIResolver(DataResolverBase): } def get_dataset( - self, workspace_id: str, dataset_id: str + self, workspace: Workspace, dataset_id: str ) -> Optional[PowerBIDataset]: """ Fetch the dataset from PowerBi for the given dataset identifier """ - if workspace_id is None or dataset_id is None: + if workspace.id is None or dataset_id is None: logger.debug("Input values are None") - logger.debug(f"{Constant.WorkspaceId}={workspace_id}") + logger.debug(f"{Constant.WorkspaceId}={workspace.id}") logger.debug(f"{Constant.DatasetId}={dataset_id}") return None @@ -447,7 +444,7 @@ def get_dataset( # Replace place holders dataset_get_endpoint = dataset_get_endpoint.format( POWERBI_BASE_URL=DataResolverBase.BASE_URL, - WORKSPACE_ID=workspace_id, + WORKSPACE_ID=workspace.id, DATASET_ID=dataset_id, ) # Hit PowerBi @@ -456,13 +453,13 @@ def get_dataset( dataset_get_endpoint, headers=self.get_authorization_header(), ) - # Check if we got response from PowerBi + # Check if we got a response from PowerBi response.raise_for_status() response_dict = response.json() logger.debug(f"datasets = {response_dict}") - # PowerBi Always return the webURL, in-case if it is None then setting complete webURL to None instead of + # PowerBi Always return the webURL, in-case if it is None, then setting complete webURL to None instead of # None/details - return new_powerbi_dataset(workspace_id, response_dict) + return new_powerbi_dataset(workspace, response_dict) def get_dataset_parameters( self, workspace_id: str, dataset_id: str @@ -910,11 +907,11 @@ def get_tiles_endpoint(self, workspace: Workspace, dashboard_id: str) -> str: ) def get_dataset( - self, workspace_id: str, dataset_id: str + self, workspace: Workspace, dataset_id: str ) -> Optional[PowerBIDataset]: datasets_endpoint = self.API_ENDPOINTS[Constant.DATASET_LIST].format( POWERBI_ADMIN_BASE_URL=DataResolverBase.ADMIN_BASE_URL, - WORKSPACE_ID=workspace_id, + WORKSPACE_ID=workspace.id, ) # Hit PowerBi logger.debug(f"Request to datasets URL={datasets_endpoint}") @@ -930,13 +927,13 @@ def get_dataset( if len(response_dict.get(Constant.VALUE, [])) == 0: logger.warning( "Dataset not found. workspace_id = %s, dataset_id = %s", - workspace_id, + workspace.id, dataset_id, ) return None raw_instance: dict = response_dict[Constant.VALUE][0] - return new_powerbi_dataset(workspace_id, raw_instance) + return new_powerbi_dataset(workspace, raw_instance) def _get_pages_by_report(self, workspace: Workspace, report_id: str) -> List[Page]: return [] # Report pages are not available in Admin API diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py index 25e97b158d48b5..37793bc32980b4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py @@ -31,6 +31,28 @@ logger = logging.getLogger(__name__) +def form_full_table_name( + config: PowerBiDashboardSourceConfig, + workspace: Workspace, + dataset_name: str, + table_name: str, +) -> str: + + full_table_name: str = "{}.{}".format( + dataset_name.replace(" ", "_"), table_name.replace(" ", "_") + ) + + if config.include_workspace_name_in_dataset_urn: + workspace_identifier: str = ( + workspace.id + if config.workspace_id_as_urn_part + else workspace.name.replace(" ", "_").lower() + ) + full_table_name = f"{workspace_identifier}.{full_table_name}" + + return full_table_name + + class PowerBiAPI: def __init__( self, @@ -52,6 +74,14 @@ def __init__( tenant_id=self.__config.tenant_id, ) + self.reporter: PowerBiDashboardSourceReport = reporter + + # A report or tile in one workspace can be built using a dataset from another workspace. + # We need to store the dataset ID (which is a UUID) mapped to its dataset instance. + # This mapping will allow us to retrieve the appropriate dataset for + # reports and tiles across different workspaces. + self.dataset_registry: Dict[str, PowerBIDataset] = {} + def log_http_error(self, message: str) -> Any: logger.warning(message) _, e, _ = sys.exc_info() @@ -158,6 +188,16 @@ def get_reports(self, workspace: Workspace) -> List[Report]: reports: List[Report] = [] try: reports = self._get_resolver().get_reports(workspace) + # Fill Report dataset + for report in reports: + if report.dataset_id: + report.dataset = self.dataset_registry.get(report.dataset_id) + if report.dataset is None: + self.reporter.info( + title="Missing Lineage For Report", + message="A cross-workspace reference that failed to be resolved. Please ensure that no global workspace is being filtered out due to the workspace_id_pattern.", + context=f"report-name: {report.name} and dataset-id: {report.dataset_id}", + ) except: self.log_http_error( message=f"Unable to fetch reports for workspace {workspace.name}" @@ -312,14 +352,14 @@ def _get_workspace_datasets(self, workspace: Workspace) -> dict: for dataset_dict in datasets: dataset_instance: PowerBIDataset = self._get_resolver().get_dataset( - workspace_id=scan_result[Constant.ID], + workspace=workspace, dataset_id=dataset_dict[Constant.ID], ) # fetch + set dataset parameters try: dataset_parameters = self._get_resolver().get_dataset_parameters( - workspace_id=scan_result[Constant.ID], + workspace_id=workspace.id, dataset_id=dataset_dict[Constant.ID], ) dataset_instance.parameters = dataset_parameters @@ -350,9 +390,11 @@ def _get_workspace_datasets(self, workspace: Workspace) -> dict: ) table = Table( name=table[Constant.NAME], - full_name="{}.{}".format( - dataset_name.replace(" ", "_"), - table[Constant.NAME].replace(" ", "_"), + full_name=form_full_table_name( + config=self.__config, + workspace=workspace, + dataset_name=dataset_name, + table_name=table[Constant.NAME], ), expression=expression, columns=[ @@ -382,7 +424,8 @@ def _get_workspace_datasets(self, workspace: Workspace) -> dict: return dataset_map def _fill_metadata_from_scan_result( - self, workspaces: List[Workspace] + self, + workspaces: List[Workspace], ) -> List[Workspace]: workspace_ids = [workspace.id for workspace in workspaces] scan_result = self._get_scan_result(workspace_ids) @@ -423,7 +466,8 @@ def _fill_metadata_from_scan_result( ) cur_workspace.scan_result = workspace_metadata cur_workspace.datasets = self._get_workspace_datasets(cur_workspace) - + # collect all datasets in the registry + self.dataset_registry.update(cur_workspace.datasets) # Fetch endorsement tag if it is enabled from configuration if self.__config.extract_endorsements_to_tags: cur_workspace.dashboard_endorsements = self._get_dashboard_endorsements( @@ -468,6 +512,16 @@ def fill_dashboards() -> None: dashboard.tiles = self._get_resolver().get_tiles( workspace, dashboard=dashboard ) + # set the dataset for tiles + for tile in dashboard.tiles: + if tile.dataset_id: + tile.dataset = self.dataset_registry.get(tile.dataset_id) + if tile.dataset is None: + self.reporter.info( + title="Missing Lineage For Tile", + message="A cross-workspace reference that failed to be resolved. Please ensure that no global workspace is being filtered out due to the workspace_id_pattern.", + context=f"workspace-name: {workspace.name}, tile-name: {tile.title}, dataset-id: {tile.dataset_id}", + ) def fill_reports() -> None: if self.__config.extract_reports is False: @@ -497,6 +551,7 @@ def fill_dashboard_tags() -> None: def fill_workspaces( self, workspaces: List[Workspace], reporter: PowerBiDashboardSourceReport ) -> Iterable[Workspace]: + workspaces = self._fill_metadata_from_scan_result(workspaces=workspaces) # First try to fill the admin detail as some regular metadata contains lineage to admin metadata for workspace in workspaces: diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index 66ee60c2eebb3f..04dfd3bab3c0bc 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -17,6 +17,38 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "User1@foo.com" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", @@ -40,13 +72,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "corpUserKey", "aspect": { "json": { - "removed": false + "username": "User2@foo.com" } }, "systemMetadata": { @@ -73,6 +105,77 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" + }, + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", @@ -114,13 +217,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -133,12 +238,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -148,22 +251,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "chartKey", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,PBI_TEST.TEST.TESTTABLE,PROD)", - "type": "TRANSFORMED" - } - ] + "dashboardTool": "powerbi", + "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" } }, "systemMetadata": { @@ -212,6 +307,24 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", @@ -228,6 +341,24 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", @@ -246,6 +377,51 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,PBI_TEST.TEST.TESTTABLE,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", @@ -298,70 +474,36 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", - "viewLanguage": "m_query" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartInfo", "aspect": { "json": { "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "big-query-with-parameter", - "description": "Library dataset description", - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + } ] } }, @@ -373,13 +515,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", "viewLanguage": "m_query" } }, @@ -391,7 +533,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -400,7 +542,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query-with-join", + "name": "big-query-with-parameter", "description": "Library dataset description", "tags": [] } @@ -412,33 +554,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-test-project.universal.D_WH_DATE,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -454,14 +571,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -471,15 +586,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "chartKey", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", - "viewLanguage": "m_query" + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -490,52 +604,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_forecast,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD),name)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV),name)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD),name)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV),name)" - ], - "confidenceScore": 1.0 - } + "typeNames": [ + "Table" ] } }, @@ -546,19 +621,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "job-history", - "description": "Library dataset description", - "tags": [] + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -568,13 +639,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePaths", "aspect": { "json": { - "removed": false + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -585,13 +658,20 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "upstreamLineage", "aspect": { "json": { - "typeNames": [ - "Table" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-test-project.universal.D_WH_DATE,PROD)", + "type": "TRANSFORMED" + } ] } }, @@ -602,20 +682,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "browsePathsV2", "aspect": { "json": { - "upstreams": [ + "path": [ { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,salesdb.HR.EMPLOYEES,PROD)", - "type": "TRANSFORMED" + "id": "demo-workspace" } ] } @@ -628,13 +703,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", "viewLanguage": "m_query" } }, @@ -646,7 +721,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -655,7 +730,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "postgres_test_table", + "name": "snowflake native-query-with-join", "description": "Library dataset description", "tags": [] } @@ -667,13 +742,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePaths", "aspect": { "json": { - "removed": false + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -684,14 +761,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -701,23 +776,67 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,mics.public.order_date,PROD)", - "type": "TRANSFORMED" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } - ] - } + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -745,18 +864,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "viewProperties", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "hr pbi test description", - "tags": [] + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -767,12 +882,18 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetProperties", "aspect": { "json": { - "removed": false + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "ms_sql_native_table", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -785,12 +906,16 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "datasetProperties", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -803,12 +928,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -819,18 +942,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "ms_sql_native_table", - "description": "hr pbi test description", - "tags": [] + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -841,20 +960,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,library.dbo.book_issue,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Table" ] } }, @@ -866,7 +978,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -881,15 +993,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -924,13 +1034,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "subTypes", "aspect": { "json": { - "username": "User1@foo.com" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -940,13 +1052,14 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "dashboardKey", "aspect": { "json": { - "username": "User2@foo.com" + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" } }, "systemMetadata": { @@ -956,50 +1069,52 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "upstreamLineage", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" - }, - "title": "test_tile", - "description": "test_tile", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" - }, + "upstreams": [ { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD)", + "type": "TRANSFORMED" }, { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" - }, + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_forecast,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV),name)" + ], + "confidenceScore": 1.0 }, { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV),name)" + ], + "confidenceScore": 1.0 } ] } @@ -1011,13 +1126,22 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "upstreamLineage", "aspect": { "json": { - "removed": false + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,library.dbo.book_issue,PROD)", + "type": "TRANSFORMED" + } + ] } }, "systemMetadata": { @@ -1027,50 +1151,27 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "ownership", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "changeType": "UPSERT", - "aspectName": "chartKey", - "aspect": { - "json": { - "dashboardTool": "powerbi", - "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "changeType": "UPSERT", - "aspectName": "browsePaths", - "aspect": { - "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "owners": [ + { + "owner": "urn:li:corpuser:users.User1@foo.com", + "type": "NONE" + }, + { + "owner": "urn:li:corpuser:users.User2@foo.com", + "type": "NONE" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } }, "systemMetadata": { @@ -1080,8 +1181,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1100,37 +1201,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" - }, - "title": "yearly_sales", - "description": "yearly_sales", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" - } - ] + "removed": false } }, "systemMetadata": { @@ -1140,8 +1217,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1156,14 +1233,22 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "upstreamLineage", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,salesdb.HR.EMPLOYEES,PROD)", + "type": "TRANSFORMED" + } + ] } }, "systemMetadata": { @@ -1173,15 +1258,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "viewProperties", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" - ] + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1191,15 +1276,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "viewProperties", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1209,17 +1294,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "datasetProperties", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } - ] + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1229,14 +1316,14 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "subTypes", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" + "typeNames": [ + "Table" ] } }, @@ -1247,77 +1334,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "PATCH", - "aspectName": "dashboardInfo", - "aspect": { - "json": [ - { - "op": "add", - "path": "/customProperties/chartCount", - "value": "2" - }, - { - "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" - }, - { - "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" - }, - { - "op": "add", - "path": "/description", - "value": "Description of test dashboard" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - }, - { - "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" - }, - { - "op": "add", - "path": "/lastModified", - "value": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1332,44 +1350,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", - "aspectName": "dashboardKey", - "aspect": { - "json": { - "dashboardTool": "powerbi", - "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "datasetProperties", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:users.User1@foo.com", - "type": "NONE" - }, - { - "owner": "urn:li:corpuser:users.User2@foo.com", - "type": "NONE" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1379,16 +1372,14 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "subTypes", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } + "typeNames": [ + "Table" ] } }, @@ -1400,14 +1391,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "dummy", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -1417,13 +1406,19 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetProperties", "aspect": { "json": { - "removed": false + "customProperties": { + "datasetId": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff/details", + "name": "employee_ctc", + "description": "Employee Management", + "tags": [] } }, "systemMetadata": { @@ -1434,12 +1429,21 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "upstreamLineage", "aspect": { "json": { - "removed": false + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,mics.public.order_date,PROD)", + "type": "TRANSFORMED" + } + ] } }, "systemMetadata": { @@ -1470,16 +1474,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "viewProperties", "aspect": { "json": { - "customProperties": { - "datasetId": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff/details", - "name": "employee_ctc", - "description": "Employee Management", - "tags": [] + "materialized": false, + "viewLogic": "dummy", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1489,8 +1489,8 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cross_workspace_dataset.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cross_workspace_dataset.json new file mode 100644 index 00000000000000..06286fcd5937a0 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cross_workspace_dataset.json @@ -0,0 +1,668 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,sales.sales_semantic_model.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "dummy", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "317456E5-1FC7-4BDC-9C84-1185825E293D", + "datasetWebUrl": "http://localhost/groups/A8D655A6-F521-477E-8C22-255018583BF4/datasets/317456E5-1FC7-4BDC-9C84-1185825E293D/details" + }, + "title": "Sale Order Tile", + "description": "Sale Order Tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,sales.sales_semantic_model.public_issue_history,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,sales.sales_semantic_model.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,sales.sales_semantic_model.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "317456E5-1FC7-4BDC-9C84-1185825E293D" + }, + "externalUrl": "http://localhost/groups/A8D655A6-F521-477E-8C22-255018583BF4/datasets/317456E5-1FC7-4BDC-9C84-1185825E293D/details", + "name": "public issue_history", + "description": "sales semantic model", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,sales.sales_semantic_model.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Tile" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)", + "changeType": "UPSERT", + "aspectName": "chartKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "chartId": "charts.885D1762-1655-46BA-AFE3-74C6EC403A9E" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,global_workspace.base_records.core_sales_set,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "dummy", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/Sales" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,global_workspace.base_records.core_sales_set,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,global_workspace.base_records.core_sales_set,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "FE362B98-956E-4394-BA37-6367EE6435E9" + }, + "externalUrl": "http://localhost/groups/C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492/datasets/FE362B98-956E-4394-BA37-6367EE6435E9/details", + "name": "core_sales_set", + "description": "base_records", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,global_workspace.base_records.core_sales_set,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sales" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "FE362B98-956E-4394-BA37-6367EE6435E9", + "datasetWebUrl": "http://localhost/groups/C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492/datasets/FE362B98-956E-4394-BA37-6367EE6435E9/details" + }, + "title": "Yearly Sales", + "description": "Yearly Sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,global_workspace.base_records.core_sales_set,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Tile" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)", + "changeType": "UPSERT", + "aspectName": "chartKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "chartId": "charts.945C2C2A-4588-45DE-8385-F24F5E39A57C" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/Sales" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sales" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "0F0ADA0E-E38A-44F6-B667-90E93A96F5A1" + }, + "title": "Not Present In Current Ingestion", + "description": "Not Present In Current Ingestion", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A1C7204F-4D04-4E5E-B886-B30EA2C64CB3)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/Sales" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Tile" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A1C7204F-4D04-4E5E-B886-B30EA2C64CB3)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A1C7204F-4D04-4E5E-B886-B30EA2C64CB3)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "3" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "Sales" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "A8D655A6-F521-477E-8C22-255018583BF4" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)", + "value": "urn:li:chart:(powerbi,charts.885D1762-1655-46BA-AFE3-74C6EC403A9E)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)", + "value": "urn:li:chart:(powerbi,charts.945C2C2A-4588-45DE-8385-F24F5E39A57C)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)", + "value": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)", + "changeType": "UPSERT", + "aspectName": "chartKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "chartId": "charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/Sales" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B847EBDA-BC48-4F92-8E16-6B46D900E7BB)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sales" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A1C7204F-4D04-4E5E-B886-B30EA2C64CB3)", + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/A1C7204F-4D04-4E5E-B886-B30EA2C64CB3" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.A1C7204F-4D04-4E5E-B886-B30EA2C64CB3)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sales" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json index 665f5d5a3bb416..099f75a190ca26 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -17,6 +17,40 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", @@ -40,8 +74,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -74,15 +108,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", - "viewLanguage": "m_query" + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -92,19 +126,17 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "browsePathsV2", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "SNOWFLAKE_TESTTABLE", - "description": "Library dataset description", - "tags": [] + "path": [ + { + "id": "demo-workspace" + } + ] } }, "systemMetadata": { @@ -117,10 +149,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -130,15 +164,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "chartKey", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -149,14 +182,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -167,7 +198,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -176,7 +207,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query", + "name": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description", "tags": [] } @@ -191,10 +222,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -204,14 +237,36 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "chartInfo", "aspect": { "json": { - "typeNames": [ - "Table" + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" + }, + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + } ] } }, @@ -223,14 +278,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -241,7 +294,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -250,7 +303,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "big-query-with-parameter", + "name": "snowflake native-query", "description": "Library dataset description", "tags": [] } @@ -263,12 +316,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -279,7 +334,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -296,15 +351,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "browsePaths", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", - "viewLanguage": "m_query" + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -314,19 +369,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query-with-join", - "description": "Library dataset description", - "tags": [] + "removed": false } }, "systemMetadata": { @@ -337,12 +386,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -352,14 +403,51 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "chartInfo", "aspect": { "json": { - "typeNames": [ - "Table" + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" + }, + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + } ] } }, @@ -371,14 +459,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -389,7 +475,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -398,7 +484,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "job-history", + "name": "big-query-with-parameter", "description": "Library dataset description", "tags": [] } @@ -410,13 +496,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -427,7 +515,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -444,15 +532,17 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "browsePathsV2", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", - "viewLanguage": "m_query" + "path": [ + { + "id": "demo-workspace" + } + ] } }, "systemMetadata": { @@ -462,19 +552,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartKey", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "postgres_test_table", - "description": "Library dataset description", - "tags": [] + "dashboardTool": "powerbi", + "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" } }, "systemMetadata": { @@ -485,12 +570,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -500,14 +587,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePaths", "aspect": { "json": { - "typeNames": [ - "Table" + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -519,14 +606,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -537,17 +622,17 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "hr pbi test description", + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query-with-join", + "description": "Library dataset description", "tags": [] } }, @@ -559,12 +644,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -577,12 +664,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "viewProperties", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -595,12 +682,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -632,14 +717,67 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", - "changeType": "UPSERT", - "aspectName": "status", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", "aspect": { - "json": { - "removed": false - } + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -666,52 +804,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" - }, - "title": "test_tile", - "description": "test_tile", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" - } - ] + "removed": false } }, "systemMetadata": { @@ -721,13 +820,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetProperties", "aspect": { "json": { - "removed": false + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -737,14 +842,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "PowerBI Tile" + "Table" ] } }, @@ -755,14 +860,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "subTypes", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -772,15 +878,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "status", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "removed": false } }, "systemMetadata": { @@ -790,17 +894,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "dashboardKey", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } - ] + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" } }, "systemMetadata": { @@ -810,35 +911,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "browsePathsV2", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" - }, - "title": "yearly_sales", - "description": "yearly_sales", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" - }, + "path": [ { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + "id": "demo-workspace" } ] } @@ -850,30 +931,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "datasetProperties", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -883,14 +953,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "PowerBI Tile" + "Table" ] } }, @@ -901,15 +971,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "viewProperties", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -919,17 +989,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "viewProperties", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } - ] + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -939,15 +1007,13 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "status", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "removed": false } }, "systemMetadata": { @@ -957,77 +1023,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "PATCH", - "aspectName": "dashboardInfo", - "aspect": { - "json": [ - { - "op": "add", - "path": "/customProperties/chartCount", - "value": "2" - }, - { - "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" - }, - { - "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" - }, - { - "op": "add", - "path": "/description", - "value": "Description of test dashboard" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - }, - { - "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" - }, - { - "op": "add", - "path": "/lastModified", - "value": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1042,14 +1039,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "dashboardKey", + "aspectName": "datasetProperties", "aspect": { "json": { - "dashboardTool": "powerbi", - "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1059,16 +1061,14 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "subTypes", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } + "typeNames": [ + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json index 26476e61a0bd7c..3c02b146a900c4 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json @@ -40,13 +40,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "tag", + "entityUrn": "urn:li:tag:Promoted", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "tagKey", "aspect": { "json": { - "removed": false + "name": "Promoted" } }, "systemMetadata": { @@ -59,12 +59,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -74,17 +72,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "corpUserKey", "aspect": { "json": { - "tags": [ - { - "tag": "urn:li:tag:Promoted" - } - ] + "username": "User2@foo.com" } }, "systemMetadata": { @@ -95,14 +89,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", - "viewLanguage": "m_query" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -112,19 +106,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "SNOWFLAKE_TESTTABLE", - "description": "Library dataset description", - "tags": [] + "removed": false } }, "systemMetadata": { @@ -134,13 +122,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePaths", "aspect": { "json": { - "removed": false + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -151,13 +141,15 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "globalTags", "aspect": { "json": { - "typeNames": [ - "Table" + "tags": [ + { + "tag": "urn:li:tag:Promoted" + } ] } }, @@ -168,17 +160,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "status", "aspect": { "json": { - "tags": [ - { - "tag": "urn:li:tag:Promoted" - } - ] + "removed": false } }, "systemMetadata": { @@ -189,13 +177,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", "viewLanguage": "m_query" } }, @@ -207,7 +195,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -216,7 +204,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query", + "name": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description", "tags": [] } @@ -228,13 +216,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -245,14 +235,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -262,15 +250,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "browsePathsV2", "aspect": { "json": { - "tags": [ + "path": [ { - "tag": "urn:li:tag:Promoted" + "id": "demo-workspace" } ] } @@ -283,13 +271,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", "viewLanguage": "m_query" } }, @@ -301,7 +289,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -310,7 +298,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "big-query-with-parameter", + "name": "snowflake native-query", "description": "Library dataset description", "tags": [] } @@ -323,12 +311,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -339,14 +329,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -356,17 +344,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "chartKey", "aspect": { "json": { - "tags": [ - { - "tag": "urn:li:tag:Promoted" - } - ] + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -377,14 +362,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", - "viewLanguage": "m_query" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -394,19 +379,37 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartInfo", "aspect": { "json": { "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query-with-join", - "description": "Library dataset description", - "tags": [] + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + } + ] } }, "systemMetadata": { @@ -417,12 +420,16 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "globalTags", "aspect": { "json": { - "removed": false + "tags": [ + { + "tag": "urn:li:tag:Promoted" + } + ] } }, "systemMetadata": { @@ -433,13 +440,15 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "globalTags", "aspect": { "json": { - "typeNames": [ - "Table" + "tags": [ + { + "tag": "urn:li:tag:Promoted" + } ] } }, @@ -450,17 +459,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "corpUserKey", "aspect": { "json": { - "tags": [ - { - "tag": "urn:li:tag:Promoted" - } - ] + "username": "User1@foo.com" } }, "systemMetadata": { @@ -471,13 +476,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", "viewLanguage": "m_query" } }, @@ -489,7 +494,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -498,7 +503,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "job-history", + "name": "big-query-with-parameter", "description": "Library dataset description", "tags": [] } @@ -510,8 +515,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -527,14 +532,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -544,16 +547,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "browsePaths", "aspect": { "json": { - "tags": [ - { - "tag": "urn:li:tag:Promoted" - } + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -565,14 +566,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", - "viewLanguage": "m_query" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -582,19 +583,52 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartInfo", "aspect": { "json": { "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "postgres_test_table", - "description": "Library dataset description", - "tags": [] + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + } + ] } }, "systemMetadata": { @@ -604,8 +638,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -621,13 +655,15 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "globalTags", "aspect": { "json": { - "typeNames": [ - "Table" + "tags": [ + { + "tag": "urn:li:tag:Promoted" + } ] } }, @@ -638,16 +674,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "subTypes", "aspect": { "json": { - "tags": [ - { - "tag": "urn:li:tag:Promoted" - } + "typeNames": [ + "PowerBI Tile" ] } }, @@ -659,13 +693,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", "viewLanguage": "m_query" } }, @@ -677,17 +711,17 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "hr pbi test description", + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query-with-join", + "description": "Library dataset description", "tags": [] } }, @@ -698,13 +732,17 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePathsV2", "aspect": { "json": { - "removed": false + "path": [ + { + "id": "demo-workspace" + } + ] } }, "systemMetadata": { @@ -715,14 +753,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -750,19 +786,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartKey", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "ms_sql_native_table", - "description": "hr pbi test description", - "tags": [] + "dashboardTool": "powerbi", + "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" } }, "systemMetadata": { @@ -791,12 +822,16 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "datasetProperties", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "ms_sql_native_table", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -806,13 +841,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "viewProperties", "aspect": { "json": { - "username": "User1@foo.com" + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -822,13 +859,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "subTypes", "aspect": { "json": { - "username": "User2@foo.com" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -838,51 +877,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" - }, - "title": "test_tile", - "description": "test_tile", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" - } + "typeNames": [ + "Table" ] } }, @@ -893,8 +895,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -909,15 +911,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "datasetProperties", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" - ] + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -927,14 +933,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "browsePaths", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -944,14 +951,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "subTypes", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" + "typeNames": [ + "Table" ] } }, @@ -962,18 +969,67 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", "aspect": { - "json": { - "path": [ - { - "id": "demo-workspace" + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } - ] - } + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -982,35 +1038,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "globalTags", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" - }, - "title": "yearly_sales", - "description": "yearly_sales", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" - }, + "tags": [ { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + "tag": "urn:li:tag:Promoted" } ] } @@ -1022,8 +1058,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1038,14 +1074,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "dashboardKey", "aspect": { "json": { "dashboardTool": "powerbi", - "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" } }, "systemMetadata": { @@ -1055,14 +1091,16 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" + "path": [ + { + "id": "demo-workspace" + } ] } }, @@ -1073,15 +1111,27 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "ownership", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "owners": [ + { + "owner": "urn:li:corpuser:users.User1@foo.com", + "type": "NONE" + }, + { + "owner": "urn:li:corpuser:users.User2@foo.com", + "type": "NONE" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } }, "systemMetadata": { @@ -1091,15 +1141,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "globalTags", "aspect": { "json": { - "path": [ + "tags": [ { - "id": "demo-workspace" + "tag": "urn:li:tag:Promoted" } ] } @@ -1111,15 +1161,15 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "viewProperties", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1129,67 +1179,16 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "PATCH", - "aspectName": "dashboardInfo", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", "aspect": { - "json": [ - { - "op": "add", - "path": "/customProperties/chartCount", - "value": "2" - }, - { - "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" - }, - { - "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" - }, - { - "op": "add", - "path": "/description", - "value": "Description of test dashboard" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - }, - { - "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" - }, - { - "op": "add", - "path": "/lastModified", - "value": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] + "json": { + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1198,13 +1197,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetProperties", "aspect": { "json": { - "removed": false + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1214,14 +1219,15 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "dashboardKey", + "aspectName": "subTypes", "aspect": { "json": { - "dashboardTool": "powerbi", - "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -1231,27 +1237,13 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "status", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:users.User1@foo.com", - "type": "NONE" - }, - { - "owner": "urn:li:corpuser:users.User2@foo.com", - "type": "NONE" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "removed": false } }, "systemMetadata": { @@ -1261,17 +1253,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "datasetProperties", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } - ] + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1281,13 +1275,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -1297,8 +1293,8 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1313,13 +1309,17 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:Promoted", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "tagKey", + "aspectName": "globalTags", "aspect": { "json": { - "name": "Promoted" + "tags": [ + { + "tag": "urn:li:tag:Promoted" + } + ] } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index 83f8f881835b7e..8290d996742f8f 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -18,19 +18,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "corpUserKey", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "public issue_history", - "description": "Library dataset description", - "tags": [] + "username": "User2@foo.com" } }, "systemMetadata": { @@ -59,42 +53,6 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", - "viewLanguage": "m_query" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", - "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { @@ -102,7 +60,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "SNOWFLAKE_TESTTABLE", + "name": "public issue_history", "description": "Library dataset description", "tags": [] } @@ -114,8 +72,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -131,7 +89,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -148,15 +106,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "browsePaths", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", - "viewLanguage": "m_query" + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -166,19 +124,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query", - "description": "Library dataset description", - "tags": [] + "removed": false } }, "systemMetadata": { @@ -189,12 +141,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -204,14 +158,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "PowerBI Tile" ] } }, @@ -223,14 +177,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -241,7 +193,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -250,7 +202,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "big-query-with-parameter", + "name": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description", "tags": [] } @@ -263,12 +215,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -278,14 +232,16 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Table" + "path": [ + { + "id": "demo-workspace" + } ] } }, @@ -297,14 +253,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -315,7 +269,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -324,7 +278,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query-with-join", + "name": "snowflake native-query", "description": "Library dataset description", "tags": [] } @@ -337,12 +291,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -353,7 +309,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -370,15 +326,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "chartKey", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", - "viewLanguage": "m_query" + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -388,19 +343,37 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartInfo", "aspect": { "json": { "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "job-history", - "description": "Library dataset description", - "tags": [] + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + } + ] } }, "systemMetadata": { @@ -411,12 +384,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -426,15 +401,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "corpUserKey", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "username": "User1@foo.com" } }, "systemMetadata": { @@ -445,14 +418,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -463,7 +434,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -472,7 +443,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "postgres_test_table", + "name": "big-query-with-parameter", "description": "Library dataset description", "tags": [] } @@ -484,8 +455,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -501,7 +472,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -518,53 +489,52 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", - "viewLanguage": "m_query" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartInfo", "aspect": { "json": { "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "hr pbi test description", - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + } + ] } }, "systemMetadata": { @@ -574,14 +544,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePaths", "aspect": { "json": { - "typeNames": [ - "Table" + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -593,13 +563,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", "viewLanguage": "m_query" } }, @@ -610,19 +580,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "ms_sql_native_table", - "description": "hr pbi test description", - "tags": [] + "removed": false } }, "systemMetadata": { @@ -633,7 +597,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -649,14 +613,18 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "datasetProperties", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query-with-join", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -666,13 +634,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "viewProperties", "aspect": { "json": { - "username": "User1@foo.com" + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -682,13 +652,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "viewProperties", "aspect": { "json": { - "username": "User2@foo.com" + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -698,52 +670,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" - }, - "title": "test_tile", - "description": "test_tile", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" - } - ] + "removed": false } }, "systemMetadata": { @@ -753,13 +686,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetProperties", "aspect": { "json": { - "removed": false + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "ms_sql_native_table", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -787,31 +726,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "changeType": "UPSERT", - "aspectName": "chartKey", - "aspect": { - "json": { - "dashboardTool": "powerbi", - "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "subTypes", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" + "typeNames": [ + "Table" ] } }, @@ -822,17 +744,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } - ] + "removed": false } }, "systemMetadata": { @@ -842,53 +760,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { - "createdFrom": "Dataset", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" - }, - "title": "yearly_sales", - "description": "yearly_sales", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -898,14 +782,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "subTypes", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -915,14 +800,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "PowerBI Tile" + "Table" ] } }, @@ -934,14 +819,13 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "chartKey", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "dashboardTool": "powerbi", + "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" } }, "systemMetadata": { @@ -952,7 +836,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1091,27 +975,71 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "viewProperties", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:users.User1@foo.com", - "type": "NONE" - }, - { - "owner": "urn:li:corpuser:users.User2@foo.com", - "type": "NONE" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1141,13 +1069,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -1157,8 +1087,8 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1171,5 +1101,75 @@ "runId": "powerbi-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:users.User1@foo.com", + "type": "NONE" + }, + { + "owner": "urn:li:corpuser:users.User2@foo.com", + "type": "NONE" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json index 93a2c533d21ca0..cc20051f9525bd 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json @@ -18,19 +18,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "corpUserKey", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "public issue_history", - "description": "Library dataset description", - "tags": [] + "username": "User2@foo.com" } }, "systemMetadata": { @@ -59,42 +53,6 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", - "viewLanguage": "m_query" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", - "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { @@ -102,7 +60,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "SNOWFLAKE_TESTTABLE", + "name": "public issue_history", "description": "Library dataset description", "tags": [] } @@ -114,8 +72,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -131,7 +89,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -148,15 +106,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "browsePaths", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", - "viewLanguage": "m_query" + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -166,19 +124,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query", - "description": "Library dataset description", - "tags": [] + "removed": false } }, "systemMetadata": { @@ -189,12 +141,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -204,14 +158,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "PowerBI Tile" ] } }, @@ -223,14 +177,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -241,7 +193,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -250,7 +202,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "big-query-with-parameter", + "name": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description", "tags": [] } @@ -263,12 +215,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -278,14 +232,16 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Table" + "path": [ + { + "id": "demo-workspace" + } ] } }, @@ -297,14 +253,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -315,7 +269,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -324,7 +278,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query-with-join", + "name": "snowflake native-query", "description": "Library dataset description", "tags": [] } @@ -337,12 +291,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -353,7 +309,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -370,15 +326,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "chartKey", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", - "viewLanguage": "m_query" + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -388,19 +343,37 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartInfo", "aspect": { "json": { "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "job-history", - "description": "Library dataset description", - "tags": [] + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + } + ] } }, "systemMetadata": { @@ -411,12 +384,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -426,15 +401,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "corpUserKey", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "username": "User1@foo.com" } }, "systemMetadata": { @@ -445,14 +418,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -463,7 +434,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -472,7 +443,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "postgres_test_table", + "name": "big-query-with-parameter", "description": "Library dataset description", "tags": [] } @@ -484,8 +455,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -501,7 +472,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -518,37 +489,70 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", - "viewLanguage": "m_query" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartInfo", "aspect": { "json": { "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "hr pbi test description", - "tags": [] + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -559,7 +563,25 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -575,14 +597,34 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query-with-join", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -611,18 +653,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "viewProperties", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "ms_sql_native_table", - "description": "hr pbi test description", - "tags": [] + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -651,11 +689,33 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "ms_sql_native_table", + "description": "hr pbi test description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "PowerBI Tile" ] } }, @@ -666,13 +726,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "subTypes", "aspect": { "json": { - "username": "User1@foo.com" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -682,13 +744,13 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "status", "aspect": { "json": { - "username": "User2@foo.com" + "removed": false } }, "systemMetadata": { @@ -698,52 +760,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { - "createdFrom": "Dataset", - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" - }, - "title": "test_tile", - "description": "test_tile", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" - } - ] + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -753,13 +782,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -769,14 +800,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "PowerBI Tile" + "Table" ] } }, @@ -807,11 +838,13 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "browsePathsV2", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" + "path": [ + { + "id": "demo-workspace" + } ] } }, @@ -822,16 +855,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "browsePaths", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -842,19 +873,25 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "dashboardInfo", "aspect": { "json": { "customProperties": { - "createdFrom": "Dataset", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" }, - "title": "yearly_sales", - "description": "yearly_sales", + "title": "test_dashboard", + "description": "Description of test dashboard", + "charts": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ], + "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -865,14 +902,7 @@ "actor": "urn:li:corpuser:unknown" } }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" - } - ] + "dashboardUrl": "https://localhost/dashboards/web/1" } }, "systemMetadata": { @@ -882,8 +912,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -898,15 +928,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "dashboardKey", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" - ] + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" } }, "systemMetadata": { @@ -916,14 +945,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "viewProperties", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -933,15 +963,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "viewProperties", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -951,17 +981,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } - ] + "removed": false } }, "systemMetadata": { @@ -971,15 +997,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "datasetProperties", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -992,32 +1022,14 @@ "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "dashboardInfo", + "aspectName": "browsePathsV2", "aspect": { "json": { - "customProperties": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - }, - "title": "test_dashboard", - "description": "Description of test dashboard", - "charts": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "path": [ + { + "id": "demo-workspace" } - }, - "dashboardUrl": "https://localhost/dashboards/web/1" + ] } }, "systemMetadata": { @@ -1027,8 +1039,26 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1043,14 +1073,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "dashboardKey", + "aspectName": "datasetProperties", "aspect": { "json": { - "dashboardTool": "powerbi", - "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1090,16 +1125,14 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "subTypes", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } + "typeNames": [ + "Table" ] } }, @@ -1108,37 +1141,5 @@ "runId": "powerbi-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json index eda831722cc91a..af3f20260af238 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json @@ -17,6 +17,38 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "User2@foo.com" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", @@ -40,8 +72,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -73,6 +105,40 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", @@ -114,13 +180,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -133,12 +201,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -148,20 +214,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "browsePathsV2", "aspect": { "json": { - "upstreams": [ + "path": [ { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.PBI_TEST.TEST.TESTTABLE,PROD)", - "type": "TRANSFORMED" + "id": "demo-workspace" } ] } @@ -212,6 +273,24 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", @@ -228,6 +307,23 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "chartKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", @@ -246,9 +342,49 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" + }, + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -259,7 +395,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.PBI_TEST.TEST.TESTTABLE,PROD)", "type": "TRANSFORMED" } ] @@ -273,14 +409,21 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "upstreamLineage", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", - "viewLanguage": "m_query" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4,PROD)", + "type": "TRANSFORMED" + } + ] } }, "systemMetadata": { @@ -290,19 +433,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "corpUserKey", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "big-query-with-parameter", - "description": "Library dataset description", - "tags": [] + "username": "User1@foo.com" } }, "systemMetadata": { @@ -315,10 +452,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -328,15 +467,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -347,14 +484,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -365,7 +500,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -374,7 +509,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query-with-join", + "name": "big-query-with-parameter", "description": "Library dataset description", "tags": [] } @@ -386,13 +521,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePaths", "aspect": { "json": { - "removed": false + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -403,7 +540,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -420,28 +557,66 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "status", "aspect": { "json": { - "upstreams": [ + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" + }, + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.GSL_TEST_DB.PUBLIC.SALES_ANALYST,PROD)", - "type": "TRANSFORMED" + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" }, { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.GSL_TEST_DB.PUBLIC.SALES_FORECAST,PROD)", - "type": "TRANSFORMED" + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" } ] } @@ -454,13 +629,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", "viewLanguage": "m_query" } }, @@ -472,7 +647,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -481,7 +656,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "job-history", + "name": "snowflake native-query-with-join", "description": "Library dataset description", "tags": [] } @@ -492,9 +667,27 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Tile" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -509,14 +702,16 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Table" + "path": [ + { + "id": "demo-workspace" + } ] } }, @@ -528,20 +723,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,high_performance_production_unit.salesdb.HR.EMPLOYEES,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Table" ] } }, @@ -553,13 +741,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", "viewLanguage": "m_query" } }, @@ -571,18 +759,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "viewProperties", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "postgres_test_table", - "description": "Library dataset description", - "tags": [] + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -593,12 +777,18 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetProperties", "aspect": { "json": { - "removed": false + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "ms_sql_native_table", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -609,14 +799,18 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "datasetProperties", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -627,21 +821,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "status", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,operational_instance.mics.public.order_date,PROD)", - "type": "TRANSFORMED" - } - ] + "removed": false } }, "systemMetadata": { @@ -651,15 +836,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "chartKey", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", - "viewLanguage": "m_query" + "dashboardTool": "powerbi", + "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" } }, "systemMetadata": { @@ -670,18 +854,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "hr pbi test description", - "tags": [] + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -707,14 +887,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePaths", "aspect": { "json": { - "typeNames": [ - "Table" + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -726,7 +906,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -737,7 +917,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.library.dbo.book_issue,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.COMMOPSDB.dbo.V_PS_CD_RETENTION,PROD)", "type": "TRANSFORMED" } ] @@ -751,36 +931,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", - "viewLanguage": "m_query" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-lineage-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "ms_sql_native_table", - "description": "hr pbi test description", - "tags": [] + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -791,43 +949,9 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-lineage-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-lineage-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "upstreamLineage", "aspect": { "json": { "upstreams": [ @@ -836,7 +960,15 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.COMMOPSDB.dbo.V_PS_CD_RETENTION,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.GSL_TEST_DB.PUBLIC.SALES_ANALYST,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,sn-2.GSL_TEST_DB.PUBLIC.SALES_FORECAST,PROD)", "type": "TRANSFORMED" } ] @@ -849,30 +981,67 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", - "changeType": "UPSERT", - "aspectName": "corpUserKey", - "aspect": { - "json": { - "username": "User1@foo.com" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-lineage-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", - "changeType": "UPSERT", - "aspectName": "corpUserKey", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", "aspect": { - "json": { - "username": "User2@foo.com" - } + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -881,50 +1050,20 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "upstreamLineage", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" - }, - "title": "test_tile", - "description": "test_tile", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" - }, + "upstreams": [ { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.library.dbo.book_issue,PROD)", + "type": "TRANSFORMED" } ] } @@ -936,8 +1075,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -952,50 +1091,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "PowerBI Tile" - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-lineage-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "dashboardKey", "aspect": { "json": { "dashboardTool": "powerbi", - "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-lineage-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "changeType": "UPSERT", - "aspectName": "browsePaths", - "aspect": { - "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" } }, "systemMetadata": { @@ -1005,8 +1108,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1025,70 +1128,27 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "ownership", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" - }, - "title": "yearly_sales", - "description": "yearly_sales", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ + "owners": [ { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + "owner": "urn:li:corpuser:users.User1@foo.com", + "type": "NONE" }, { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + "owner": "urn:li:corpuser:users.User2@foo.com", + "type": "NONE" } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-lineage-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-lineage-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "changeType": "UPSERT", - "aspectName": "chartKey", - "aspect": { - "json": { - "dashboardTool": "powerbi", - "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } }, "systemMetadata": { @@ -1098,14 +1158,21 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "upstreamLineage", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,high_performance_production_unit.salesdb.HR.EMPLOYEES,PROD)", + "type": "TRANSFORMED" + } ] } }, @@ -1116,15 +1183,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "viewProperties", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1134,17 +1201,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "datasetProperties", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } - ] + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1154,15 +1223,15 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "viewProperties", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1172,77 +1241,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "PATCH", - "aspectName": "dashboardInfo", - "aspect": { - "json": [ - { - "op": "add", - "path": "/customProperties/chartCount", - "value": "2" - }, - { - "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" - }, - { - "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" - }, - { - "op": "add", - "path": "/description", - "value": "Description of test dashboard" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - }, - { - "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" - }, - { - "op": "add", - "path": "/lastModified", - "value": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-lineage-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1257,14 +1257,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "dashboardKey", + "aspectName": "datasetProperties", "aspect": { "json": { - "dashboardTool": "powerbi", - "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1274,27 +1279,15 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "subTypes", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:users.User1@foo.com", - "type": "NONE" - }, - { - "owner": "urn:li:corpuser:users.User2@foo.com", - "type": "NONE" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -1304,16 +1297,14 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "subTypes", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } + "typeNames": [ + "Table" ] } }, @@ -1324,8 +1315,8 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1340,13 +1331,22 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "upstreamLineage", "aspect": { "json": { - "removed": false + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,operational_instance.mics.public.order_date,PROD)", + "type": "TRANSFORMED" + } + ] } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json index 6f502cdfc0f5bf..f4cd657b860e02 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json @@ -33,6 +33,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)", @@ -56,13 +72,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "viewProperties", "aspect": { "json": { - "username": "User2@foo.com" + "materialized": false, + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -72,15 +90,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "corpUserKey", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", - "viewLanguage": "m_query" + "username": "User2@foo.com" } }, "systemMetadata": { @@ -91,7 +107,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -127,30 +143,16 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -161,7 +163,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -178,15 +180,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -213,13 +213,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", "viewLanguage": "m_query" } }, @@ -229,16 +229,69 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" + }, + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -249,7 +302,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -258,7 +311,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "big-query-with-parameter", + "name": "snowflake native-query", "description": "Library dataset description", "tags": [] } @@ -271,13 +324,31 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "PowerBI Tile" ] } }, @@ -305,7 +376,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -314,7 +385,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query", + "name": "big-query-with-parameter", "description": "Library dataset description", "tags": [] } @@ -327,14 +398,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "viewProperties", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -345,12 +416,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -361,7 +434,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -399,7 +472,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -419,12 +492,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", - "viewLanguage": "m_query" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -434,14 +507,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePaths", "aspect": { "json": { - "typeNames": [ - "Table" + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -452,13 +525,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "chartKey", "aspect": { "json": { - "removed": false + "dashboardTool": "powerbi", + "chartId": "myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" } }, "systemMetadata": { @@ -471,16 +545,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "viewProperties", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query-with-join", - "description": "Library dataset description", - "tags": [] + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -490,15 +560,21 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "browsePathsV2", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", - "viewLanguage": "m_query" + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)" + }, + { + "id": "demo-workspace" + } + ] } }, "systemMetadata": { @@ -509,69 +585,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", - "viewLanguage": "m_query" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "changeType": "UPSERT", - "aspectName": "chartInfo", - "aspect": { - "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" - }, - "title": "test_tile", - "description": "test_tile", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.public_issue_history,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_testtable,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.big-query-with-parameter,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.job-history,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)" - } - ] + "removed": false } }, "systemMetadata": { @@ -582,7 +601,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -591,7 +610,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "postgres_test_table", + "name": "snowflake native-query-with-join", "description": "Library dataset description", "tags": [] } @@ -606,24 +625,6 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "PowerBI Tile" - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)", - "changeType": "UPSERT", "aspectName": "status", "aspect": { "json": { @@ -637,14 +638,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.snowflake_native-query-with-join,PROD)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "subTypes", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" + "typeNames": [ + "Table" ] } }, @@ -655,15 +656,27 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "ownership", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "owners": [ + { + "owner": "urn:li:corpuser:users.User1@foo.com", + "type": "NONE" + }, + { + "owner": "urn:li:corpuser:users.User2@foo.com", + "type": "NONE" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } }, "systemMetadata": { @@ -673,14 +686,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "dashboardKey", "aspect": { "json": { "dashboardTool": "powerbi", - "chartId": "myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" } }, "systemMetadata": { @@ -691,18 +704,34 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "chartInfo", "aspect": { "json": { - "path": [ + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" + }, + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)" + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)" }, { - "id": "demo-workspace" + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)" } ] } @@ -715,7 +744,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -730,14 +759,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePaths", "aspect": { "json": { - "typeNames": [ - "Table" + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -748,35 +777,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "hr pbi test description", - "tags": [] + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -787,13 +796,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", "viewLanguage": "m_query" } }, @@ -804,15 +813,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "chartKey", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "dashboardTool": "powerbi", + "chartId": "myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -859,16 +867,85 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", - "viewLanguage": "m_query" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -878,27 +955,21 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "browsePathsV2", "aspect": { "json": { - "owners": [ + "path": [ { - "owner": "urn:li:corpuser:users.User1@foo.com", - "type": "NONE" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)" }, { - "owner": "urn:li:corpuser:users.User2@foo.com", - "type": "NONE" + "id": "demo-workspace" } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + ] } }, "systemMetadata": { @@ -908,14 +979,15 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "dashboardKey", + "aspectName": "browsePaths", "aspect": { "json": { - "dashboardTool": "powerbi", - "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -925,35 +997,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "browsePathsV2", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" - }, - "title": "yearly_sales", - "description": "yearly_sales", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ + "path": [ { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)" }, { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.ms_sql_native_table,PROD)" + "id": "demo-workspace" } ] } @@ -965,8 +1021,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -981,15 +1037,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "datasetProperties", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -999,14 +1059,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "PowerBI Tile" + "Table" ] } }, @@ -1017,14 +1077,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "viewProperties", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385" + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1034,67 +1095,16 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "PATCH", - "aspectName": "dashboardInfo", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", "aspect": { - "json": [ - { - "op": "add", - "path": "/customProperties/chartCount", - "value": "2" - }, - { - "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" - }, - { - "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" - }, - { - "op": "add", - "path": "/description", - "value": "Description of test dashboard" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "value": "urn:li:chart:(powerbi,myPlatformInstance.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "value": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - }, - { - "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" - }, - { - "op": "add", - "path": "/lastModified", - "value": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] + "json": { + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1103,21 +1113,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)" - }, - { - "id": "demo-workspace" - } - ] + "removed": false } }, "systemMetadata": { @@ -1127,15 +1129,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,myPlatformInstance.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "datasetProperties", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -1145,21 +1151,13 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.library-dataset.postgres_test_table,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,myPlatformInstance)" - }, - { - "id": "demo-workspace" - } - ] + "removed": false } }, "systemMetadata": { @@ -1169,13 +1167,15 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,myPlatformInstance.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,myplatforminstance.hr_pbi_test.dbo_book_issue,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json index 4393a87d1f5707..e134d795af9ef0 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json @@ -86,15 +86,30 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "chartInfo", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", - "viewLanguage": "m_query" + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" + }, + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [] } }, "systemMetadata": { @@ -104,30 +119,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "schemaMetadata", + "aspectName": "status", "aspect": { "json": { - "schemaName": "dbo_book_issue", - "platform": "urn:li:dataPlatform:powerbi", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [] + "removed": false } }, "systemMetadata": { @@ -137,19 +135,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartKey", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "", - "tags": [] + "dashboardTool": "powerbi", + "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" } }, "systemMetadata": { @@ -159,13 +152,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePaths", "aspect": { "json": { - "removed": false + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -175,14 +170,33 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "container", "aspect": { "json": { - "typeNames": [ - "Table" + "container": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", + "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" + } ] } }, @@ -196,10 +210,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "viewProperties", "aspect": { "json": { - "container": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -209,15 +225,38 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "chartInfo", "aspect": { "json": { - "tags": [ + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" + }, + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ { - "tag": "urn:li:tag:Certified" + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)" } ] } @@ -232,19 +271,16 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "datasetProperties", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", - "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" - }, - { - "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", - "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" - } - ] + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "", + "tags": [] } }, "systemMetadata": { @@ -255,14 +291,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -273,12 +307,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "ms_sql_native_table", + "schemaName": "dbo_book_issue", "platform": "urn:li:dataPlatform:powerbi", "version": 0, "created": { @@ -305,19 +339,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "ms_sql_native_table", - "description": "", - "tags": [] + "removed": false } }, "systemMetadata": { @@ -353,12 +381,16 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "globalTags", "aspect": { "json": { - "removed": false + "tags": [ + { + "tag": "urn:li:tag:Certified" + } + ] } }, "systemMetadata": { @@ -368,14 +400,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "PowerBI Tile" ] } }, @@ -389,10 +421,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "viewProperties", "aspect": { "json": { - "container": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -403,15 +437,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "subTypes", "aspect": { "json": { - "tags": [ - { - "tag": "urn:li:tag:Certified" - } + "typeNames": [ + "Table" ] } }, @@ -425,19 +457,16 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "datasetProperties", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", - "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" - }, - { - "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", - "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" - } - ] + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "ms_sql_native_table", + "description": "", + "tags": [] } }, "systemMetadata": { @@ -448,14 +477,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"database.sql.net\", \"analytics\", [Query=\"select * from analytics.sales_revenue\"])\nin\n Source", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -466,12 +493,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "revenue", + "schemaName": "ms_sql_native_table", "platform": "urn:li:dataPlatform:powerbi", "version": 0, "created": { @@ -488,59 +515,7 @@ "rawSchema": "" } }, - "fields": [ - { - "fieldPath": "op_item_id", - "nullable": false, - "description": "op_item_id column description", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "op_id", - "nullable": false, - "description": "op_id description", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "op_product_name", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "event_name", - "nullable": false, - "description": "let\n x column description", - "type": { - "type": { - "com.linkedin.schema.NullType": {} - } - }, - "nativeDataType": "measure", - "recursive": false, - "isPartOfKey": false - } - ] + "fields": [] } }, "systemMetadata": { @@ -550,19 +525,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartKey", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "revenue", - "description": "", - "tags": [] + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -598,29 +568,15 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "globalTags", "aspect": { "json": { - "typeNames": [ - "Table" + "tags": [ + { + "tag": "urn:li:tag:Certified" + } ] } }, @@ -632,7 +588,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -650,14 +606,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "viewProperties", "aspect": { "json": { - "tags": [ - { - "tag": "urn:li:tag:Certified" - } - ] + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"database.sql.net\", \"analytics\", [Query=\"select * from analytics.sales_revenue\"])\nin\n Source", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -668,20 +622,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "subTypes", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", - "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" - }, - { - "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", - "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" - } + "typeNames": [ + "Table" ] } }, @@ -692,30 +639,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { - "createdFrom": "Dataset", - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" - }, - "title": "test_tile", - "description": "test_tile", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" }, - "inputs": [] + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "revenue", + "description": "", + "tags": [] } }, "systemMetadata": { @@ -725,8 +661,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -741,14 +677,82 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "schemaMetadata", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" + "schemaName": "revenue", + "platform": "urn:li:dataPlatform:powerbi", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "op_item_id", + "nullable": false, + "description": "op_item_id column description", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "op_id", + "nullable": false, + "description": "op_id description", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "op_product_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "event_name", + "nullable": false, + "description": "let\n x column description", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "measure", + "recursive": false, + "isPartOfKey": false + } + ] } }, "systemMetadata": { @@ -759,7 +763,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", "aspectName": "browsePaths", "aspect": { @@ -776,13 +780,22 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "upstreamLineage", "aspect": { "json": { - "container": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.analytics.analytics.sales_revenue,PROD)", + "type": "TRANSFORMED" + } + ] } }, "systemMetadata": { @@ -795,17 +808,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "globalTags", "aspect": { "json": { - "upstreams": [ + "tags": [ { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.analytics.analytics.sales_revenue,PROD)", - "type": "TRANSFORMED" + "tag": "urn:li:tag:Certified" } ] } @@ -817,18 +825,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", - "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" - } - ] + "container": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" } }, "systemMetadata": { @@ -838,39 +841,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" - }, - "title": "yearly_sales", - "description": "yearly_sales", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)" - } + "typeNames": [ + "Table" ] } }, @@ -881,13 +859,22 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePathsV2", "aspect": { "json": { - "removed": false + "path": [ + { + "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", + "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" + }, + { + "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", + "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" + } + ] } }, "systemMetadata": { @@ -897,15 +884,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "container", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" - ] + "container": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" } }, "systemMetadata": { @@ -915,14 +900,22 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "browsePathsV2", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + "path": [ + { + "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", + "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" + }, + { + "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", + "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" + } + ] } }, "systemMetadata": { @@ -932,14 +925,21 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "browsePathsV2", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" + "path": [ + { + "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", + "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" + }, + { + "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", + "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" + } ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json index 6da5f5781112e8..e75557982da581 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json @@ -17,6 +17,40 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.public_issue_history,DEV)", @@ -40,14 +74,67 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.public_issue_history,DEV)", - "changeType": "UPSERT", - "aspectName": "status", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", "aspect": { - "json": { - "removed": false - } + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -74,15 +161,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -92,19 +177,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "dashboardKey", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "SNOWFLAKE_TESTTABLE", - "description": "Library dataset description", - "tags": [] + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" } }, "systemMetadata": { @@ -117,10 +197,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -130,14 +212,20 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Table" + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)" + }, + { + "id": "demo-workspace" + } ] } }, @@ -149,14 +237,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -167,7 +253,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -176,7 +262,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query", + "name": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description", "tags": [] } @@ -189,12 +275,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -207,12 +295,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "viewProperties", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -225,12 +313,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -262,13 +348,27 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.big-query-with-parameter,DEV)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "ownership", "aspect": { "json": { - "removed": false + "owners": [ + { + "owner": "urn:li:corpuser:users.User1@foo.com", + "type": "NONE" + }, + { + "owner": "urn:li:corpuser:users.User2@foo.com", + "type": "NONE" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } }, "systemMetadata": { @@ -297,14 +397,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -315,7 +413,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -324,7 +422,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query-with-join", + "name": "snowflake native-query", "description": "Library dataset description", "tags": [] } @@ -337,12 +435,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -353,7 +453,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -373,12 +473,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -411,12 +509,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -444,15 +544,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "corpUserKey", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", - "viewLanguage": "m_query" + "username": "User2@foo.com" } }, "systemMetadata": { @@ -462,19 +560,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "postgres_test_table", - "description": "Library dataset description", - "tags": [] + "removed": false } }, "systemMetadata": { @@ -484,13 +576,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "corpUserKey", "aspect": { "json": { - "removed": false + "username": "User1@foo.com" } }, "systemMetadata": { @@ -500,15 +592,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -518,15 +608,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "browsePaths", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", - "viewLanguage": "m_query" + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -536,19 +626,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "hr pbi test description", - "tags": [] + "removed": false } }, "systemMetadata": { @@ -559,12 +643,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -574,14 +660,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "PowerBI Tile" ] } }, @@ -593,14 +679,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -611,17 +695,17 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "ms_sql_native_table", - "description": "hr pbi test description", + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query-with-join", + "description": "Library dataset description", "tags": [] } }, @@ -632,13 +716,21 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePathsV2", "aspect": { "json": { - "removed": false + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)" + }, + { + "id": "demo-workspace" + } + ] } }, "systemMetadata": { @@ -649,7 +741,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -666,29 +758,14 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", - "changeType": "UPSERT", - "aspectName": "corpUserKey", - "aspect": { - "json": { - "username": "User1@foo.com" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "chartKey", "aspect": { "json": { - "username": "User2@foo.com" + "dashboardTool": "powerbi", + "chartId": "aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" } }, "systemMetadata": { @@ -753,13 +830,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -770,13 +849,13 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePaths", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -787,32 +866,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "changeType": "UPSERT", - "aspectName": "chartKey", - "aspect": { - "json": { - "dashboardTool": "powerbi", - "chartId": "aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "status", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "removed": false } }, "systemMetadata": { @@ -822,21 +882,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "datasetProperties", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)" - }, - { - "id": "demo-workspace" - } - ] + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -849,34 +907,10 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" - }, - "title": "yearly_sales", - "description": "yearly_sales", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)" - } - ] + "removed": false } }, "systemMetadata": { @@ -886,13 +920,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -905,11 +941,12 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "subTypes", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385" + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -922,11 +959,17 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)" + }, + { + "id": "demo-workspace" + } ] } }, @@ -940,12 +983,11 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "chartKey", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "dashboardTool": "powerbi", + "chartId": "aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -958,16 +1000,32 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "chartInfo", "aspect": { "json": { - "path": [ + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" + }, + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)" + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)" }, { - "id": "demo-workspace" + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)" } ] } @@ -979,15 +1037,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "datasetProperties", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -997,67 +1059,16 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "PATCH", - "aspectName": "dashboardInfo", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", "aspect": { - "json": [ - { - "op": "add", - "path": "/customProperties/chartCount", - "value": "2" - }, - { - "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" - }, - { - "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" - }, - { - "op": "add", - "path": "/description", - "value": "Description of test dashboard" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - }, - { - "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" - }, - { - "op": "add", - "path": "/lastModified", - "value": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] + "json": { + "typeNames": [ + "Table" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1066,13 +1077,15 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1082,14 +1095,13 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "dashboardKey", + "aspectName": "status", "aspect": { "json": { - "dashboardTool": "powerbi", - "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" + "removed": false } }, "systemMetadata": { @@ -1099,27 +1111,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "datasetProperties", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:users.User1@foo.com", - "type": "NONE" - }, - { - "owner": "urn:li:corpuser:users.User2@foo.com", - "type": "NONE" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "ms_sql_native_table", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -1129,20 +1133,14 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "subTypes", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)" - }, - { - "id": "demo-workspace" - } + "typeNames": [ + "Table" ] } }, @@ -1153,13 +1151,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1169,8 +1169,8 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,aws-ap-south-1.hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json b/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json index b8963a0d7782d2..87e89e5ed74f2e 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json @@ -39,6 +39,40 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1645599600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1645599600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", @@ -107,40 +141,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } - }, - "systemMetadata": { - "lastObserved": 1645599600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1645599600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json index e327ca695beb77..bc0feef4a127d3 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json @@ -17,6 +17,77 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" + }, + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", @@ -40,8 +111,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -74,15 +145,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", - "viewLanguage": "m_query" + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -92,19 +163,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "chartKey", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "SNOWFLAKE_TESTTABLE", - "description": "Library dataset description", - "tags": [] + "dashboardTool": "powerbi", + "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" } }, "systemMetadata": { @@ -117,10 +183,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -130,14 +198,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePaths", "aspect": { "json": { - "typeNames": [ - "Table" + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -149,14 +217,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -167,7 +233,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -176,7 +242,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query", + "name": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description", "tags": [] } @@ -191,10 +257,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -204,14 +272,16 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Table" + "path": [ + { + "id": "demo-workspace" + } ] } }, @@ -223,14 +293,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -241,7 +309,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -250,7 +318,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "big-query-with-parameter", + "name": "snowflake native-query", "description": "Library dataset description", "tags": [] } @@ -263,12 +331,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -279,7 +349,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -296,15 +366,37 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "chartInfo", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", - "viewLanguage": "m_query" + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" + }, + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + } + ] } }, "systemMetadata": { @@ -314,19 +406,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query-with-join", - "description": "Library dataset description", - "tags": [] + "removed": false } }, "systemMetadata": { @@ -337,12 +423,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -352,15 +440,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "chartKey", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -371,14 +458,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -389,7 +474,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -398,7 +483,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "job-history", + "name": "big-query-with-parameter", "description": "Library dataset description", "tags": [] } @@ -410,13 +495,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -427,7 +514,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -444,15 +531,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "browsePaths", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", - "viewLanguage": "m_query" + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -462,19 +549,17 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "browsePathsV2", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "postgres_test_table", - "description": "Library dataset description", - "tags": [] + "path": [ + { + "id": "demo-workspace" + } + ] } }, "systemMetadata": { @@ -485,12 +570,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -500,14 +587,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePaths", "aspect": { "json": { - "typeNames": [ - "Table" + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -519,14 +606,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -537,17 +622,17 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "hr pbi test description", + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query-with-join", + "description": "Library dataset description", "tags": [] } }, @@ -559,12 +644,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "viewProperties", "aspect": { "json": { - "removed": false + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -577,12 +664,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "viewProperties", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -595,12 +682,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "status", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", - "viewLanguage": "m_query" + "removed": false } }, "systemMetadata": { @@ -632,14 +717,67 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", - "changeType": "UPSERT", - "aspectName": "status", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", "aspect": { - "json": { - "removed": false - } + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -666,52 +804,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" - }, - "title": "test_tile", - "description": "test_tile", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" - } - ] + "removed": false } }, "systemMetadata": { @@ -721,13 +820,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetProperties", "aspect": { "json": { - "removed": false + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -737,14 +842,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "PowerBI Tile" + "Table" ] } }, @@ -755,14 +860,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "subTypes", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -772,15 +878,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "status", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "removed": false } }, "systemMetadata": { @@ -790,126 +894,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "demo-workspace" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "changeType": "UPSERT", - "aspectName": "chartInfo", - "aspect": { - "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" - }, - "title": "yearly_sales", - "description": "yearly_sales", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "dashboardKey", "aspect": { "json": { "dashboardTool": "powerbi", - "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "PowerBI Tile" - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "changeType": "UPSERT", - "aspectName": "browsePaths", - "aspect": { - "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" } }, "systemMetadata": { @@ -919,8 +911,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -940,13 +932,13 @@ }, { "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", "aspectName": "browsePaths", "aspect": { "json": { "paths": [ - "/powerbi/demo-workspace" + "/powerbi/second-demo-workspace" ] } }, @@ -958,7 +950,7 @@ }, { "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { @@ -966,37 +958,22 @@ { "op": "add", "path": "/customProperties/chartCount", - "value": "2" + "value": "0" }, { "op": "add", "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "value": "second-demo-workspace" }, { "op": "add", "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "value": "64ED5CAD-7C22-4684-8180-826122881108" }, { "op": "add", "path": "/title", - "value": "test_dashboard" - }, - { - "op": "add", - "path": "/description", - "value": "Description of test dashboard" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + "value": "test_dashboard2" }, { "op": "add", @@ -1027,7 +1004,7 @@ }, { "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1042,14 +1019,15 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "dashboardKey", + "aspectName": "viewProperties", "aspect": { "json": { - "dashboardTool": "powerbi", - "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1059,17 +1037,15 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "viewProperties", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } - ] + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1079,15 +1055,13 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "status", "aspect": { "json": { - "paths": [ - "/powerbi/second-demo-workspace" - ] + "removed": false } }, "systemMetadata": { @@ -1097,52 +1071,20 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", - "changeType": "PATCH", - "aspectName": "dashboardInfo", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", "aspect": { - "json": [ - { - "op": "add", - "path": "/customProperties/chartCount", - "value": "0" - }, - { - "op": "add", - "path": "/customProperties/workspaceName", - "value": "second-demo-workspace" - }, - { - "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C22-4684-8180-826122881108" - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard2" - }, - { - "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, - { - "op": "add", - "path": "/lastModified", - "value": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1154,6 +1096,41 @@ "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", "aspectName": "status", "aspect": { "json": { @@ -1167,14 +1144,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "dashboardKey", + "aspectName": "datasetProperties", "aspect": { "json": { - "dashboardTool": "powerbi", - "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE" + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1202,5 +1184,23 @@ "runId": "powerbi-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json index 90c8ee5d0379e3..db5af68508f101 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json @@ -17,6 +17,38 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "User2@foo.com" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", @@ -40,8 +72,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -73,6 +105,40 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", @@ -114,13 +180,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -133,12 +201,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -148,20 +214,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "browsePathsV2", "aspect": { "json": { - "upstreams": [ + "path": [ { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_production_instance.pbi_test.test.testtable,PROD)", - "type": "TRANSFORMED" + "id": "demo-workspace" } ] } @@ -214,23 +275,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -250,19 +295,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "status", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,operations_analytics.transformed_prod.v_aps_sme_units_v4,PROD)", - "type": "TRANSFORMED" - } - ] + "removed": false } }, "systemMetadata": { @@ -272,15 +308,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "chartKey", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", - "viewLanguage": "m_query" + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -291,18 +326,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "big-query-with-parameter", - "description": "Library dataset description", - "tags": [] + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -312,13 +343,37 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "chartInfo", "aspect": { "json": { - "removed": false + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" + }, + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + } + ] } }, "systemMetadata": { @@ -329,13 +384,20 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "upstreamLineage", "aspect": { "json": { - "typeNames": [ - "Table" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_production_instance.pbi_test.test.testtable,PROD)", + "type": "TRANSFORMED" + } ] } }, @@ -347,7 +409,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -358,7 +420,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-computing-dev-account.my-test-project.universal.d_wh_date,QA)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,operations_analytics.transformed_prod.v_aps_sme_units_v4,PROD)", "type": "TRANSFORMED" } ] @@ -370,15 +432,31 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "User1@foo.com" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", "viewLanguage": "m_query" } }, @@ -390,7 +468,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -399,7 +477,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query-with-join", + "name": "big-query-with-parameter", "description": "Library dataset description", "tags": [] } @@ -411,8 +489,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -428,14 +506,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -445,29 +521,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "browsePaths", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_forecast,PROD)", - "type": "TRANSFORMED" - } + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -479,14 +540,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", - "viewLanguage": "m_query" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -496,19 +557,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "job-history", - "description": "Library dataset description", - "tags": [] + "removed": false } }, "systemMetadata": { @@ -518,13 +573,52 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "chartInfo", "aspect": { "json": { - "removed": false + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" + }, + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + } + ] } }, "systemMetadata": { @@ -535,13 +629,20 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "upstreamLineage", "aspect": { "json": { - "typeNames": [ - "Table" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-computing-dev-account.my-test-project.universal.d_wh_date,QA)", + "type": "TRANSFORMED" + } ] } }, @@ -552,21 +653,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle-sales-instance.salesdb.hr.employees,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "PowerBI Tile" ] } }, @@ -578,13 +672,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", "viewLanguage": "m_query" } }, @@ -596,7 +690,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -605,7 +699,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "postgres_test_table", + "name": "snowflake native-query-with-join", "description": "Library dataset description", "tags": [] } @@ -617,13 +711,17 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePathsV2", "aspect": { "json": { - "removed": false + "path": [ + { + "id": "demo-workspace" + } + ] } }, "systemMetadata": { @@ -634,14 +732,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "removed": false } }, "systemMetadata": { @@ -651,22 +747,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "chartKey", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,mics.public.order_date,PROD)", - "type": "TRANSFORMED" - } - ] + "dashboardTool": "powerbi", + "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" } }, "systemMetadata": { @@ -695,18 +783,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "viewProperties", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "hr pbi test description", - "tags": [] + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -717,12 +801,18 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetProperties", "aspect": { "json": { - "removed": false + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "ms_sql_native_table", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -735,12 +825,16 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "datasetProperties", "aspect": { "json": { - "typeNames": [ - "Table" - ] + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -751,21 +845,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "status", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,library.dbo.book_issue,PROD)", - "type": "TRANSFORMED" - } - ] + "removed": false } }, "systemMetadata": { @@ -776,14 +861,14 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", - "viewLanguage": "m_query" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -796,16 +881,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "ms_sql_native_table", - "description": "hr pbi test description", - "tags": [] + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -816,7 +897,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -831,14 +912,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePaths", "aspect": { "json": { - "typeNames": [ - "Table" + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -874,13 +955,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "subTypes", "aspect": { "json": { - "username": "User1@foo.com" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -890,14 +973,67 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", - "changeType": "UPSERT", - "aspectName": "corpUserKey", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", "aspect": { - "json": { - "username": "User2@foo.com" - } + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] }, "systemMetadata": { "lastObserved": 1643871600000, @@ -906,50 +1042,28 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "upstreamLineage", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" - }, - "title": "test_tile", - "description": "test_tile", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" - }, + "upstreams": [ { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_analyst,PROD)", + "type": "TRANSFORMED" }, { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,gsl_test_db.public.sales_forecast,PROD)", + "type": "TRANSFORMED" } ] } @@ -961,30 +1075,21 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "upstreamLineage", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,library.dbo.book_issue,PROD)", + "type": "TRANSFORMED" + } ] } }, @@ -995,14 +1100,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "status", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" + "removed": false } }, "systemMetadata": { @@ -1012,15 +1116,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "dashboardKey", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" } }, "systemMetadata": { @@ -1030,8 +1133,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1050,53 +1153,27 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "ownership", "aspect": { "json": { - "customProperties": { - "createdFrom": "Dataset", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" - }, - "title": "yearly_sales", - "description": "yearly_sales", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ + "owners": [ { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + "owner": "urn:li:corpuser:users.User1@foo.com", + "type": "NONE" }, { - "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + "owner": "urn:li:corpuser:users.User2@foo.com", + "type": "NONE" } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } }, "systemMetadata": { @@ -1106,14 +1183,22 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "upstreamLineage", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle-sales-instance.salesdb.hr.employees,PROD)", + "type": "TRANSFORMED" + } + ] } }, "systemMetadata": { @@ -1123,15 +1208,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "viewProperties", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" - ] + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1141,15 +1226,15 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "viewProperties", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" - ] + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1159,17 +1244,19 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "datasetProperties", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } - ] + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1179,14 +1266,14 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "subTypes", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" + "typeNames": [ + "Table" ] } }, @@ -1197,77 +1284,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "PATCH", - "aspectName": "dashboardInfo", - "aspect": { - "json": [ - { - "op": "add", - "path": "/customProperties/chartCount", - "value": "2" - }, - { - "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" - }, - { - "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" - }, - { - "op": "add", - "path": "/description", - "value": "Description of test dashboard" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", - "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - }, - { - "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" - }, - { - "op": "add", - "path": "/lastModified", - "value": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1282,44 +1300,19 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "changeType": "UPSERT", - "aspectName": "dashboardKey", - "aspect": { - "json": { - "dashboardTool": "powerbi", - "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "datasetProperties", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:users.User1@foo.com", - "type": "NONE" - }, - { - "owner": "urn:li:corpuser:users.User2@foo.com", - "type": "NONE" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -1329,16 +1322,14 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "subTypes", "aspect": { "json": { - "path": [ - { - "id": "demo-workspace" - } + "typeNames": [ + "Table" ] } }, @@ -1349,8 +1340,8 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1365,13 +1356,22 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "upstreamLineage", "aspect": { "json": { - "removed": false + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,mics.public.order_date,PROD)", + "type": "TRANSFORMED" + } + ] } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/powerbi/mock_data/cross_workspace_mock_response.json b/metadata-ingestion/tests/integration/powerbi/mock_data/cross_workspace_mock_response.json new file mode 100644 index 00000000000000..9b6931727bc1a3 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/mock_data/cross_workspace_mock_response.json @@ -0,0 +1,220 @@ +{ + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "A8D655A6-F521-477E-8C22-255018583BF4", + "isReadOnly": true, + "name": "Sales", + "type": "Workspace", + "state": "Active" + }, + { + "id": "C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492", + "isReadOnly": true, + "name": "Global Workspace", + "type": "Workspace", + "state": "Active" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/A8D655A6-F521-477E-8C22-255018583BF4/dashboards": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "A1C7204F-4D04-4E5E-B886-B30EA2C64CB3", + "isReadOnly": true, + "displayName": "test_dashboard", + "description": "Description of test dashboard", + "embedUrl": "https://localhost/dashboards/embed/1", + "webUrl": "https://localhost/dashboards/web/1" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492/dashboards": { + "method": "GET", + "status_code": 200, + "json": { + "value": [] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/A8D655A6-F521-477E-8C22-255018583BF4/dashboards/A1C7204F-4D04-4E5E-B886-B30EA2C64CB3/tiles": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "885D1762-1655-46BA-AFE3-74C6EC403A9E", + "title": "Sale Order Tile", + "embedUrl": "https://localhost/tiles/embed/1", + "datasetId": "317456E5-1FC7-4BDC-9C84-1185825E293D" + }, + { + "id": "945C2C2A-4588-45DE-8385-F24F5E39A57C", + "title": "Yearly Sales", + "embedUrl": "https://localhost/tiles/embed/2", + "datasetId": "FE362B98-956E-4394-BA37-6367EE6435E9" + }, + { + "id": "B847EBDA-BC48-4F92-8E16-6B46D900E7BB", + "title": "Not Present In Current Ingestion", + "embedUrl": "https://localhost/tiles/embed/2", + "datasetId": "0F0ADA0E-E38A-44F6-B667-90E93A96F5A1" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/62DAF926-0B18-4FF1-982C-2A3EB6B8F0E4": { + "method": "GET", + "status_code": 200, + "json": { + "workspaces": [ + { + "id": "A8D655A6-F521-477E-8C22-255018583BF4", + "name": "Sales", + "type": "Workspace", + "state": "Active", + "datasets": [ + { + "id": "317456E5-1FC7-4BDC-9C84-1185825E293D", + "endorsementDetails": { + "endorsement": "Promoted" + }, + "name": "sales semantic model", + "tables": [ + { + "name": "public issue_history", + "source": [ + { + "expression": "dummy" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + } + ] + } + ] + }, + { + "id": "C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492", + "name": "Global Workspace", + "type": "Workspace", + "state": "Active", + "datasets": [ + { + "id": "FE362B98-956E-4394-BA37-6367EE6435E9", + "endorsementDetails": { + "endorsement": "Promoted" + }, + "name": "base_records", + "tables": [ + { + "name": "core_sales_set", + "source": [ + { + "expression": "dummy" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + } + ] + } + ] + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/62DAF926-0B18-4FF1-982C-2A3EB6B8F0E4": { + "method": "GET", + "status_code": 200, + "json": { + "status": "SUCCEEDED" + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/81B02907-E2A3-45C3-B505-3781839C8CAA": { + "method": "GET", + "status_code": 200, + "json": { + "workspaces": [ + { + "id": "C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492", + "name": "Global Workspace", + "type": "Workspace", + "state": "Active", + "datasets": [ + { + "id": "FE362B98-956E-4394-BA37-6367EE6435E9", + "endorsementDetails": { + "endorsement": "Promoted" + }, + "name": "base_records", + "tables": [ + { + "name": "core_sales_set", + "source": [ + { + "expression": "dummy" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + } + ] + } + ] + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/81B02907-E2A3-45C3-B505-3781839C8CAA": { + "method": "GET", + "status_code": 200, + "json": { + "status": "SUCCEEDED" + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/A8D655A6-F521-477E-8C22-255018583BF4/datasets/317456E5-1FC7-4BDC-9C84-1185825E293D": { + "method": "GET", + "status_code": 200, + "json": { + "id": "317456E5-1FC7-4BDC-9C84-1185825E293D", + "name": "sales semantic model", + "description": "sales semantic model", + "webUrl": "http://localhost/groups/A8D655A6-F521-477E-8C22-255018583BF4/datasets/317456E5-1FC7-4BDC-9C84-1185825E293D" + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492/datasets/FE362B98-956E-4394-BA37-6367EE6435E9": { + "method": "GET", + "status_code": 200, + "json": { + "id": "FE362B98-956E-4394-BA37-6367EE6435E9", + "name": "base_records", + "description": "base_records", + "webUrl": "http://localhost/groups/C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492/datasets/FE362B98-956E-4394-BA37-6367EE6435E9" + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/mock_data/default_mock_response.json b/metadata-ingestion/tests/integration/powerbi/mock_data/default_mock_response.json new file mode 100644 index 00000000000000..28972fbded9e82 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/mock_data/default_mock_response.json @@ -0,0 +1,558 @@ +{ + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "64ED5CAD-7C10-4684-8180-826122881108", + "isReadOnly": true, + "name": "demo-workspace", + "type": "Workspace", + "state": "Active" + + }, + { + "id": "64ED5CAD-7C22-4684-8180-826122881108", + "isReadOnly": true, + "name": "second-demo-workspace", + "type": "Workspace", + "state": "Active" + + }, + { + "id": "64ED5CAD-7322-4684-8180-826122881108", + "isReadOnly": true, + "name": "Workspace 2", + "type": "Workspace", + "state": "Active" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE", + "isReadOnly": true, + "displayName": "test_dashboard", + "description": "Description of test dashboard", + "embedUrl": "https://localhost/dashboards/embed/1", + "webUrl": "https://localhost/dashboards/web/1" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "7D668CAD-8FFC-4505-9215-655BCA5BEBAE", + "isReadOnly": true, + "displayName": "test_dashboard2", + "embedUrl": "https://localhost/dashboards/embed/1", + "webUrl": "https://localhost/dashboards/web/1" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/reports/5b218778-e7a5-4d73-8187-f10824047715/users": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "identifier": "User1@foo.com", + "displayName": "user1", + "emailAddress": "User1@foo.com", + "datasetUserAccessRight": "ReadWrite", + "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46", + "principalType": "User" + }, + { + "identifier": "User2@foo.com", + "displayName": "user2", + "emailAddress": "User2@foo.com", + "datasetUserAccessRight": "ReadWrite", + "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS", + "principalType": "User" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/users": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "identifier": "User1@foo.com", + "displayName": "user1", + "emailAddress": "User1@foo.com", + "datasetUserAccessRight": "ReadWrite", + "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46", + "principalType": "User" + }, + { + "identifier": "User2@foo.com", + "displayName": "user2", + "emailAddress": "User2@foo.com", + "datasetUserAccessRight": "ReadWrite", + "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS", + "principalType": "User" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/users": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "identifier": "User3@foo.com", + "displayName": "user3", + "emailAddress": "User3@foo.com", + "datasetUserAccessRight": "ReadWrite", + "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46", + "principalType": "User" + }, + { + "identifier": "User4@foo.com", + "displayName": "user4", + "emailAddress": "User4@foo.com", + "datasetUserAccessRight": "ReadWrite", + "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS", + "principalType": "User" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/tiles": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0", + "title": "test_tile", + "embedUrl": "https://localhost/tiles/embed/1", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + { + "id": "23212598-23b5-4980-87cc-5fc0ecd84385", + "title": "yearly_sales", + "embedUrl": "https://localhost/tiles/embed/2", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/tiles": { + "method": "GET", + "status_code": 200, + "json": { + "value": [] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445": { + "method": "GET", + "status_code": 200, + "json": { + "id": "05169CD2-E713-41E6-9600-1D8066D95445", + "name": "library-dataset", + "description": "Library dataset description", + "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445" + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445": { + "method": "GET", + "status_code": 200, + "json": { + "id": "05169CD2-E713-41E6-96AA-1D8066D95445", + "name": "library-dataset", + "description": "Library dataset description", + "webUrl": "http://localhost/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445" + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed": { + "method": "GET", + "status_code": 200, + "json": { + "id": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "name": "hr_pbi_test", + "description": "hr pbi test description", + "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed" + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/datasources": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", + "datasourceType": "PostgreSql", + "connectionDetails": { + "database": "library_db", + "server": "foo" + } + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445/datasources": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", + "datasourceType": "PostgreSql", + "connectionDetails": { + "database": "library_db", + "server": "foo" + } + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4674efd1-603c-4129-8d82-03cf2be05aff": { + "method": "GET", + "status_code": 200, + "json": { + "status": "SUCCEEDED" + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/a674efd1-603c-4129-8d82-03cf2be05aff": { + "method": "GET", + "status_code": 200, + "json": { + "status": "SUCCEEDED" + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff": { + "method": "GET", + "status_code": 200, + "json": { + "workspaces": [ + { + "id": "64ED5CAD-7C10-4684-8180-826122881108", + "name": "demo-workspace", + "type": "Workspace", + "state": "Active", + "datasets": [ + { + "id": "05169CD2-E713-41E6-9600-1D8066D95445", + "endorsementDetails": { + "endorsement": "Promoted" + }, + "name": "test_sf_pbi_test", + "tables": [ + { + "name": "public issue_history", + "source": [ + { + "expression": "dummy" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + }, + { + "name": "SNOWFLAKE_TESTTABLE", + "source": [ + { + "expression": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + }, + { + "name": "snowflake native-query", + "source": [ + { + "expression": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + }, + { + "name": "big-query-with-parameter", + "source": [ + { + "expression": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + }, + { + "name": "snowflake native-query-with-join", + "source": [ + { + "expression": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + }, + { + "name": "job-history", + "source": [ + { + "expression": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + }, + { + "name": "postgres_test_table", + "source": [ + { + "expression": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + } + ] + }, + { + "id": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "name": "hr_pbi_test", + "tables": [ + { + "name": "dbo_book_issue", + "source": [ + { + "expression": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + }, + { + "name": "ms_sql_native_table", + "source": [ + { + "expression": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"" + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3" + } + ] + } + ] + }, + { + "id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff", + "tables": [ + { + "name": "employee_ctc", + "source": [ + { + "expression": "dummy" + } + ] + } + ] + } + ], + "dashboards": [ + { + "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE", + "isReadOnly": true + } + ], + "reports": [ + { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "reportType": "PaginatedReport", + "id": "5b218778-e7a5-4d73-8187-f10824047715", + "name": "SalesMarketing", + "description": "Acryl sales marketing report" + } + ] + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/a674efd1-603c-4129-8d82-03cf2be05aff": { + "method": "GET", + "status_code": 200, + "json": { + "workspaces": [ + { + "id": "64ED5CAD-7C22-4684-8180-826122881108", + "name": "second-demo-workspace", + "type": "Workspace", + "state": "Active", + "datasets": [ + { + "id": "05169CD2-E713-41E6-96AA-1D8066D95445", + "tables": [ + { + "name": "public articles", + "source": [ + { + "expression": "dummy" + } + ] + } + ] + } + ], + "dashboards": [ + { + "id": "7D668CAD-8FFC-4505-9215-655BCA5BEBAE", + "isReadOnly": true + } + ], + "reports": [ + { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "id": "5b218778-e7a5-4d73-8187-f10824047715", + "reportType": "PaginatedReport", + "name": "SalesMarketing", + "description": "Acryl sales marketing report" + } + ] + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "id": "5b218778-e7a5-4d73-8187-f10824047715", + "reportType": "PowerBIReport", + "name": "SalesMarketing", + "description": "Acryl sales marketing report", + "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715", + "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48" + }, + { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "id": "584cf13a-1485-41c2-a514-b1bb66fff163", + "reportType": "PaginatedReport", + "name": "Printable SalesMarketing", + "description": "Acryl sales marketing report", + "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/584cf13a-1485-41c2-a514-b1bb66fff163", + "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=584cf13a-1485-41c2-a514-b1bb66fff163&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715": { + "method": "GET", + "status_code": 200, + "json": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "id": "5b218778-e7a5-4d73-8187-f10824047715", + "reportType": "PaginatedReport", + "name": "SalesMarketing", + "description": "Acryl sales marketing report", + "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715", + "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48" + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715/pages": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "displayName": "Regional Sales Analysis", + "name": "ReportSection", + "order": "0" + }, + { + "displayName": "Geographic Analysis", + "name": "ReportSection1", + "order": "1" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/parameters": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "name": "Parameter - Source", + "type": "Text", + "isRequired": true, + "currentValue": "my-test-project" + }, + { + "name": "My bq project", + "type": "Text", + "isRequired": true, + "currentValue": "gcp_billing" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff": { + "method": "GET", + "status_code": 200, + "json": { + "id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff", + "name": "employee-dataset", + "description": "Employee Management", + "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff" + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/584cf13a-1485-41c2-a514-b1bb66fff163/pages": { + "method": "GET", + "status_code": 400, + "text": "{\"error\":{\"code\":\"InvalidRequest\",\"message\":\"Request is currently not supported for RDL reports\"}}" + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/mock_data/workspace_type_filter.json b/metadata-ingestion/tests/integration/powerbi/mock_data/workspace_type_filter.json new file mode 100644 index 00000000000000..9a6ea36e7ab46a --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/mock_data/workspace_type_filter.json @@ -0,0 +1,76 @@ +{ + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C", + "isReadOnly": true, + "name": "Jane Smith Workspace", + "type": "PersonalGroup", + "state": "Active" + }, + { + "id": "C6B5DBBC-7580-406C-A6BE-72628C28801C", + "isReadOnly": true, + "name": "Sales", + "type": "Workspace", + "state": "Active" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [] + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4278EDC0-85AA-4BF2-B96A-2BC6C82B73C3": { + "method": "GET", + "status_code": 200, + "json": { + "workspaces": [ + { + "id": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C", + "name": "Jane Smith Workspace", + "type": "PersonalGroup", + "state": "Active", + "datasets": [] + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/90E9E256-3D6D-4D38-86C8-6CCCBD8C170C/dashboards": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE", + "isReadOnly": true, + "displayName": "test_dashboard", + "description": "Description of test dashboard", + "embedUrl": "https://localhost/dashboards/embed/1", + "webUrl": "https://localhost/dashboards/web/1" + } + ] + } + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4278EDC0-85AA-4BF2-B96A-2BC6C82B73C3": { + "method": "GET", + "status_code": 200, + "json": { + "status": "SUCCEEDED" + } + }, + "https://api.powerbi.com/v1.0/myorg/groups/90E9E256-3D6D-4D38-86C8-6CCCBD8C170C/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/tiles": { + "method": "GET", + "status_code": 200, + "json": { + "value": [] + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index 43f77b059e41f2..78cf1031074779 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -1,14 +1,17 @@ import datetime +import json import logging import re import sys -from typing import Any, Dict, List, Optional, cast +from pathlib import Path +from typing import Any, Dict, List, Optional, Union, cast from unittest import mock from unittest.mock import MagicMock import pytest from freezegun import freeze_time +from datahub.ingestion.api.source import StructuredLogLevel from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.source.powerbi.config import ( Constant, @@ -71,597 +74,42 @@ def scan_init_response(request, context): "64ED5CAD-7C10-4684-8180-826122881108||64ED5CAD-7C22-4684-8180-826122881108": { "id": "a674efd1-603c-4129-8d82-03cf2be05aff" }, - "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C": { - "id": "4278EDC0-85AA-4BF2-B96A-2BC6C82B73C3" + "A8D655A6-F521-477E-8C22-255018583BF4": { + "id": "62DAF926-0B18-4FF1-982C-2A3EB6B8F0E4" + }, + "C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492": { + "id": "81B02907-E2A3-45C3-B505-3781839C8CAA", }, } return w_id_vs_response[workspace_id] -def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -> None: - override_data = override_data or {} +def read_mock_data(path: Union[Path, str]) -> dict: + with open(path) as p: + return json.load(p) + + +def register_mock_api( + pytestconfig: pytest.Config, request_mock: Any, override_data: Optional[dict] = None +) -> None: + + default_mock_data_path = ( + pytestconfig.rootpath + / "tests/integration/powerbi/mock_data/default_mock_response.json" + ) + api_vs_response = { - "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "id": "64ED5CAD-7C10-4684-8180-826122881108", - "isReadOnly": True, - "name": "demo-workspace", - "type": "Workspace", - }, - { - "id": "64ED5CAD-7C22-4684-8180-826122881108", - "isReadOnly": True, - "name": "second-demo-workspace", - "type": "Workspace", - }, - { - "id": "64ED5CAD-7322-4684-8180-826122881108", - "isReadOnly": True, - "name": "Workspace 2", - "type": "Workspace", - }, - ], - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": { - "method": "GET", - "status_code": 200, - "json": { - "value": [], - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE", - "isReadOnly": True, - "displayName": "test_dashboard", - "description": "Description of test dashboard", - "embedUrl": "https://localhost/dashboards/embed/1", - "webUrl": "https://localhost/dashboards/web/1", - } - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "id": "7D668CAD-8FFC-4505-9215-655BCA5BEBAE", - "isReadOnly": True, - "displayName": "test_dashboard2", - "embedUrl": "https://localhost/dashboards/embed/1", - "webUrl": "https://localhost/dashboards/web/1", - } - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/admin/reports/5b218778-e7a5-4d73-8187-f10824047715/users": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "identifier": "User1@foo.com", - "displayName": "user1", - "emailAddress": "User1@foo.com", - "datasetUserAccessRight": "ReadWrite", - "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46", - "principalType": "User", - }, - { - "identifier": "User2@foo.com", - "displayName": "user2", - "emailAddress": "User2@foo.com", - "datasetUserAccessRight": "ReadWrite", - "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS", - "principalType": "User", - }, - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/users": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "identifier": "User1@foo.com", - "displayName": "user1", - "emailAddress": "User1@foo.com", - "datasetUserAccessRight": "ReadWrite", - "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46", - "principalType": "User", - }, - { - "identifier": "User2@foo.com", - "displayName": "user2", - "emailAddress": "User2@foo.com", - "datasetUserAccessRight": "ReadWrite", - "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS", - "principalType": "User", - }, - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/users": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "identifier": "User3@foo.com", - "displayName": "user3", - "emailAddress": "User3@foo.com", - "datasetUserAccessRight": "ReadWrite", - "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46", - "principalType": "User", - }, - { - "identifier": "User4@foo.com", - "displayName": "user4", - "emailAddress": "User4@foo.com", - "datasetUserAccessRight": "ReadWrite", - "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS", - "principalType": "User", - }, - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/tiles": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "id": "B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0", - "title": "test_tile", - "embedUrl": "https://localhost/tiles/embed/1", - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - }, - { - "id": "23212598-23b5-4980-87cc-5fc0ecd84385", - "title": "yearly_sales", - "embedUrl": "https://localhost/tiles/embed/2", - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - }, - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/90E9E256-3D6D-4D38-86C8-6CCCBD8C170C/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/tiles": { - "method": "GET", - "status_code": 200, - "json": {"value": []}, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/tiles": { - "method": "GET", - "status_code": 200, - "json": {"value": []}, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445": { - "method": "GET", - "status_code": 200, - "json": { - "id": "05169CD2-E713-41E6-9600-1D8066D95445", - "name": "library-dataset", - "description": "Library dataset description", - "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445", - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445": { - "method": "GET", - "status_code": 200, - "json": { - "id": "05169CD2-E713-41E6-96AA-1D8066D95445", - "name": "library-dataset", - "description": "Library dataset description", - "webUrl": "http://localhost/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445", - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed": { - "method": "GET", - "status_code": 200, - "json": { - "id": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "name": "hr_pbi_test", - "description": "hr pbi test description", - "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed", - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/datasources": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", - "datasourceType": "PostgreSql", - "connectionDetails": { - "database": "library_db", - "server": "foo", - }, - }, - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445/datasources": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", - "datasourceType": "PostgreSql", - "connectionDetails": { - "database": "library_db", - "server": "foo", - }, - }, - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4674efd1-603c-4129-8d82-03cf2be05aff": { - "method": "GET", - "status_code": 200, - "json": { - "status": "SUCCEEDED", - }, - }, - "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/a674efd1-603c-4129-8d82-03cf2be05aff": { - "method": "GET", - "status_code": 200, - "json": { - "status": "SUCCEEDED", - }, - }, - "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff": { - "method": "GET", - "status_code": 200, - "json": { - "workspaces": [ - { - "id": "64ED5CAD-7C10-4684-8180-826122881108", - "name": "demo-workspace", - "state": "Active", - "type": "Workspace", - "datasets": [ - { - "id": "05169CD2-E713-41E6-9600-1D8066D95445", - "endorsementDetails": {"endorsement": "Promoted"}, - "name": "test_sf_pbi_test", - "tables": [ - { - "name": "public issue_history", - "source": [ - { - "expression": "dummy", - } - ], - "datasourceUsages": [ - { - "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", - } - ], - }, - { - "name": "SNOWFLAKE_TESTTABLE", - "source": [ - { - "expression": 'let\n Source = Snowflake.Databases("hp123rt5.ap-southeast-2.fakecomputing.com","PBI_TEST_WAREHOUSE_PROD",[Role="PBI_TEST_MEMBER"]),\n PBI_TEST_Database = Source{[Name="PBI_TEST",Kind="Database"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name="TEST",Kind="Schema"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name="TESTTABLE",Kind="Table"]}[Data]\nin\n TESTTABLE_Table', - } - ], - "datasourceUsages": [ - { - "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", - } - ], - }, - { - "name": "snowflake native-query", - "source": [ - { - "expression": 'let\n Source = Value.NativeQuery(Snowflake.Databases("bu20658.ap-southeast-2.snowflakecomputing.com","operations_analytics_warehouse_prod",[Role="OPERATIONS_ANALYTICS_MEMBER"]){[Name="OPERATIONS_ANALYTICS"]}[Data], "SELECT#(lf)concat((UPPER(REPLACE(SELLER,\'-\',\'\'))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,\'-\',\'\'))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4", null, [EnableFolding=true]),\n #"Added Conditional Column" = Table.AddColumn(Source, "SME Units ENT", each if [DEAL_TYPE] = "SME Unit" then [UNIT] else 0),\n #"Added Conditional Column1" = Table.AddColumn(#"Added Conditional Column", "Banklink Units", each if [DEAL_TYPE] = "Banklink" then [UNIT] else 0),\n #"Removed Columns" = Table.RemoveColumns(#"Added Conditional Column1",{"Banklink Units"}),\n #"Added Custom" = Table.AddColumn(#"Removed Columns", "Banklink Units", each if [DEAL_TYPE] = "Banklink" and [SALES_TYPE] = "3 - Upsell"\nthen [UNIT]\n\nelse if [SALES_TYPE] = "Adjusted BL Migration"\nthen [UNIT]\n\nelse 0),\n #"Added Custom1" = Table.AddColumn(#"Added Custom", "SME Units in $ (*$361)", each if [DEAL_TYPE] = "SME Unit" \nand [SALES_TYPE] <> "4 - Renewal"\n then [UNIT] * 361\nelse 0),\n #"Added Custom2" = Table.AddColumn(#"Added Custom1", "Banklink in $ (*$148)", each [Banklink Units] * 148)\nin\n #"Added Custom2"', - } - ], - "datasourceUsages": [ - { - "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", - } - ], - }, - { - "name": "big-query-with-parameter", - "source": [ - { - "expression": 'let\n Source = GoogleBigQuery.Database([BillingProject = #"Parameter - Source"]),\n#"gcp-project" = Source{[Name=#"Parameter - Source"]}[Data],\nuniversal_Schema = #"gcp-project"{[Name="universal",Kind="Schema"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name="D_WH_DATE",Kind="Table"]}[Data],\n#"Filtered Rows" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#"Filtered Rows1" = Table.SelectRows(#"Filtered Rows", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#"Filtered Rows1"', - } - ], - "datasourceUsages": [ - { - "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", - } - ], - }, - { - "name": "snowflake native-query-with-join", - "source": [ - { - "expression": 'let\n Source = Value.NativeQuery(Snowflake.Databases("xaa48144.snowflakecomputing.com","GSL_TEST_WH",[Role="ACCOUNTADMIN"]){[Name="GSL_TEST_DB"]}[Data], "select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, \'mo\')", null, [EnableFolding=true])\nin\n Source', - } - ], - "datasourceUsages": [ - { - "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", - } - ], - }, - { - "name": "job-history", - "source": [ - { - "expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', - } - ], - "datasourceUsages": [ - { - "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", - } - ], - }, - { - "name": "postgres_test_table", - "source": [ - { - "expression": 'let\n Source = PostgreSQL.Database("localhost" , "mics" ),\n public_order_date = Source{[Schema="public",Item="order_date"]}[Data] \n in \n public_order_date', - } - ], - "datasourceUsages": [ - { - "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", - } - ], - }, - ], - }, - { - "id": "ba0130a1-5b03-40de-9535-b34e778ea6ed", - "name": "hr_pbi_test", - "tables": [ - { - "name": "dbo_book_issue", - "source": [ - { - "expression": 'let\n Source = Sql.Database("localhost", "library"),\n dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data]\n in dbo_book_issue', - } - ], - "datasourceUsages": [ - { - "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", - } - ], - }, - { - "name": "ms_sql_native_table", - "source": [ - { - "expression": 'let\n Source = Sql.Database("AUPRDWHDB", "COMMOPSDB", [Query="select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,\'-\',\'\'))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,\'-\',\'\'))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION", CommandTimeout=#duration(0, 1, 30, 0)]),\n #"Changed Type" = Table.TransformColumnTypes(Source,{{"mth_date", type date}}),\n #"Added Custom" = Table.AddColumn(#"Changed Type", "Month", each Date.Month([mth_date])),\n #"Added Custom1" = Table.AddColumn(#"Added Custom", "TPV Opening", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #"Added Custom1"', - } - ], - "datasourceUsages": [ - { - "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", - } - ], - }, - ], - }, - { - "id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff", - "tables": [ - { - "name": "employee_ctc", - "source": [ - { - "expression": "dummy", - } - ], - } - ], - }, - ], - "dashboards": [ - { - "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE", - "isReadOnly": True, - } - ], - "reports": [ - { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "id": "5b218778-e7a5-4d73-8187-f10824047715", - "reportType": "PaginatedReport", - "name": "SalesMarketing", - "description": "Acryl sales marketing report", - } - ], - }, - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/a674efd1-603c-4129-8d82-03cf2be05aff": { - "method": "GET", - "status_code": 200, - "json": { - "workspaces": [ - { - "id": "64ED5CAD-7C22-4684-8180-826122881108", - "name": "second-demo-workspace", - "type": "Workspace", - "state": "Active", - "datasets": [ - { - "id": "05169CD2-E713-41E6-96AA-1D8066D95445", - "tables": [ - { - "name": "public articles", - "source": [ - { - "expression": "dummy", - } - ], - } - ], - } - ], - "dashboards": [ - { - "id": "7D668CAD-8FFC-4505-9215-655BCA5BEBAE", - "isReadOnly": True, - } - ], - "reports": [ - { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "id": "5b218778-e7a5-4d73-8187-f10824047715", - "reportType": "PowerBIReport", - "name": "SalesMarketing", - "description": "Acryl sales marketing report", - }, - { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "id": "584cf13a-1485-41c2-a514-b1bb66fff163", - "reportType": "PaginatedReport", - "name": "SalesMarketing", - "description": "Acryl sales marketing report", - }, - ], - }, - ] - }, - }, "https://api.powerbi.com/v1.0/myorg/admin/workspaces/getInfo": { "method": "POST", "status_code": 200, "json": scan_init_response, }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "id": "5b218778-e7a5-4d73-8187-f10824047715", - "reportType": "PowerBIReport", - "name": "SalesMarketing", - "description": "Acryl sales marketing report", - "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715", - "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48", - }, - { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "id": "584cf13a-1485-41c2-a514-b1bb66fff163", - "reportType": "PaginatedReport", - "name": "Printable SalesMarketing", - "description": "Acryl sales marketing report", - "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/584cf13a-1485-41c2-a514-b1bb66fff163", - "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=584cf13a-1485-41c2-a514-b1bb66fff163&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48", - }, - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715": { - "method": "GET", - "status_code": 200, - "json": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "id": "5b218778-e7a5-4d73-8187-f10824047715", - "reportType": "PowerBIReport", - "name": "SalesMarketing", - "description": "Acryl sales marketing report", - "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715", - "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48", - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/584cf13a-1485-41c2-a514-b1bb66fff163": { - "method": "GET", - "status_code": 200, - "json": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", - "id": "584cf13a-1485-41c2-a514-b1bb66fff163", - "reportType": "PaginatedReport", - "name": "Printable SalesMarketing", - "description": "Acryl sales marketing report", - "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/584cf13a-1485-41c2-a514-b1bb66fff163", - "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=584cf13a-1485-41c2-a514-b1bb66fff163&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48", - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715/pages": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "displayName": "Regional Sales Analysis", - "name": "ReportSection", - "order": "0", - }, - { - "displayName": "Geographic Analysis", - "name": "ReportSection1", - "order": "1", - }, - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/584cf13a-1485-41c2-a514-b1bb66fff163/pages": { - "method": "GET", - "status_code": 400, # Pages API is not supported for PaginatedReport - "text": '{"error":{"code":"InvalidRequest","message":"Request is currently not supported for RDL reports"}}', - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/parameters": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "name": "Parameter - Source", - "type": "Text", - "isRequired": True, - "currentValue": "my-test-project", - }, - { - "name": "My bq project", - "type": "Text", - "isRequired": True, - "currentValue": "gcp_billing", - }, - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff": { - "method": "GET", - "status_code": 200, - "json": { - "id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff", - "name": "employee-dataset", - "description": "Employee Management", - "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff", - }, - }, } - api_vs_response.update(override_data) + api_vs_response.update(read_mock_data(default_mock_data_path)) + + api_vs_response.update(override_data or {}) for url in api_vs_response.keys(): request_mock.register_uri( @@ -708,7 +156,7 @@ def test_powerbi_ingest( test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) pipeline = Pipeline.create( { @@ -755,75 +203,11 @@ def test_powerbi_workspace_type_filter( register_mock_api( request_mock=requests_mock, - override_data={ - "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "id": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C", - "isReadOnly": True, - "name": "Jane Smith Workspace", - "type": "PersonalGroup", - "state": "Active", - }, - { - "id": "C6B5DBBC-7580-406C-A6BE-72628C28801C", - "isReadOnly": True, - "name": "Sales", - "type": "Workspace", - "state": "Active", - }, - ], - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": { - "method": "GET", - "status_code": 200, - "json": { - "value": [], - }, - }, - "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4278EDC0-85AA-4BF2-B96A-2BC6C82B73C3": { - "method": "GET", - "status_code": 200, - "json": { - "workspaces": [ - { - "id": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C", - "name": "Jane Smith Workspace", - "type": "PersonalGroup", - "state": "Active", - "datasets": [], - }, - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/groups/90E9E256-3D6D-4D38-86C8-6CCCBD8C170C/dashboards": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE", - "isReadOnly": True, - "displayName": "test_dashboard", - "description": "Description of test dashboard", - "embedUrl": "https://localhost/dashboards/embed/1", - "webUrl": "https://localhost/dashboards/web/1", - } - ] - }, - }, - "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4278EDC0-85AA-4BF2-B96A-2BC6C82B73C3": { - "method": "GET", - "status_code": 200, - "json": { - "status": "SUCCEEDED", - }, - }, - }, + pytestconfig=pytestconfig, + override_data=read_mock_data( + pytestconfig.rootpath + / "tests/integration/powerbi/mock_data/workspace_type_filter.json" + ), ) default_config: dict = default_source_config() @@ -878,7 +262,7 @@ def test_powerbi_ingest_patch_disabled( test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) pipeline = Pipeline.create( { @@ -945,7 +329,7 @@ def test_powerbi_platform_instance_ingest( test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) output_path: str = f"{tmp_path}/powerbi_platform_instance_mces.json" @@ -991,7 +375,7 @@ def test_powerbi_ingest_urn_lower_case( ) -> None: test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) pipeline = Pipeline.create( { @@ -1038,7 +422,7 @@ def test_override_ownership( ) -> None: test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) pipeline = Pipeline.create( { @@ -1083,7 +467,7 @@ def test_scan_all_workspaces( test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) pipeline = Pipeline.create( { @@ -1135,7 +519,7 @@ def test_extract_reports( test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) pipeline = Pipeline.create( { @@ -1181,7 +565,7 @@ def test_extract_lineage( test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) pipeline = Pipeline.create( { @@ -1233,7 +617,7 @@ def test_extract_endorsements( ) -> None: test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) pipeline = Pipeline.create( { @@ -1281,6 +665,7 @@ def test_admin_access_is_not_allowed( test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api( + pytestconfig=pytestconfig, request_mock=requests_mock, override_data={ "https://api.powerbi.com/v1.0/myorg/admin/workspaces/getInfo": { @@ -1342,7 +727,7 @@ def test_workspace_container( test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) pipeline = Pipeline.create( { @@ -1389,7 +774,7 @@ def test_access_token_expiry_with_long_expiry( ) -> None: enable_logging() - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) pipeline = Pipeline.create( { @@ -1431,7 +816,7 @@ def test_access_token_expiry_with_short_expiry( ) -> None: enable_logging() - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) pipeline = Pipeline.create( { @@ -1486,7 +871,7 @@ def test_dataset_type_mapping_should_set_to_all( """ Here we don't need to run the pipeline. We need to verify dataset_type_mapping is set to default dataplatform """ - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) new_config: dict = {**default_source_config()} @@ -1523,7 +908,7 @@ def test_dataset_type_mapping_error( Here we don't need to run the pipeline. We need to verify if both dataset_type_mapping and server_to_platform_instance are set then value error should get raised """ - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) with pytest.raises(Exception, match=r"dataset_type_mapping is deprecated"): Pipeline.create( @@ -1578,7 +963,7 @@ def test_server_to_platform_map( "localhost:1521": {"platform_instance": "oracle-sales-instance", "env": "PROD"}, } - register_mock_api(request_mock=requests_mock) + register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) output_path: str = f"{tmp_path}/powerbi_server_to_platform_instance_mces.json" @@ -1684,6 +1069,7 @@ def validate_pipeline(pipeline: Pipeline) -> None: ], users=[], tags=[], + dataset_id=report[Constant.DATASET_ID], dataset=mock_workspace.datasets.get(report[Constant.DATASET_ID]), ) for report in mock_reports @@ -1711,6 +1097,7 @@ def test_reports_with_failed_page_request( Test that all reports are fetched even if a single page request fails """ register_mock_api( + pytestconfig=pytestconfig, request_mock=requests_mock, override_data={ "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports": { @@ -1832,6 +1219,7 @@ def test_independent_datasets_extraction( test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api( + pytestconfig=pytestconfig, request_mock=requests_mock, override_data={ "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { @@ -1935,6 +1323,7 @@ def test_cll_extraction( test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api( + pytestconfig=pytestconfig, request_mock=requests_mock, ) @@ -1989,6 +1378,7 @@ def test_cll_extraction_flags( ) -> None: register_mock_api( + pytestconfig=pytestconfig, request_mock=requests_mock, ) @@ -2017,3 +1407,86 @@ def test_cll_extraction_flags( }, } ) + + +@freeze_time(FROZEN_TIME) +@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) +@pytest.mark.integration +def test_powerbi_cross_workspace_reference_info_message( + mock_msal: MagicMock, + pytestconfig: pytest.Config, + tmp_path: str, + mock_time: datetime.datetime, + requests_mock: Any, +) -> None: + enable_logging() + + register_mock_api( + pytestconfig=pytestconfig, + request_mock=requests_mock, + override_data=read_mock_data( + path=pytestconfig.rootpath + / "tests/integration/powerbi/mock_data/cross_workspace_mock_response.json" + ), + ) + + config = default_source_config() + + del config["workspace_id"] + + config["workspace_id_pattern"] = { + "allow": [ + "A8D655A6-F521-477E-8C22-255018583BF4", + "C5DA6EA8-625E-4AB1-90B6-CAEA0BF9F492", + ] + } + + config["include_workspace_name_in_dataset_urn"] = True + + pipeline = Pipeline.create( + { + "run_id": "powerbi-test", + "source": { + "type": "powerbi", + "config": { + **config, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/powerbi_mces.json", + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + + assert isinstance(pipeline.source, PowerBiDashboardSource) # to silent the lint + + info_entries: dict = pipeline.source.reporter._structured_logs._entries.get( + StructuredLogLevel.INFO, {} + ) # type :ignore + + is_entry_present: bool = False + # Printing INFO entries + for key, entry in info_entries.items(): + if entry.title == "Missing Lineage For Tile": + is_entry_present = True + break + + assert ( + is_entry_present + ), 'Info message "Missing Lineage For Tile" should be present in reporter' + + test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" + + golden_file = "golden_test_cross_workspace_dataset.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=f"{tmp_path}/powerbi_mces.json", + golden_path=f"{test_resources_dir}/{golden_file}", + ) From 4be66458de67cca608377bc426b449fee8e32453 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 16 Oct 2024 10:45:03 -0500 Subject: [PATCH 26/40] docs(custom-plugins): add overview image (#11634) --- metadata-models-custom/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/metadata-models-custom/README.md b/metadata-models-custom/README.md index 1d26251bc13c91..83917530a41d0b 100644 --- a/metadata-models-custom/README.md +++ b/metadata-models-custom/README.md @@ -204,6 +204,19 @@ that were loaded for debugging purposes. } ``` +#### Custom Plugin Ecosystem Overview + +The following diagram shows the overall picture of the various validators, mutators, and side effects shown within +the context of typical read/write operations within DataHub. Each component is discussed in further detail in the +sections below. + +

+ +

+ +In the diagram above, the circles represent Aspects (custom aspects or standard). As the Aspects progress they can be mutated/changed, +rejected, or additional aspects can be generated by side effects. + #### Custom Validators Custom aspects might require that instances of those aspects adhere to specific conditions or rules. These conditions could vary wildly depending on the use case however they could be as simple From 5db78c6d8affde40bbaf7a6d569b3bced793aba3 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:02:58 -0500 Subject: [PATCH 27/40] fix(ci): fix build and test workflow (#11644) --- .github/workflows/build-and-test.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 52148ef1b91f95..7d6df187952197 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -67,17 +67,16 @@ jobs: timezoneLinux: ${{ matrix.timezone }} - name: Check out the repo uses: acryldata/sane-checkout-action@v3 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: pip - name: Set up JDK 17 uses: actions/setup-java@v4 with: distribution: "zulu" java-version: 17 - uses: gradle/actions/setup-gradle@v3 - - uses: actions/setup-python@v5 - if: ${{ needs.setup.outputs.ingestion_change == 'true' }} - with: - python-version: "3.10" - cache: pip - name: Gradle build (and test) for NOT metadata ingestion if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }} run: | From d34717fd82eea5592b25cf3840fcea0a7a7acc04 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 16 Oct 2024 13:50:33 -0700 Subject: [PATCH 28/40] fix(ingest): remove default value from DatahubClientConfig.server (#11570) --- docs/how/updating-datahub.md | 4 +- .../client/dagster_generator.py | 7 - .../sensors/datahub_sensors.py | 16 +- .../api/entities/dataproduct/dataproduct.py | 2 +- .../src/datahub/ingestion/graph/config.py | 2 +- ...atahub_ingestion_checkpointing_provider.py | 16 +- metadata-ingestion/tests/conftest.py | 5 +- .../test_business_glossary.py | 12 +- .../tests/test_helpers/state_helpers.py | 10 +- .../tests/unit/graph/test_client.py | 2 +- .../state/test_redundant_run_skip_handler.py | 10 +- .../unit/stateful_ingestion/test_configs.py | 39 +-- .../tests/unit/test_glue_source.py | 18 +- .../tests/unit/test_transform_dataset.py | 254 +++++++++--------- smoke-test/pytest.ini | 3 + smoke-test/tests/test_stateful_ingestion.py | 4 - 16 files changed, 204 insertions(+), 200 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 5b4769ed30e3ed..8911d282f86bbe 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -40,7 +40,9 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #11484 - Rest API authorization enabled by default - #10472 - `SANDBOX` added as a FabricType. No rollbacks allowed once metadata with this fabric type is added without manual cleanups in databases. - #11619 - schema field/column paths can no longer be empty strings -- #11619 - schema field/column paths can no longer be duplicated within the schema +- #11619 - schema field/column paths can no longer be duplicated within the schema +- #11570 - The `DatahubClientConfig`'s server field no longer defaults to `http://localhost:8080`. Be sure to explicitly set this. +- #11570 - If a `datahub_api` is explicitly passed to a stateful ingestion config provider, it will be used. We previously ignored it if the pipeline context also had a graph object. ### Potential Downtime diff --git a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py index df9d0fc423fcfa..a2cf159dd12f6e 100644 --- a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py +++ b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py @@ -78,12 +78,6 @@ class Constant: # Default config constants DEFAULT_DATAHUB_REST_URL = "http://localhost:8080" - # Environment variable contants - DATAHUB_REST_URL = "DATAHUB_REST_URL" - DATAHUB_ENV = "DATAHUB_ENV" - DATAHUB_PLATFORM_INSTANCE = "DATAHUB_PLATFORM_INSTANCE" - DAGSTER_UI_URL = "DAGSTER_UI_URL" - # Datahub inputs/outputs constant DATAHUB_INPUTS = "datahub.inputs" DATAHUB_OUTPUTS = "datahub.outputs" @@ -154,7 +148,6 @@ class DatasetLineage(NamedTuple): class DatahubDagsterSourceConfig(DatasetSourceConfigMixin): datahub_client_config: DatahubClientConfig = pydantic.Field( - default=DatahubClientConfig(), description="Datahub client config", ) diff --git a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py index 4633014222d054..ebb2c82d952b1f 100644 --- a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py +++ b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py @@ -1,5 +1,6 @@ import os import traceback +import warnings from collections import defaultdict from types import ModuleType from typing import Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, Union @@ -38,7 +39,7 @@ from dagster._core.events import DagsterEventType, HandledOutputData, LoadedInputData from dagster._core.execution.stats import RunStepKeyStatsSnapshot from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.ingestion.graph.client import DataHubGraph +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph from datahub.metadata.schema_classes import SubTypesClass from datahub.sql_parsing.sqlglot_lineage import ( SqlParsingResult, @@ -47,6 +48,7 @@ from datahub.utilities.urns.dataset_urn import DatasetUrn from datahub_dagster_plugin.client.dagster_generator import ( + Constant, DagsterEnvironment, DagsterGenerator, DatahubDagsterSourceConfig, @@ -182,7 +184,17 @@ def __init__( if config: self.config = config else: - self.config = DatahubDagsterSourceConfig() + # This is a temporary warning for backwards compatibility. Eventually, we'll remove this + # branch and make the config required. + warnings.warn( + "Using the default DataHub client config is deprecated. Pass in a config object explicitly.", + stacklevel=2, + ) + self.config = DatahubDagsterSourceConfig( + datahub_client_config=DatahubClientConfig( + server=Constant.DEFAULT_DATAHUB_REST_URL + ) + ) self.graph = DataHubGraph( self.config.datahub_client_config, ) diff --git a/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py b/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py index 8f58fa469a7d96..2097922c151366 100644 --- a/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py +++ b/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py @@ -117,7 +117,7 @@ class DataProduct(ConfigModel): @pydantic.validator("assets", each_item=True) def assets_must_be_urns(cls, v: str) -> str: try: - Urn.create_from_string(v) + Urn.from_string(v) except Exception as e: raise ValueError(f"asset {v} is not an urn: {e}") from e diff --git a/metadata-ingestion/src/datahub/ingestion/graph/config.py b/metadata-ingestion/src/datahub/ingestion/graph/config.py index cf0ec45b71458c..5f269e14e1a4af 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/config.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/config.py @@ -8,7 +8,7 @@ class DatahubClientConfig(ConfigModel): # TODO: Having a default for the server doesn't make a ton of sense. This should be handled # by callers / the CLI, but the actual client should not have any magic. - server: str = "http://localhost:8080" + server: str token: Optional[str] = None timeout_sec: Optional[int] = None retry_status_codes: Optional[List[int]] = None diff --git a/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py b/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py index 442abb3aaf4cf8..8f4a53ffc3ed58 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py @@ -17,7 +17,7 @@ class DatahubIngestionStateProviderConfig(IngestionCheckpointingProviderConfig): - datahub_api: DatahubClientConfig = DatahubClientConfig() + datahub_api: Optional[DatahubClientConfig] = None class DatahubIngestionCheckpointingProvider(IngestionCheckpointingProviderBase): @@ -31,8 +31,8 @@ def __init__( self.graph = graph if not self._is_server_stateful_ingestion_capable(): raise ConfigurationError( - "Datahub server is not capable of supporting stateful ingestion." - " Please consider upgrading to the latest server version to use this feature." + "Datahub server is not capable of supporting stateful ingestion. " + "Please consider upgrading to the latest server version to use this feature." ) @classmethod @@ -40,11 +40,15 @@ def create( cls, config_dict: Dict[str, Any], ctx: PipelineContext ) -> "DatahubIngestionCheckpointingProvider": config = DatahubIngestionStateProviderConfig.parse_obj(config_dict) - if ctx.graph: - # Use the pipeline-level graph if set + if config.datahub_api is not None: + return cls(DataHubGraph(config.datahub_api)) + elif ctx.graph: + # Use the pipeline-level graph if set. return cls(ctx.graph) else: - return cls(DataHubGraph(config.datahub_api)) + raise ValueError( + "A graph instance is required. Either pass one in the pipeline context, or set it explicitly in the stateful ingestion provider config." + ) def _is_server_stateful_ingestion_capable(self) -> bool: server_config = self.graph.get_config() if self.graph else None diff --git a/metadata-ingestion/tests/conftest.py b/metadata-ingestion/tests/conftest.py index d0716e34ee2b6f..db025e7f806c06 100644 --- a/metadata-ingestion/tests/conftest.py +++ b/metadata-ingestion/tests/conftest.py @@ -25,7 +25,10 @@ docker_compose_command, docker_compose_runner, ) -from tests.test_helpers.state_helpers import mock_datahub_graph # noqa: F401,E402 +from tests.test_helpers.state_helpers import ( # noqa: F401,E402 + mock_datahub_graph, + mock_datahub_graph_instance, +) try: # See https://github.com/spulec/freezegun/issues/98#issuecomment-590553475. diff --git a/metadata-ingestion/tests/integration/business-glossary/test_business_glossary.py b/metadata-ingestion/tests/integration/business-glossary/test_business_glossary.py index b6e1aca4d4fedb..73b90df65c04fe 100644 --- a/metadata-ingestion/tests/integration/business-glossary/test_business_glossary.py +++ b/metadata-ingestion/tests/integration/business-glossary/test_business_glossary.py @@ -3,7 +3,6 @@ import pytest from freezegun import freeze_time -from datahub.ingestion.graph.client import DatahubClientConfig from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.source.metadata import business_glossary from tests.test_helpers import mce_helpers @@ -41,7 +40,12 @@ def get_default_recipe( @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_glossary_ingest( - mock_datahub_graph, pytestconfig, tmp_path, mock_time, enable_auto_id, golden_file + mock_datahub_graph_instance, + pytestconfig, + tmp_path, + mock_time, + enable_auto_id, + golden_file, ): test_resources_dir = pytestconfig.rootpath / "tests/integration/business-glossary" @@ -55,9 +59,7 @@ def test_glossary_ingest( enable_auto_id=enable_auto_id, ) ) - pipeline.ctx.graph = mock_datahub_graph( - DatahubClientConfig() - ) # Mock to resolve domain + pipeline.ctx.graph = mock_datahub_graph_instance pipeline.run() pipeline.raise_from_status() diff --git a/metadata-ingestion/tests/test_helpers/state_helpers.py b/metadata-ingestion/tests/test_helpers/state_helpers.py index 76f2ab283790fb..f68aef742fc730 100644 --- a/metadata-ingestion/tests/test_helpers/state_helpers.py +++ b/metadata-ingestion/tests/test_helpers/state_helpers.py @@ -1,5 +1,5 @@ import types -from typing import Any, Dict, Optional, Type, cast +from typing import Any, Callable, Dict, Optional, Type, cast from unittest.mock import MagicMock, create_autospec import pytest @@ -10,6 +10,7 @@ IngestionCheckpointingProviderBase, ) from datahub.ingestion.graph.client import DataHubGraph +from datahub.ingestion.graph.config import DatahubClientConfig from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.source.state.checkpoint import Checkpoint from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState @@ -101,6 +102,13 @@ def monkey_patch_get_latest_timeseries_value( return mock_datahub_graph_ctx.mock_graph +@pytest.fixture +def mock_datahub_graph_instance( + mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph] +) -> DataHubGraph: + return mock_datahub_graph(DatahubClientConfig(server="http://fake.domain.local")) + + def get_current_checkpoint_from_pipeline( pipeline: Pipeline, ) -> Optional[Checkpoint[GenericCheckpointState]]: diff --git a/metadata-ingestion/tests/unit/graph/test_client.py b/metadata-ingestion/tests/unit/graph/test_client.py index faed1f51b29aa1..16795ef8c7f814 100644 --- a/metadata-ingestion/tests/unit/graph/test_client.py +++ b/metadata-ingestion/tests/unit/graph/test_client.py @@ -11,7 +11,7 @@ @patch("datahub.emitter.rest_emitter.DataHubRestEmitter.test_connection") def test_get_aspect(mock_test_connection): mock_test_connection.return_value = {} - graph = DataHubGraph(DatahubClientConfig()) + graph = DataHubGraph(DatahubClientConfig(server="http://fake-domain.local")) user_urn = "urn:li:corpuser:foo" with patch("requests.Session.get") as mock_get: mock_response = Mock() diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py index be6efd3e121ff1..85c86f8d205d9a 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py @@ -12,17 +12,11 @@ from datahub.ingestion.source.state.stale_entity_removal_handler import ( StatefulStaleMetadataRemovalConfig, ) -from datahub.ingestion.source.state.stateful_ingestion_base import ( - DynamicTypedStateProviderConfig, -) from datahub.ingestion.source.state.usage_common_state import ( BaseTimeWindowCheckpointState, ) from datahub.utilities.time import datetime_to_ts_millis -GMS_PORT = 8080 -GMS_SERVER = f"http://localhost:{GMS_PORT}" - @pytest.fixture def stateful_source(mock_datahub_graph: DataHubGraph) -> Iterable[SnowflakeV2Source]: @@ -39,9 +33,7 @@ def stateful_source(mock_datahub_graph: DataHubGraph) -> Iterable[SnowflakeV2Sou password="TST_PWD", stateful_ingestion=StatefulStaleMetadataRemovalConfig( enabled=True, - state_provider=DynamicTypedStateProviderConfig( - type="datahub", config={"datahub_api": {"server": GMS_SERVER}} - ), + # Uses the graph from the pipeline context. ), ) diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py b/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py index 0e6d60e3440b20..ba40962866f8cc 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py +++ b/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py @@ -14,16 +14,12 @@ ) # 0. Common client configs. -datahub_client_configs: Dict[str, Any] = { - "full": { - "server": "http://localhost:8080", - "token": "dummy_test_tok", - "timeout_sec": 10, - "extra_headers": {}, - "max_threads": 10, - }, - "simple": {}, - "default": {}, +datahub_client_full_config = { + "server": "http://localhost:8080", + "token": "dummy_test_tok", + "timeout_sec": 10, + "extra_headers": {}, + "max_threads": 10, } @@ -41,7 +37,7 @@ "checkpointing_valid_full_config": ( DatahubIngestionStateProviderConfig, { - "datahub_api": datahub_client_configs["full"], + "datahub_api": datahub_client_full_config, }, DatahubIngestionStateProviderConfig( # This test verifies that the max_threads arg is ignored. @@ -57,27 +53,14 @@ ), False, ), - # Simple config - "checkpointing_valid_simple_config": ( - DatahubIngestionStateProviderConfig, - { - "datahub_api": datahub_client_configs["simple"], - }, - DatahubIngestionStateProviderConfig( - datahub_api=DatahubClientConfig( - server="http://localhost:8080", - ), - ), - False, - ), # Default "checkpointing_default": ( DatahubIngestionStateProviderConfig, { - "datahub_api": datahub_client_configs["default"], + "datahub_api": None, }, DatahubIngestionStateProviderConfig( - datahub_api=DatahubClientConfig(), + datahub_api=None, ), False, ), @@ -102,7 +85,7 @@ "max_checkpoint_state_size": 1024, "state_provider": { "type": "datahub", - "config": datahub_client_configs["full"], + "config": datahub_client_full_config, }, "ignore_old_state": True, "ignore_new_state": True, @@ -114,7 +97,7 @@ ignore_new_state=True, state_provider=DynamicTypedStateProviderConfig( type="datahub", - config=datahub_client_configs["full"], + config=datahub_client_full_config, ), ), False, diff --git a/metadata-ingestion/tests/unit/test_glue_source.py b/metadata-ingestion/tests/unit/test_glue_source.py index 45b9899eacaa77..eb1e7f3fe41d9a 100644 --- a/metadata-ingestion/tests/unit/test_glue_source.py +++ b/metadata-ingestion/tests/unit/test_glue_source.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from typing import Any, Callable, Dict, Optional, Tuple, Type, cast +from typing import Any, Dict, Optional, Tuple, Type, cast from unittest.mock import patch import pydantic @@ -11,7 +11,7 @@ import datahub.metadata.schema_classes as models from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.extractor.schema_util import avro_schema_to_mce_fields -from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph +from datahub.ingestion.graph.client import DataHubGraph from datahub.ingestion.sink.file import write_metadata_file from datahub.ingestion.source.aws.glue import ( GlueProfilingConfig, @@ -74,7 +74,7 @@ def glue_source( platform_instance: Optional[str] = None, - mock_datahub_graph: Optional[Callable[[DatahubClientConfig], DataHubGraph]] = None, + mock_datahub_graph_instance: Optional[DataHubGraph] = None, use_s3_bucket_tags: bool = True, use_s3_object_tags: bool = True, extract_delta_schema_from_parameters: bool = False, @@ -83,8 +83,8 @@ def glue_source( extract_transforms: bool = True, ) -> GlueSource: pipeline_context = PipelineContext(run_id="glue-source-tes") - if mock_datahub_graph: - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + if mock_datahub_graph_instance: + pipeline_context.graph = mock_datahub_graph_instance return GlueSource( ctx=pipeline_context, config=GlueSourceConfig( @@ -493,14 +493,14 @@ def test_glue_with_malformed_delta_schema_ingest( def test_glue_ingest_include_table_lineage( tmp_path: Path, pytestconfig: PytestConfig, - mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph], + mock_datahub_graph_instance: DataHubGraph, platform_instance: str, mce_file: str, mce_golden_file: str, ) -> None: glue_source_instance = glue_source( platform_instance=platform_instance, - mock_datahub_graph=mock_datahub_graph, + mock_datahub_graph_instance=mock_datahub_graph_instance, emit_s3_lineage=True, ) @@ -589,14 +589,14 @@ def test_glue_ingest_include_table_lineage( def test_glue_ingest_include_column_lineage( tmp_path: Path, pytestconfig: PytestConfig, - mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph], + mock_datahub_graph_instance: DataHubGraph, platform_instance: str, mce_file: str, mce_golden_file: str, ) -> None: glue_source_instance = glue_source( platform_instance=platform_instance, - mock_datahub_graph=mock_datahub_graph, + mock_datahub_graph_instance=mock_datahub_graph_instance, emit_s3_lineage=True, include_column_lineage=True, use_s3_bucket_tags=False, diff --git a/metadata-ingestion/tests/unit/test_transform_dataset.py b/metadata-ingestion/tests/unit/test_transform_dataset.py index 46c6390b184d36..2e2e85b5d18113 100644 --- a/metadata-ingestion/tests/unit/test_transform_dataset.py +++ b/metadata-ingestion/tests/unit/test_transform_dataset.py @@ -1,17 +1,7 @@ import json import re from datetime import datetime, timezone -from typing import ( - Any, - Callable, - Dict, - List, - MutableSequence, - Optional, - Type, - Union, - cast, -) +from typing import Any, Dict, List, MutableSequence, Optional, Type, Union, cast from unittest import mock from uuid import uuid4 @@ -24,7 +14,7 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api import workunit from datahub.ingestion.api.common import EndOfStream, PipelineContext, RecordEnvelope -from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph +from datahub.ingestion.graph.client import DataHubGraph from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.transformer.add_dataset_browse_path import ( AddDatasetBrowsePathTransformer, @@ -1106,7 +1096,7 @@ def test_pattern_dataset_ownership_with_invalid_type_transformation(mock_time): def test_pattern_container_and_dataset_ownership_transformation( - mock_time, mock_datahub_graph + mock_time, mock_datahub_graph_instance ): def fake_get_aspect( entity_urn: str, @@ -1127,7 +1117,7 @@ def fake_get_aspect( pipeline_context = PipelineContext( run_id="test_pattern_container_and_dataset_ownership_transformation" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance pipeline_context.graph.get_aspect = fake_get_aspect # type: ignore # No owner aspect for the first dataset @@ -1240,7 +1230,7 @@ def fake_get_aspect( def test_pattern_container_and_dataset_ownership_with_no_container( - mock_time, mock_datahub_graph + mock_time, mock_datahub_graph_instance ): def fake_get_aspect( entity_urn: str, @@ -1252,7 +1242,7 @@ def fake_get_aspect( pipeline_context = PipelineContext( run_id="test_pattern_container_and_dataset_ownership_with_no_container" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance pipeline_context.graph.get_aspect = fake_get_aspect # type: ignore # No owner aspect for the first dataset @@ -1357,7 +1347,7 @@ def fake_get_aspect( def test_pattern_container_and_dataset_ownership_with_no_match( - mock_time, mock_datahub_graph + mock_time, mock_datahub_graph_instance ): def fake_get_aspect( entity_urn: str, @@ -1375,7 +1365,7 @@ def fake_get_aspect( pipeline_context = PipelineContext( run_id="test_pattern_container_and_dataset_ownership_with_no_match" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance pipeline_context.graph.get_aspect = fake_get_aspect # type: ignore # No owner aspect for the first dataset @@ -1598,10 +1588,10 @@ def run_simple_add_dataset_properties_transformer_semantics( semantics: TransformerSemantics, new_properties: dict, server_properties: dict, - mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph], + mock_datahub_graph_instance: DataHubGraph, ) -> List[RecordEnvelope]: pipeline_context = PipelineContext(run_id="test_pattern_dataset_schema_terms") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # fake the server response def fake_dataset_properties(entity_urn: str) -> models.DatasetPropertiesClass: @@ -1624,7 +1614,7 @@ def fake_dataset_properties(entity_urn: str) -> models.DatasetPropertiesClass: return output -def test_simple_add_dataset_properties_overwrite(mock_datahub_graph): +def test_simple_add_dataset_properties_overwrite(mock_datahub_graph_instance): new_properties = {"new-simple-property": "new-value"} server_properties = {"p1": "value1"} @@ -1632,7 +1622,7 @@ def test_simple_add_dataset_properties_overwrite(mock_datahub_graph): semantics=TransformerSemantics.OVERWRITE, new_properties=new_properties, server_properties=server_properties, - mock_datahub_graph=mock_datahub_graph, + mock_datahub_graph_instance=mock_datahub_graph_instance, ) assert len(output) == 2 @@ -1648,7 +1638,7 @@ def test_simple_add_dataset_properties_overwrite(mock_datahub_graph): } -def test_simple_add_dataset_properties_patch(mock_datahub_graph): +def test_simple_add_dataset_properties_patch(mock_datahub_graph_instance): new_properties = {"new-simple-property": "new-value"} server_properties = {"p1": "value1"} @@ -1656,7 +1646,7 @@ def test_simple_add_dataset_properties_patch(mock_datahub_graph): semantics=TransformerSemantics.PATCH, new_properties=new_properties, server_properties=server_properties, - mock_datahub_graph=mock_datahub_graph, + mock_datahub_graph_instance=mock_datahub_graph_instance, ) assert len(output) == 2 @@ -2334,24 +2324,24 @@ def run_container_transformer_pipeline( return outputs -def test_simple_add_dataset_domain_aspect_name(mock_datahub_graph): +def test_simple_add_dataset_domain_aspect_name(mock_datahub_graph_instance): pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance transformer = SimpleAddDatasetDomain.create({"domains": []}, pipeline_context) assert transformer.aspect_name() == models.DomainsClass.ASPECT_NAME -def test_simple_add_dataset_domain(mock_datahub_graph): +def test_simple_add_dataset_domain(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, @@ -2372,14 +2362,14 @@ def test_simple_add_dataset_domain(mock_datahub_graph): assert acryl_domain in transformed_aspect.domains -def test_simple_add_dataset_domain_mce_support(mock_datahub_graph): +def test_simple_add_dataset_domain_mce_support(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, @@ -2403,14 +2393,14 @@ def test_simple_add_dataset_domain_mce_support(mock_datahub_graph): assert acryl_domain in transformed_aspect.domains -def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph): +def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, @@ -2431,13 +2421,13 @@ def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph): assert acryl_domain in transformed_aspect.domains -def test_simple_add_dataset_domain_semantics_overwrite(mock_datahub_graph): +def test_simple_add_dataset_domain_semantics_overwrite(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") server_domain = builder.make_domain_urn("test.io") pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # Return fake aspect to simulate server behaviour def fake_get_domain(entity_urn: str) -> models.DomainsClass: @@ -2469,14 +2459,14 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: def test_simple_add_dataset_domain_semantics_patch( - pytestconfig, tmp_path, mock_time, mock_datahub_graph + pytestconfig, tmp_path, mock_time, mock_datahub_graph_instance ): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") server_domain = builder.make_domain_urn("test.io") pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # Return fake aspect to simulate server behaviour def fake_get_domain(entity_urn: str) -> models.DomainsClass: @@ -2508,11 +2498,11 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: assert server_domain in transformed_aspect.domains -def test_pattern_add_dataset_domain_aspect_name(mock_datahub_graph): +def test_pattern_add_dataset_domain_aspect_name(mock_datahub_graph_instance): pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance transformer = PatternAddDatasetDomain.create( {"domain_pattern": {"rules": {}}}, pipeline_context @@ -2520,7 +2510,7 @@ def test_pattern_add_dataset_domain_aspect_name(mock_datahub_graph): assert transformer.aspect_name() == models.DomainsClass.ASPECT_NAME -def test_pattern_add_dataset_domain_match(mock_datahub_graph): +def test_pattern_add_dataset_domain_match(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" @@ -2528,7 +2518,7 @@ def test_pattern_add_dataset_domain_match(mock_datahub_graph): pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, @@ -2551,7 +2541,7 @@ def test_pattern_add_dataset_domain_match(mock_datahub_graph): assert acryl_domain in transformed_aspect.domains -def test_pattern_add_dataset_domain_no_match(mock_datahub_graph): +def test_pattern_add_dataset_domain_no_match(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*" @@ -2559,7 +2549,7 @@ def test_pattern_add_dataset_domain_no_match(mock_datahub_graph): pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, @@ -2582,7 +2572,7 @@ def test_pattern_add_dataset_domain_no_match(mock_datahub_graph): assert acryl_domain not in transformed_aspect.domains -def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph): +def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" @@ -2590,7 +2580,7 @@ def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph): pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, @@ -2614,7 +2604,9 @@ def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph): assert acryl_domain in transformed_aspect.domains -def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph): +def test_pattern_add_dataset_domain_replace_existing_no_match( + mock_datahub_graph_instance, +): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*" @@ -2622,7 +2614,7 @@ def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, @@ -2644,14 +2636,14 @@ def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph assert len(transformed_aspect.domains) == 0 -def test_pattern_add_dataset_domain_semantics_overwrite(mock_datahub_graph): +def test_pattern_add_dataset_domain_semantics_overwrite(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") server_domain = builder.make_domain_urn("test.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # Return fake aspect to simulate server behaviour def fake_get_domain(entity_urn: str) -> models.DomainsClass: @@ -2683,7 +2675,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: def test_pattern_add_dataset_domain_semantics_patch( - pytestconfig, tmp_path, mock_time, mock_datahub_graph + pytestconfig, tmp_path, mock_time, mock_datahub_graph_instance ): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") @@ -2691,7 +2683,7 @@ def test_pattern_add_dataset_domain_semantics_patch( pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # Return fake aspect to simulate server behaviour def fake_get_domain(entity_urn: str) -> models.DomainsClass: @@ -2723,9 +2715,11 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: assert server_domain in transformed_aspect.domains -def test_simple_dataset_ownership_transformer_semantics_patch(mock_datahub_graph): +def test_simple_dataset_ownership_transformer_semantics_patch( + mock_datahub_graph_instance, +): pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance server_owner: str = builder.make_owner_urn( "mohd@acryl.io", owner_type=builder.OwnerType.USER @@ -2783,7 +2777,9 @@ def fake_ownership_class(entity_urn: str) -> models.OwnershipClass: assert server_owner in owner_urns -def test_pattern_container_and_dataset_domain_transformation(mock_datahub_graph): +def test_pattern_container_and_dataset_domain_transformation( + mock_datahub_graph_instance, +): datahub_domain = builder.make_domain_urn("datahubproject.io") acryl_domain = builder.make_domain_urn("acryl_domain") server_domain = builder.make_domain_urn("server_domain") @@ -2807,7 +2803,7 @@ def fake_get_aspect( pipeline_context = PipelineContext( run_id="test_pattern_container_and_dataset_domain_transformation" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance pipeline_context.graph.get_aspect = fake_get_aspect # type: ignore with_domain_aspect = make_generic_dataset_mcp( @@ -2889,7 +2885,7 @@ def fake_get_aspect( def test_pattern_container_and_dataset_domain_transformation_with_no_container( - mock_datahub_graph, + mock_datahub_graph_instance, ): datahub_domain = builder.make_domain_urn("datahubproject.io") acryl_domain = builder.make_domain_urn("acryl_domain") @@ -2905,7 +2901,7 @@ def fake_get_aspect( pipeline_context = PipelineContext( run_id="test_pattern_container_and_dataset_domain_transformation_with_no_container" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance pipeline_context.graph.get_aspect = fake_get_aspect # type: ignore with_domain_aspect = make_generic_dataset_mcp( @@ -2955,7 +2951,7 @@ def fake_get_aspect( assert server_domain in second_domain_aspect.domains -def test_pattern_add_container_dataset_domain_no_match(mock_datahub_graph): +def test_pattern_add_container_dataset_domain_no_match(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*" @@ -2963,7 +2959,7 @@ def test_pattern_add_container_dataset_domain_no_match(mock_datahub_graph): pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance def fake_get_aspect( entity_urn: str, @@ -3003,10 +2999,10 @@ def fake_get_aspect( def run_pattern_dataset_schema_terms_transformation_semantics( semantics: TransformerSemantics, - mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph], + mock_datahub_graph_instance: DataHubGraph, ) -> List[RecordEnvelope]: pipeline_context = PipelineContext(run_id="test_pattern_dataset_schema_terms") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # fake the server response def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass: @@ -3113,10 +3109,10 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass: def test_pattern_dataset_schema_terms_transformation_patch( - mock_time, mock_datahub_graph + mock_time, mock_datahub_graph_instance ): output = run_pattern_dataset_schema_terms_transformation_semantics( - TransformerSemantics.PATCH, mock_datahub_graph + TransformerSemantics.PATCH, mock_datahub_graph_instance ) assert len(output) == 2 # Check that glossary terms were added. @@ -3146,10 +3142,10 @@ def test_pattern_dataset_schema_terms_transformation_patch( def test_pattern_dataset_schema_terms_transformation_overwrite( - mock_time, mock_datahub_graph + mock_time, mock_datahub_graph_instance ): output = run_pattern_dataset_schema_terms_transformation_semantics( - TransformerSemantics.OVERWRITE, mock_datahub_graph + TransformerSemantics.OVERWRITE, mock_datahub_graph_instance ) assert len(output) == 2 @@ -3181,10 +3177,10 @@ def test_pattern_dataset_schema_terms_transformation_overwrite( def run_pattern_dataset_schema_tags_transformation_semantics( semantics: TransformerSemantics, - mock_datahub_graph: Callable[[DatahubClientConfig], DataHubGraph], + mock_datahub_graph_instance: DataHubGraph, ) -> List[RecordEnvelope]: pipeline_context = PipelineContext(run_id="test_pattern_dataset_schema_terms") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # fake the server response def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass: @@ -3284,10 +3280,10 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass: def test_pattern_dataset_schema_tags_transformation_overwrite( - mock_time, mock_datahub_graph + mock_time, mock_datahub_graph_instance ): output = run_pattern_dataset_schema_tags_transformation_semantics( - TransformerSemantics.OVERWRITE, mock_datahub_graph + TransformerSemantics.OVERWRITE, mock_datahub_graph_instance ) assert len(output) == 2 @@ -3318,10 +3314,10 @@ def test_pattern_dataset_schema_tags_transformation_overwrite( def test_pattern_dataset_schema_tags_transformation_patch( - mock_time, mock_datahub_graph + mock_time, mock_datahub_graph_instance ): output = run_pattern_dataset_schema_tags_transformation_semantics( - TransformerSemantics.PATCH, mock_datahub_graph + TransformerSemantics.PATCH, mock_datahub_graph_instance ) assert len(output) == 2 @@ -3542,9 +3538,11 @@ def fake_ownership_class(entity_urn: str) -> models.OwnershipClass: assert set(out_owners) == set(cleaned_owner_urn) -def test_clean_owner_urn_transformation_remove_fixed_string(mock_datahub_graph): +def test_clean_owner_urn_transformation_remove_fixed_string( + mock_datahub_graph_instance, +): pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance user_emails = [ "ABCDEF:email_id@example.com", @@ -3581,9 +3579,11 @@ def test_clean_owner_urn_transformation_remove_fixed_string(mock_datahub_graph): _test_clean_owner_urns(pipeline_context, in_owner_urns, config, expected_owner_urns) -def test_clean_owner_urn_transformation_remove_multiple_values(mock_datahub_graph): +def test_clean_owner_urn_transformation_remove_multiple_values( + mock_datahub_graph_instance, +): pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance user_emails = [ "ABCDEF:email_id@example.com", @@ -3620,9 +3620,11 @@ def test_clean_owner_urn_transformation_remove_multiple_values(mock_datahub_grap _test_clean_owner_urns(pipeline_context, in_owner_urns, config, expected_owner_urns) -def test_clean_owner_urn_transformation_remove_values_using_regex(mock_datahub_graph): +def test_clean_owner_urn_transformation_remove_values_using_regex( + mock_datahub_graph_instance, +): pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance user_emails = [ "ABCDEF:email_id@example.com", @@ -3659,9 +3661,9 @@ def test_clean_owner_urn_transformation_remove_values_using_regex(mock_datahub_g _test_clean_owner_urns(pipeline_context, in_owner_urns, config, expected_owner_urns) -def test_clean_owner_urn_transformation_remove_digits(mock_datahub_graph): +def test_clean_owner_urn_transformation_remove_digits(mock_datahub_graph_instance): pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance user_emails = [ "ABCDEF:email_id@example.com", @@ -3698,9 +3700,9 @@ def test_clean_owner_urn_transformation_remove_digits(mock_datahub_graph): _test_clean_owner_urns(pipeline_context, in_owner_urns, config, expected_owner_urns) -def test_clean_owner_urn_transformation_remove_pattern(mock_datahub_graph): +def test_clean_owner_urn_transformation_remove_pattern(mock_datahub_graph_instance): pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance user_emails = [ "ABCDEF:email_id@example.com", @@ -3738,10 +3740,10 @@ def test_clean_owner_urn_transformation_remove_pattern(mock_datahub_graph): def test_clean_owner_urn_transformation_remove_word_in_capital_letters( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance user_emails = [ "ABCDEF:email_id@example.com", @@ -3781,10 +3783,10 @@ def test_clean_owner_urn_transformation_remove_word_in_capital_letters( def test_clean_owner_urn_transformation_remove_pattern_with_alphanumeric_value( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance user_emails = [ "ABCDEF:email_id@example.com", @@ -3822,10 +3824,10 @@ def test_clean_owner_urn_transformation_remove_pattern_with_alphanumeric_value( def test_clean_owner_urn_transformation_should_not_remove_system_identifier( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance user_emails = [ "ABCDEF:email_id@example.com", @@ -3850,12 +3852,12 @@ def test_clean_owner_urn_transformation_should_not_remove_system_identifier( def test_replace_external_url_word_replace( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context: PipelineContext = PipelineContext( run_id="test_replace_external_url" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_dataset_transformer_pipeline( transformer_type=ReplaceExternalUrlDataset, @@ -3877,12 +3879,12 @@ def test_replace_external_url_word_replace( def test_replace_external_regex_replace_1( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context: PipelineContext = PipelineContext( run_id="test_replace_external_url" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_dataset_transformer_pipeline( transformer_type=ReplaceExternalUrlDataset, @@ -3904,12 +3906,12 @@ def test_replace_external_regex_replace_1( def test_replace_external_regex_replace_2( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context: PipelineContext = PipelineContext( run_id="test_replace_external_url" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_dataset_transformer_pipeline( transformer_type=ReplaceExternalUrlDataset, @@ -3931,12 +3933,12 @@ def test_replace_external_regex_replace_2( def test_pattern_cleanup_usage_statistics_user_1( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context: PipelineContext = PipelineContext( run_id="test_pattern_cleanup_usage_statistics_user" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance TS_1 = datetime(year=2023, month=1, day=1, tzinfo=timezone.utc) @@ -3985,12 +3987,12 @@ def test_pattern_cleanup_usage_statistics_user_1( def test_pattern_cleanup_usage_statistics_user_2( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context: PipelineContext = PipelineContext( run_id="test_pattern_cleanup_usage_statistics_user" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance TS_1 = datetime(year=2023, month=1, day=1, tzinfo=timezone.utc) @@ -4039,12 +4041,12 @@ def test_pattern_cleanup_usage_statistics_user_2( def test_pattern_cleanup_usage_statistics_user_3( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context: PipelineContext = PipelineContext( run_id="test_pattern_cleanup_usage_statistics_user" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance TS_1 = datetime(year=2023, month=1, day=1, tzinfo=timezone.utc) @@ -4092,7 +4094,7 @@ def test_pattern_cleanup_usage_statistics_user_3( assert output[0].record.aspect.userCounts == expectedUsageStatistics.userCounts -def test_domain_mapping_based_on_tags_with_valid_tags(mock_datahub_graph): +def test_domain_mapping_based_on_tags_with_valid_tags(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") server_domain = builder.make_domain_urn("test.io") @@ -4103,7 +4105,7 @@ def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass: return models.GlobalTagsClass(tags=[TagAssociationClass(tag=tag_one)]) pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance pipeline_context.graph.get_tags = fake_get_tags # type: ignore @@ -4126,13 +4128,15 @@ def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass: assert server_domain not in transformed_aspect.domains -def test_domain_mapping_based_on_tags_with_no_matching_tags(mock_datahub_graph): +def test_domain_mapping_based_on_tags_with_no_matching_tags( + mock_datahub_graph_instance, +): acryl_domain = builder.make_domain_urn("acryl.io") server_domain = builder.make_domain_urn("test.io") non_matching_tag = builder.make_tag_urn("nonMatching") pipeline_context = PipelineContext(run_id="no_match_pipeline") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # Return fake aspect to simulate server behaviour def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass: @@ -4157,11 +4161,11 @@ def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass: assert server_domain in transformed_aspect.domains -def test_domain_mapping_based_on_tags_with_empty_config(mock_datahub_graph): +def test_domain_mapping_based_on_tags_with_empty_config(mock_datahub_graph_instance): some_tag = builder.make_tag_urn("someTag") pipeline_context = PipelineContext(run_id="empty_config_pipeline") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # Return fake aspect to simulate server behaviour def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass: @@ -4180,7 +4184,9 @@ def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass: assert len(output[0].record.aspect.domains) == 0 -def test_domain_mapping_based__r_on_tags_with_multiple_tags(mock_datahub_graph): +def test_domain_mapping_based__r_on_tags_with_multiple_tags( + mock_datahub_graph_instance, +): # Two tags that match different rules in the domain mapping configuration tag_one = builder.make_tag_urn("test:tag_1") tag_two = builder.make_tag_urn("test:tag_2") @@ -4189,7 +4195,7 @@ def test_domain_mapping_based__r_on_tags_with_multiple_tags(mock_datahub_graph): hr = builder.make_domain_urn("hr") pipeline_context = PipelineContext(run_id="multiple_matches_pipeline") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # Return fake aspect to simulate server behaviour def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass: @@ -4226,11 +4232,11 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: assert len(transformed_aspect.domains) == 3 -def test_domain_mapping_based_on_tags_with_empty_tags(mock_datahub_graph): +def test_domain_mapping_based_on_tags_with_empty_tags(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") server_domain = builder.make_domain_urn("test.io") pipeline_context = PipelineContext(run_id="empty_config_pipeline") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # Return fake aspect to simulate server behaviour def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass: @@ -4254,11 +4260,11 @@ def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass: assert server_domain not in transformed_aspect.domains -def test_domain_mapping_based_on_tags_with_no_tags(mock_datahub_graph): +def test_domain_mapping_based_on_tags_with_no_tags(mock_datahub_graph_instance): acryl_domain = builder.make_domain_urn("acryl.io") server_domain = builder.make_domain_urn("test.io") pipeline_context = PipelineContext(run_id="empty_config_pipeline") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance # Return fake aspect to simulate server behaviour def fake_get_tags(entity_urn: str) -> Optional[models.GlobalTagsClass]: @@ -4282,7 +4288,7 @@ def fake_get_tags(entity_urn: str) -> Optional[models.GlobalTagsClass]: assert server_domain not in transformed_aspect.domains -def test_tags_to_terms_transformation(mock_datahub_graph): +def test_tags_to_terms_transformation(mock_datahub_graph_instance): # Create domain URNs for the test term_urn_example1 = builder.make_term_urn("example1") term_urn_example2 = builder.make_term_urn("example2") @@ -4349,7 +4355,7 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass: ) pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance pipeline_context.graph.get_tags = fake_get_tags # type: ignore pipeline_context.graph.get_schema_metadata = fake_schema_metadata # type: ignore @@ -4383,7 +4389,7 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass: } -def test_tags_to_terms_with_no_matching_terms(mock_datahub_graph): +def test_tags_to_terms_with_no_matching_terms(mock_datahub_graph_instance): # Setup for test where no tags match the provided term mappings def fake_get_tags_no_match(entity_urn: str) -> models.GlobalTagsClass: return models.GlobalTagsClass( @@ -4394,7 +4400,7 @@ def fake_get_tags_no_match(entity_urn: str) -> models.GlobalTagsClass: ) pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance pipeline_context.graph.get_tags = fake_get_tags_no_match # type: ignore # No matching terms in config @@ -4420,13 +4426,13 @@ def fake_get_tags_no_match(entity_urn: str) -> models.GlobalTagsClass: assert len(terms_aspect.terms) == 1 -def test_tags_to_terms_with_missing_tags(mock_datahub_graph): +def test_tags_to_terms_with_missing_tags(mock_datahub_graph_instance): # Setup for test where no tags are present def fake_get_no_tags(entity_urn: str) -> models.GlobalTagsClass: return models.GlobalTagsClass(tags=[]) pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance pipeline_context.graph.get_tags = fake_get_no_tags # type: ignore config = {"tags": ["example1", "example2"]} @@ -4451,7 +4457,7 @@ def fake_get_no_tags(entity_urn: str) -> models.GlobalTagsClass: assert len(terms_aspect.terms) == 1 -def test_tags_to_terms_with_partial_match(mock_datahub_graph): +def test_tags_to_terms_with_partial_match(mock_datahub_graph_instance): # Setup for partial match scenario def fake_get_partial_match_tags(entity_urn: str) -> models.GlobalTagsClass: return models.GlobalTagsClass( @@ -4466,7 +4472,7 @@ def fake_get_partial_match_tags(entity_urn: str) -> models.GlobalTagsClass: ) pipeline_context = PipelineContext(run_id="transformer_pipe_line") - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph = mock_datahub_graph_instance pipeline_context.graph.get_tags = fake_get_partial_match_tags # type: ignore config = {"tags": ["example1"]} # Only 'example1' has a term mapped @@ -4493,12 +4499,12 @@ def fake_get_partial_match_tags(entity_urn: str) -> models.GlobalTagsClass: def test_replace_external_url_container_word_replace( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context: PipelineContext = PipelineContext( run_id="test_replace_external_url_container" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_container_transformer_pipeline( transformer_type=ReplaceExternalUrlContainer, @@ -4521,12 +4527,12 @@ def test_replace_external_url_container_word_replace( def test_replace_external_regex_container_replace_1( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context: PipelineContext = PipelineContext( run_id="test_replace_external_url_container" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_container_transformer_pipeline( transformer_type=ReplaceExternalUrlContainer, @@ -4549,12 +4555,12 @@ def test_replace_external_regex_container_replace_1( def test_replace_external_regex_container_replace_2( - mock_datahub_graph, + mock_datahub_graph_instance, ): pipeline_context: PipelineContext = PipelineContext( run_id="test_replace_external_url_container" ) - pipeline_context.graph = mock_datahub_graph(DatahubClientConfig) + pipeline_context.graph = mock_datahub_graph_instance output = run_container_transformer_pipeline( transformer_type=ReplaceExternalUrlContainer, diff --git a/smoke-test/pytest.ini b/smoke-test/pytest.ini index 61ce840fd713c6..3762344cec4bea 100644 --- a/smoke-test/pytest.ini +++ b/smoke-test/pytest.ini @@ -1,3 +1,6 @@ [pytest] markers = read_only: marks tests as read only (deselect with '-m "not read_only"') + ; no_cypress_suite0: main smoke tests; expressed as the negative of the others + no_cypress_suite1: main smoke tests, suite 1 + test_run_cypress: run cypress tests diff --git a/smoke-test/tests/test_stateful_ingestion.py b/smoke-test/tests/test_stateful_ingestion.py index c0df51dd9d98e1..4436cf26c2fd78 100644 --- a/smoke-test/tests/test_stateful_ingestion.py +++ b/smoke-test/tests/test_stateful_ingestion.py @@ -65,10 +65,6 @@ def get_current_checkpoint_from_pipeline( "enabled": True, "remove_stale_metadata": True, "fail_safe_threshold": 100.0, - "state_provider": { - "type": "datahub", - "config": {"datahub_api": {"server": auth_session.gms_url()}}, - }, }, } From b8144699fdb37c3e935bef92b628d1d956e3f666 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 16 Oct 2024 19:18:32 -0700 Subject: [PATCH 29/40] chore(ingest): reorganize unit tests (#11636) --- .../tests/unit/{ => api}/test_apis.py | 0 .../test_entity_filter_report.py} | 0 .../tests/unit/{ => api}/test_pipeline.py | 10 +++---- .../unit/{ => api}/test_plugin_system.py | 0 .../{test_report.py => test_source_report.py} | 0 .../tests/unit/{ => api}/test_workunit.py | 0 .../{ => bigquery}/test_bigquery_lineage.py | 0 .../{ => bigquery}/test_bigquery_profiler.py | 0 .../{ => bigquery}/test_bigquery_source.py | 0 .../test_bigquery_sql_lineage.py | 0 .../{ => bigquery}/test_bigquery_usage.py | 0 .../test_bigqueryv2_usage_source.py | 0 .../test_bq_get_partition_range.py | 0 .../tests/unit/{ => cli}/test_check.py | 0 .../unit/{ => cli}/test_check_upgrade.py | 0 .../tests/unit/{ => cli}/test_cli_utils.py | 0 .../{ => config}/test_key_value_pattern.py | 0 .../tests/unit/glue/__init__.py | 0 .../tests/unit/{ => glue}/test_glue_source.py | 12 ++------ .../unit/{ => glue}/test_glue_source_stubs.py | 0 .../tests/unit/redshift/__init__.py | 0 .../{ => redshift}/redshift_query_mocker.py | 0 .../{ => redshift}/test_redshift_config.py | 0 .../{ => redshift}/test_redshift_lineage.py | 2 +- .../{ => redshift}/test_redshift_source.py | 0 .../tests/unit/sagemaker/__init__.py | 0 .../unit/sagemaker/test_sagemaker_source.py | 2 +- .../test_sagemaker_source_stubs.py | 0 .../tests/unit/{graph => sdk}/test_client.py | 0 .../unit/{ => sdk}/test_kafka_emitter.py | 0 .../tests/unit/{ => sdk}/test_mce_builder.py | 0 .../tests/unit/{ => sdk}/test_mcp_builder.py | 0 .../tests/unit/{ => sdk}/test_mcp_wrapper.py | 0 .../tests/unit/{ => sdk}/test_rest_emitter.py | 0 .../tests/unit/{ => serde}/test_codegen.py | 0 .../state}/test_ldap_state.py | 0 .../unit/{ => utilities}/test_cli_logging.py | 0 .../unit/{ => utilities}/test_ordered_set.py | 0 .../tests/unit/utilities/test_perf_timer.py | 28 ++++++++++--------- .../test_serialized_lru_cache.py | 0 .../{ => utilities}/test_topological_sort.py | 0 .../unit/{ => utilities}/test_utilities.py | 0 42 files changed, 25 insertions(+), 29 deletions(-) rename metadata-ingestion/tests/unit/{ => api}/test_apis.py (100%) rename metadata-ingestion/tests/unit/{test_report.py => api/test_entity_filter_report.py} (100%) rename metadata-ingestion/tests/unit/{ => api}/test_pipeline.py (97%) rename metadata-ingestion/tests/unit/{ => api}/test_plugin_system.py (100%) rename metadata-ingestion/tests/unit/api/{test_report.py => test_source_report.py} (100%) rename metadata-ingestion/tests/unit/{ => api}/test_workunit.py (100%) rename metadata-ingestion/tests/unit/{ => bigquery}/test_bigquery_lineage.py (100%) rename metadata-ingestion/tests/unit/{ => bigquery}/test_bigquery_profiler.py (100%) rename metadata-ingestion/tests/unit/{ => bigquery}/test_bigquery_source.py (100%) rename metadata-ingestion/tests/unit/{ => bigquery}/test_bigquery_sql_lineage.py (100%) rename metadata-ingestion/tests/unit/{ => bigquery}/test_bigquery_usage.py (100%) rename metadata-ingestion/tests/unit/{ => bigquery}/test_bigqueryv2_usage_source.py (100%) rename metadata-ingestion/tests/unit/{ => bigquery}/test_bq_get_partition_range.py (100%) rename metadata-ingestion/tests/unit/{ => cli}/test_check.py (100%) rename metadata-ingestion/tests/unit/{ => cli}/test_check_upgrade.py (100%) rename metadata-ingestion/tests/unit/{ => cli}/test_cli_utils.py (100%) rename metadata-ingestion/tests/unit/{ => config}/test_key_value_pattern.py (100%) create mode 100644 metadata-ingestion/tests/unit/glue/__init__.py rename metadata-ingestion/tests/unit/{ => glue}/test_glue_source.py (97%) rename metadata-ingestion/tests/unit/{ => glue}/test_glue_source_stubs.py (100%) create mode 100644 metadata-ingestion/tests/unit/redshift/__init__.py rename metadata-ingestion/tests/unit/{ => redshift}/redshift_query_mocker.py (100%) rename metadata-ingestion/tests/unit/{ => redshift}/test_redshift_config.py (100%) rename metadata-ingestion/tests/unit/{ => redshift}/test_redshift_lineage.py (99%) rename metadata-ingestion/tests/unit/{ => redshift}/test_redshift_source.py (100%) create mode 100644 metadata-ingestion/tests/unit/sagemaker/__init__.py rename metadata-ingestion/tests/unit/{ => sagemaker}/test_sagemaker_source_stubs.py (100%) rename metadata-ingestion/tests/unit/{graph => sdk}/test_client.py (100%) rename metadata-ingestion/tests/unit/{ => sdk}/test_kafka_emitter.py (100%) rename metadata-ingestion/tests/unit/{ => sdk}/test_mce_builder.py (100%) rename metadata-ingestion/tests/unit/{ => sdk}/test_mcp_builder.py (100%) rename metadata-ingestion/tests/unit/{ => sdk}/test_mcp_wrapper.py (100%) rename metadata-ingestion/tests/unit/{ => sdk}/test_rest_emitter.py (100%) rename metadata-ingestion/tests/unit/{ => serde}/test_codegen.py (100%) rename metadata-ingestion/tests/unit/{ => stateful_ingestion/state}/test_ldap_state.py (100%) rename metadata-ingestion/tests/unit/{ => utilities}/test_cli_logging.py (100%) rename metadata-ingestion/tests/unit/{ => utilities}/test_ordered_set.py (100%) rename metadata-ingestion/tests/unit/{ => utilities}/test_serialized_lru_cache.py (100%) rename metadata-ingestion/tests/unit/{ => utilities}/test_topological_sort.py (100%) rename metadata-ingestion/tests/unit/{ => utilities}/test_utilities.py (100%) diff --git a/metadata-ingestion/tests/unit/test_apis.py b/metadata-ingestion/tests/unit/api/test_apis.py similarity index 100% rename from metadata-ingestion/tests/unit/test_apis.py rename to metadata-ingestion/tests/unit/api/test_apis.py diff --git a/metadata-ingestion/tests/unit/test_report.py b/metadata-ingestion/tests/unit/api/test_entity_filter_report.py similarity index 100% rename from metadata-ingestion/tests/unit/test_report.py rename to metadata-ingestion/tests/unit/api/test_entity_filter_report.py diff --git a/metadata-ingestion/tests/unit/test_pipeline.py b/metadata-ingestion/tests/unit/api/test_pipeline.py similarity index 97% rename from metadata-ingestion/tests/unit/test_pipeline.py rename to metadata-ingestion/tests/unit/api/test_pipeline.py index a462f281367973..432d8e11c1c0b4 100644 --- a/metadata-ingestion/tests/unit/test_pipeline.py +++ b/metadata-ingestion/tests/unit/api/test_pipeline.py @@ -224,9 +224,9 @@ def test_configure_with_file_sink_does_not_init_graph(self, mock_source, tmp_pat def test_run_including_fake_transformation(self): pipeline = Pipeline.create( { - "source": {"type": "tests.unit.test_pipeline.FakeSource"}, + "source": {"type": "tests.unit.api.test_pipeline.FakeSource"}, "transformers": [ - {"type": "tests.unit.test_pipeline.AddStatusRemovedTransformer"} + {"type": "tests.unit.api.test_pipeline.AddStatusRemovedTransformer"} ], "sink": {"type": "tests.test_helpers.sink_helpers.RecordingSink"}, "run_id": "pipeline_test", @@ -253,7 +253,7 @@ def test_run_including_registered_transformation(self): pipeline = Pipeline.create( { - "source": {"type": "tests.unit.test_pipeline.FakeSource"}, + "source": {"type": "tests.unit.api.test_pipeline.FakeSource"}, "transformers": [ { "type": "simple_add_dataset_ownership", @@ -297,7 +297,7 @@ def test_pipeline_return_code(self, tmp_path, source, strict_warnings, exit_code --- run_id: pipeline_test source: - type: tests.unit.test_pipeline.{source} + type: tests.unit.api.test_pipeline.{source} config: {{}} sink: type: console @@ -379,7 +379,7 @@ def test_pipeline_return_code(self, tmp_path, source, strict_warnings, exit_code def test_pipeline_process_commits(self, commit_policy, source, should_commit): pipeline = Pipeline.create( { - "source": {"type": f"tests.unit.test_pipeline.{source}"}, + "source": {"type": f"tests.unit.api.test_pipeline.{source}"}, "sink": {"type": "console"}, "run_id": "pipeline_test", } diff --git a/metadata-ingestion/tests/unit/test_plugin_system.py b/metadata-ingestion/tests/unit/api/test_plugin_system.py similarity index 100% rename from metadata-ingestion/tests/unit/test_plugin_system.py rename to metadata-ingestion/tests/unit/api/test_plugin_system.py diff --git a/metadata-ingestion/tests/unit/api/test_report.py b/metadata-ingestion/tests/unit/api/test_source_report.py similarity index 100% rename from metadata-ingestion/tests/unit/api/test_report.py rename to metadata-ingestion/tests/unit/api/test_source_report.py diff --git a/metadata-ingestion/tests/unit/test_workunit.py b/metadata-ingestion/tests/unit/api/test_workunit.py similarity index 100% rename from metadata-ingestion/tests/unit/test_workunit.py rename to metadata-ingestion/tests/unit/api/test_workunit.py diff --git a/metadata-ingestion/tests/unit/test_bigquery_lineage.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_lineage.py similarity index 100% rename from metadata-ingestion/tests/unit/test_bigquery_lineage.py rename to metadata-ingestion/tests/unit/bigquery/test_bigquery_lineage.py diff --git a/metadata-ingestion/tests/unit/test_bigquery_profiler.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_profiler.py similarity index 100% rename from metadata-ingestion/tests/unit/test_bigquery_profiler.py rename to metadata-ingestion/tests/unit/bigquery/test_bigquery_profiler.py diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_source.py similarity index 100% rename from metadata-ingestion/tests/unit/test_bigquery_source.py rename to metadata-ingestion/tests/unit/bigquery/test_bigquery_source.py diff --git a/metadata-ingestion/tests/unit/test_bigquery_sql_lineage.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_sql_lineage.py similarity index 100% rename from metadata-ingestion/tests/unit/test_bigquery_sql_lineage.py rename to metadata-ingestion/tests/unit/bigquery/test_bigquery_sql_lineage.py diff --git a/metadata-ingestion/tests/unit/test_bigquery_usage.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_usage.py similarity index 100% rename from metadata-ingestion/tests/unit/test_bigquery_usage.py rename to metadata-ingestion/tests/unit/bigquery/test_bigquery_usage.py diff --git a/metadata-ingestion/tests/unit/test_bigqueryv2_usage_source.py b/metadata-ingestion/tests/unit/bigquery/test_bigqueryv2_usage_source.py similarity index 100% rename from metadata-ingestion/tests/unit/test_bigqueryv2_usage_source.py rename to metadata-ingestion/tests/unit/bigquery/test_bigqueryv2_usage_source.py diff --git a/metadata-ingestion/tests/unit/test_bq_get_partition_range.py b/metadata-ingestion/tests/unit/bigquery/test_bq_get_partition_range.py similarity index 100% rename from metadata-ingestion/tests/unit/test_bq_get_partition_range.py rename to metadata-ingestion/tests/unit/bigquery/test_bq_get_partition_range.py diff --git a/metadata-ingestion/tests/unit/test_check.py b/metadata-ingestion/tests/unit/cli/test_check.py similarity index 100% rename from metadata-ingestion/tests/unit/test_check.py rename to metadata-ingestion/tests/unit/cli/test_check.py diff --git a/metadata-ingestion/tests/unit/test_check_upgrade.py b/metadata-ingestion/tests/unit/cli/test_check_upgrade.py similarity index 100% rename from metadata-ingestion/tests/unit/test_check_upgrade.py rename to metadata-ingestion/tests/unit/cli/test_check_upgrade.py diff --git a/metadata-ingestion/tests/unit/test_cli_utils.py b/metadata-ingestion/tests/unit/cli/test_cli_utils.py similarity index 100% rename from metadata-ingestion/tests/unit/test_cli_utils.py rename to metadata-ingestion/tests/unit/cli/test_cli_utils.py diff --git a/metadata-ingestion/tests/unit/test_key_value_pattern.py b/metadata-ingestion/tests/unit/config/test_key_value_pattern.py similarity index 100% rename from metadata-ingestion/tests/unit/test_key_value_pattern.py rename to metadata-ingestion/tests/unit/config/test_key_value_pattern.py diff --git a/metadata-ingestion/tests/unit/glue/__init__.py b/metadata-ingestion/tests/unit/glue/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/metadata-ingestion/tests/unit/test_glue_source.py b/metadata-ingestion/tests/unit/glue/test_glue_source.py similarity index 97% rename from metadata-ingestion/tests/unit/test_glue_source.py rename to metadata-ingestion/tests/unit/glue/test_glue_source.py index eb1e7f3fe41d9a..57f48db1129c42 100644 --- a/metadata-ingestion/tests/unit/test_glue_source.py +++ b/metadata-ingestion/tests/unit/glue/test_glue_source.py @@ -35,7 +35,7 @@ validate_all_providers_have_committed_successfully, ) from tests.test_helpers.type_helpers import PytestConfig -from tests.unit.test_glue_source_stubs import ( +from tests.unit.glue.test_glue_source_stubs import ( databases_1, databases_2, get_bucket_tagging, @@ -71,6 +71,8 @@ GMS_PORT = 8080 GMS_SERVER = f"http://localhost:{GMS_PORT}" +test_resources_dir = Path(__file__).parent + def glue_source( platform_instance: Optional[str] = None, @@ -247,7 +249,6 @@ def test_glue_ingest( write_metadata_file(tmp_path / mce_file, mce_objects) # Verify the output. - test_resources_dir = pytestconfig.rootpath / "tests/unit/glue" mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / mce_file, @@ -312,8 +313,6 @@ def test_config_without_platform(): @freeze_time(FROZEN_TIME) def test_glue_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): - test_resources_dir = pytestconfig.rootpath / "tests/unit/glue" - deleted_actor_golden_mcs = "{}/glue_deleted_actor_mces_golden.json".format( test_resources_dir ) @@ -438,7 +437,6 @@ def test_glue_with_delta_schema_ingest( write_metadata_file(tmp_path / "glue_delta_mces.json", mce_objects) # Verify the output. - test_resources_dir = pytestconfig.rootpath / "tests/unit/glue" mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / "glue_delta_mces.json", @@ -475,7 +473,6 @@ def test_glue_with_malformed_delta_schema_ingest( write_metadata_file(tmp_path / "glue_malformed_delta_mces.json", mce_objects) # Verify the output. - test_resources_dir = pytestconfig.rootpath / "tests/unit/glue" mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / "glue_malformed_delta_mces.json", @@ -571,7 +568,6 @@ def test_glue_ingest_include_table_lineage( write_metadata_file(tmp_path / mce_file, mce_objects) # Verify the output. - test_resources_dir = pytestconfig.rootpath / "tests/unit/glue" mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / mce_file, @@ -678,7 +674,6 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass: write_metadata_file(tmp_path / mce_file, mce_objects) # Verify the output. - test_resources_dir = pytestconfig.rootpath / "tests/unit/glue" mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / mce_file, @@ -716,7 +711,6 @@ def test_glue_ingest_with_profiling( write_metadata_file(tmp_path / mce_file, mce_objects) # Verify the output. - test_resources_dir = pytestconfig.rootpath / "tests/unit/glue" mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / mce_file, diff --git a/metadata-ingestion/tests/unit/test_glue_source_stubs.py b/metadata-ingestion/tests/unit/glue/test_glue_source_stubs.py similarity index 100% rename from metadata-ingestion/tests/unit/test_glue_source_stubs.py rename to metadata-ingestion/tests/unit/glue/test_glue_source_stubs.py diff --git a/metadata-ingestion/tests/unit/redshift/__init__.py b/metadata-ingestion/tests/unit/redshift/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/metadata-ingestion/tests/unit/redshift_query_mocker.py b/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py similarity index 100% rename from metadata-ingestion/tests/unit/redshift_query_mocker.py rename to metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py diff --git a/metadata-ingestion/tests/unit/test_redshift_config.py b/metadata-ingestion/tests/unit/redshift/test_redshift_config.py similarity index 100% rename from metadata-ingestion/tests/unit/test_redshift_config.py rename to metadata-ingestion/tests/unit/redshift/test_redshift_config.py diff --git a/metadata-ingestion/tests/unit/test_redshift_lineage.py b/metadata-ingestion/tests/unit/redshift/test_redshift_lineage.py similarity index 99% rename from metadata-ingestion/tests/unit/test_redshift_lineage.py rename to metadata-ingestion/tests/unit/redshift/test_redshift_lineage.py index 78b7169a93f3c8..2e3eb8fde1292b 100644 --- a/metadata-ingestion/tests/unit/test_redshift_lineage.py +++ b/metadata-ingestion/tests/unit/redshift/test_redshift_lineage.py @@ -26,7 +26,7 @@ SqlParsingDebugInfo, SqlParsingResult, ) -from tests.unit.redshift_query_mocker import mock_cursor +from tests.unit.redshift.redshift_query_mocker import mock_cursor def test_get_sources_from_query(): diff --git a/metadata-ingestion/tests/unit/test_redshift_source.py b/metadata-ingestion/tests/unit/redshift/test_redshift_source.py similarity index 100% rename from metadata-ingestion/tests/unit/test_redshift_source.py rename to metadata-ingestion/tests/unit/redshift/test_redshift_source.py diff --git a/metadata-ingestion/tests/unit/sagemaker/__init__.py b/metadata-ingestion/tests/unit/sagemaker/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py b/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py index 995d176c213b24..2450e6fa8fe564 100644 --- a/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py +++ b/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py @@ -14,7 +14,7 @@ job_types, ) from tests.test_helpers import mce_helpers -from tests.unit.test_sagemaker_source_stubs import ( +from tests.unit.sagemaker.test_sagemaker_source_stubs import ( describe_endpoint_response_1, describe_endpoint_response_2, describe_feature_group_response_1, diff --git a/metadata-ingestion/tests/unit/test_sagemaker_source_stubs.py b/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source_stubs.py similarity index 100% rename from metadata-ingestion/tests/unit/test_sagemaker_source_stubs.py rename to metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source_stubs.py diff --git a/metadata-ingestion/tests/unit/graph/test_client.py b/metadata-ingestion/tests/unit/sdk/test_client.py similarity index 100% rename from metadata-ingestion/tests/unit/graph/test_client.py rename to metadata-ingestion/tests/unit/sdk/test_client.py diff --git a/metadata-ingestion/tests/unit/test_kafka_emitter.py b/metadata-ingestion/tests/unit/sdk/test_kafka_emitter.py similarity index 100% rename from metadata-ingestion/tests/unit/test_kafka_emitter.py rename to metadata-ingestion/tests/unit/sdk/test_kafka_emitter.py diff --git a/metadata-ingestion/tests/unit/test_mce_builder.py b/metadata-ingestion/tests/unit/sdk/test_mce_builder.py similarity index 100% rename from metadata-ingestion/tests/unit/test_mce_builder.py rename to metadata-ingestion/tests/unit/sdk/test_mce_builder.py diff --git a/metadata-ingestion/tests/unit/test_mcp_builder.py b/metadata-ingestion/tests/unit/sdk/test_mcp_builder.py similarity index 100% rename from metadata-ingestion/tests/unit/test_mcp_builder.py rename to metadata-ingestion/tests/unit/sdk/test_mcp_builder.py diff --git a/metadata-ingestion/tests/unit/test_mcp_wrapper.py b/metadata-ingestion/tests/unit/sdk/test_mcp_wrapper.py similarity index 100% rename from metadata-ingestion/tests/unit/test_mcp_wrapper.py rename to metadata-ingestion/tests/unit/sdk/test_mcp_wrapper.py diff --git a/metadata-ingestion/tests/unit/test_rest_emitter.py b/metadata-ingestion/tests/unit/sdk/test_rest_emitter.py similarity index 100% rename from metadata-ingestion/tests/unit/test_rest_emitter.py rename to metadata-ingestion/tests/unit/sdk/test_rest_emitter.py diff --git a/metadata-ingestion/tests/unit/test_codegen.py b/metadata-ingestion/tests/unit/serde/test_codegen.py similarity index 100% rename from metadata-ingestion/tests/unit/test_codegen.py rename to metadata-ingestion/tests/unit/serde/test_codegen.py diff --git a/metadata-ingestion/tests/unit/test_ldap_state.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_ldap_state.py similarity index 100% rename from metadata-ingestion/tests/unit/test_ldap_state.py rename to metadata-ingestion/tests/unit/stateful_ingestion/state/test_ldap_state.py diff --git a/metadata-ingestion/tests/unit/test_cli_logging.py b/metadata-ingestion/tests/unit/utilities/test_cli_logging.py similarity index 100% rename from metadata-ingestion/tests/unit/test_cli_logging.py rename to metadata-ingestion/tests/unit/utilities/test_cli_logging.py diff --git a/metadata-ingestion/tests/unit/test_ordered_set.py b/metadata-ingestion/tests/unit/utilities/test_ordered_set.py similarity index 100% rename from metadata-ingestion/tests/unit/test_ordered_set.py rename to metadata-ingestion/tests/unit/utilities/test_ordered_set.py diff --git a/metadata-ingestion/tests/unit/utilities/test_perf_timer.py b/metadata-ingestion/tests/unit/utilities/test_perf_timer.py index 6129b3e37d8bc3..1de76a32fb708a 100644 --- a/metadata-ingestion/tests/unit/utilities/test_perf_timer.py +++ b/metadata-ingestion/tests/unit/utilities/test_perf_timer.py @@ -10,37 +10,39 @@ def test_perf_timer_simple(): with PerfTimer() as timer: - time.sleep(1) - assert approx(timer.elapsed_seconds()) == 1 + time.sleep(0.4) + assert approx(timer.elapsed_seconds()) == 0.4 - assert approx(timer.elapsed_seconds()) == 1 + assert approx(timer.elapsed_seconds()) == 0.4 def test_perf_timer_paused_timer(): with PerfTimer() as current_timer: - time.sleep(1) - assert approx(current_timer.elapsed_seconds()) == 1 + time.sleep(0.5) + assert approx(current_timer.elapsed_seconds()) == 0.5 with current_timer.pause(): - time.sleep(2) - assert approx(current_timer.elapsed_seconds()) == 1 - assert approx(current_timer.elapsed_seconds()) == 1 - time.sleep(1) + time.sleep(0.3) + assert approx(current_timer.elapsed_seconds()) == 0.5 + assert approx(current_timer.elapsed_seconds()) == 0.5 + time.sleep(0.2) - assert approx(current_timer.elapsed_seconds()) == 2 + assert approx(current_timer.elapsed_seconds()) == 0.7 def test_generator_with_paused_timer(): + n = 4 + def generator_function(): with PerfTimer() as inner_timer: time.sleep(1) - for i in range(10): + for i in range(n): time.sleep(0.2) with inner_timer.pause(): time.sleep(0.2) yield i - assert approx(inner_timer.elapsed_seconds()) == 1 + 0.2 * 10 + assert approx(inner_timer.elapsed_seconds()) == 1 + 0.2 * n with PerfTimer() as outer_timer: seq = generator_function() list([i for i in seq]) - assert approx(outer_timer.elapsed_seconds()) == 1 + 0.2 * 10 + 0.2 * 10 + assert approx(outer_timer.elapsed_seconds()) == 1 + 0.2 * n + 0.2 * n diff --git a/metadata-ingestion/tests/unit/test_serialized_lru_cache.py b/metadata-ingestion/tests/unit/utilities/test_serialized_lru_cache.py similarity index 100% rename from metadata-ingestion/tests/unit/test_serialized_lru_cache.py rename to metadata-ingestion/tests/unit/utilities/test_serialized_lru_cache.py diff --git a/metadata-ingestion/tests/unit/test_topological_sort.py b/metadata-ingestion/tests/unit/utilities/test_topological_sort.py similarity index 100% rename from metadata-ingestion/tests/unit/test_topological_sort.py rename to metadata-ingestion/tests/unit/utilities/test_topological_sort.py diff --git a/metadata-ingestion/tests/unit/test_utilities.py b/metadata-ingestion/tests/unit/utilities/test_utilities.py similarity index 100% rename from metadata-ingestion/tests/unit/test_utilities.py rename to metadata-ingestion/tests/unit/utilities/test_utilities.py From 8b42ac8cdebef89f8aff0ac030c72db9c54aad3d Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 16 Oct 2024 20:47:48 -0700 Subject: [PATCH 30/40] fix(ingest): run sqllineage in process by default (#11650) --- .../ingestion/source/looker/lookml_config.py | 3 - .../src/datahub/ingestion/source/redash.py | 23 +- .../src/datahub/utilities/sql_parser.py | 2 +- .../lookml/lookml_mces_badsql_parser.json | 2820 ----------------- .../tests/integration/lookml/test_lookml.py | 48 - 5 files changed, 13 insertions(+), 2883 deletions(-) delete mode 100644 metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py index 0bcee14ec77a1a..da837da1613864 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py @@ -124,9 +124,6 @@ class LookMLSourceConfig( description="List of regex patterns for LookML views to include in the extraction.", ) parse_table_names_from_sql: bool = Field(True, description="See note below.") - sql_parser: str = Field( - "datahub.utilities.sql_parser.DefaultSQLParser", description="See note below." - ) api: Optional[LookerAPIConfig] project_name: Optional[str] = Field( None, diff --git a/metadata-ingestion/src/datahub/ingestion/source/redash.py b/metadata-ingestion/src/datahub/ingestion/source/redash.py index 38cf0bebcbc12f..5fd63e7f93f92a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redash.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redash.py @@ -2,7 +2,6 @@ import math import sys from dataclasses import dataclass, field -from multiprocessing.pool import ThreadPool from typing import Dict, Iterable, List, Optional, Set, Type import dateutil.parser as dp @@ -43,6 +42,7 @@ from datahub.utilities.lossy_collections import LossyDict, LossyList from datahub.utilities.perf_timer import PerfTimer from datahub.utilities.sql_parser import SQLParser +from datahub.utilities.threaded_iterator_executor import ThreadedIteratorExecutor logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -646,11 +646,11 @@ def _emit_dashboard_mces(self) -> Iterable[MetadataWorkUnit]: self.report.total_dashboards = total_dashboards self.report.max_page_dashboards = max_page - dash_exec_pool = ThreadPool(self.config.parallelism) - for response in dash_exec_pool.imap_unordered( - self._process_dashboard_response, range(1, max_page + 1) - ): - yield from response + yield from ThreadedIteratorExecutor.process( + self._process_dashboard_response, + [(page,) for page in range(1, max_page + 1)], + max_workers=self.config.parallelism, + ) def _get_chart_type_from_viz_data(self, viz_data: Dict) -> str: """ @@ -769,11 +769,12 @@ def _emit_chart_mces(self) -> Iterable[MetadataWorkUnit]: logger.info(f"/api/queries total count {total_queries} and max page {max_page}") self.report.total_queries = total_queries self.report.max_page_queries = max_page - chart_exec_pool = ThreadPool(self.config.parallelism) - for response in chart_exec_pool.imap_unordered( - self._process_query_response, range(1, max_page + 1) - ): - yield from response + + yield from ThreadedIteratorExecutor.process( + self._process_query_response, + [(page,) for page in range(1, max_page + 1)], + max_workers=self.config.parallelism, + ) def add_config_to_report(self) -> None: self.report.api_page_limit = self.config.api_page_limit diff --git a/metadata-ingestion/src/datahub/utilities/sql_parser.py b/metadata-ingestion/src/datahub/utilities/sql_parser.py index 61693b52b350fb..b88f8fd8c73029 100644 --- a/metadata-ingestion/src/datahub/utilities/sql_parser.py +++ b/metadata-ingestion/src/datahub/utilities/sql_parser.py @@ -46,7 +46,7 @@ class SqlLineageSQLParser(SQLParser): def __init__( self, sql_query: str, - use_external_process: bool = True, + use_external_process: bool = False, use_raw_names: bool = False, ) -> None: super().__init__(sql_query, use_external_process) diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json deleted file mode 100644 index 5b39e8dd96ac2a..00000000000000 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json +++ /dev/null @@ -1,2820 +0,0 @@ -[ -{ - "entityType": "container", - "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "looker", - "env": "PROD", - "project_name": "lkml_samples" - }, - "name": "lkml_samples", - "env": "PROD" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:looker" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "LookML Project" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Folders" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "SELECT\n is_latest,\n country,\n city,\n timestamp,\n measurement\n FROM\n my_table", - "viewLanguage": "sql" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),country)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),city)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),is_latest)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),timestamp)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),measurement)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),average_measurement)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "my_view", - "platform": "urn:li:dataPlatform:looker", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "country", - "nullable": false, - "description": "The country", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "city", - "nullable": false, - "description": "City", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "is_latest", - "nullable": false, - "description": "Is latest data", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "yesno", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "timestamp", - "nullable": false, - "description": "Timestamp of measurement", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "time", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - }, - { - "tag": "urn:li:tag:Temporal" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "average_measurement", - "nullable": false, - "description": "My measurement", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "average", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Measure" - } - ] - }, - "isPartOfKey": false - } - ], - "primaryKeys": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "foo.view.lkml", - "looker.model": "data" - }, - "name": "my_view", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "SELECT\n country,\n city,\n timestamp,\n measurement\n FROM\n ${my_view.SQL_TABLE_NAME} AS my_view", - "viewLanguage": "sql" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),country)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),city)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),timestamp)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),measurement)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),average_measurement)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "my_derived_view", - "platform": "urn:li:dataPlatform:looker", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "country", - "nullable": false, - "description": "The country", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "city", - "nullable": false, - "description": "City", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "timestamp", - "nullable": false, - "description": "Timestamp of measurement", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "time", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - }, - { - "tag": "urn:li:tag:Temporal" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "average_measurement", - "nullable": false, - "description": "My measurement", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "average", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Measure" - } - ] - }, - "isPartOfKey": false - } - ], - "primaryKeys": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "bar.view.lkml", - "looker.model": "data" - }, - "name": "my_derived_view", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "view: include_able_view {\n sql_table_name: looker_schema.include_able ;;\n}\n", - "viewLanguage": "lookml" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.looker_schema.include_able,PROD)", - "type": "VIEW" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "included_view_file.view.lkml", - "looker.model": "data" - }, - "name": "include_able_view", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "include: \"/included_view_file.view\"\n\nview: looker_events {\n sql_table_name: looker_schema.events ;;\n}\n\nview: extending_looker_events {\n extends: [looker_events]\n\n measure: additional_measure {\n type: count\n }\n}\n\nview: autodetect_sql_name_based_on_view_name {}\n\nview: test_include_external_view {\n extends: [include_able_view]\n}\n", - "viewLanguage": "lookml" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.looker_schema.events,PROD)", - "type": "VIEW" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "view_declarations.view.lkml", - "looker.model": "data" - }, - "name": "looker_events", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "include: \"/included_view_file.view\"\n\nview: looker_events {\n sql_table_name: looker_schema.events ;;\n}\n\nview: extending_looker_events {\n extends: [looker_events]\n\n measure: additional_measure {\n type: count\n }\n}\n\nview: autodetect_sql_name_based_on_view_name {}\n\nview: test_include_external_view {\n extends: [include_able_view]\n}\n", - "viewLanguage": "lookml" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.looker_schema.events,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.looker_schema.events,PROD),additional_measure)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD),additional_measure)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "extending_looker_events", - "platform": "urn:li:dataPlatform:looker", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "additional_measure", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "count", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Measure" - } - ] - }, - "isPartOfKey": false - } - ], - "primaryKeys": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "view_declarations.view.lkml", - "looker.model": "data" - }, - "name": "extending_looker_events", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "include: \"/included_view_file.view\"\n\nview: looker_events {\n sql_table_name: looker_schema.events ;;\n}\n\nview: extending_looker_events {\n extends: [looker_events]\n\n measure: additional_measure {\n type: count\n }\n}\n\nview: autodetect_sql_name_based_on_view_name {}\n\nview: test_include_external_view {\n extends: [include_able_view]\n}\n", - "viewLanguage": "lookml" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.autodetect_sql_name_based_on_view_name,PROD)", - "type": "VIEW" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "view_declarations.view.lkml", - "looker.model": "data" - }, - "name": "autodetect_sql_name_based_on_view_name", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "include: \"/included_view_file.view\"\n\nview: looker_events {\n sql_table_name: looker_schema.events ;;\n}\n\nview: extending_looker_events {\n extends: [looker_events]\n\n measure: additional_measure {\n type: count\n }\n}\n\nview: autodetect_sql_name_based_on_view_name {}\n\nview: test_include_external_view {\n extends: [include_able_view]\n}\n", - "viewLanguage": "lookml" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.looker_schema.include_able,PROD)", - "type": "VIEW" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "view_declarations.view.lkml", - "looker.model": "data" - }, - "name": "test_include_external_view", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "SELECT date AS DATE,\n platform AS aliased_platform,\n country", - "viewLanguage": "sql" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/nested" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),date)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),date)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),platform)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),aliased_platform)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),country)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),country)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "fragment_derived_view", - "platform": "urn:li:dataPlatform:looker", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "date", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "unknown", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "aliased_platform", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "unknown", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "country", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "unknown", - "recursive": false, - "isPartOfKey": false - } - ], - "primaryKeys": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "nested/fragment_derived.view.lkml", - "looker.model": "data" - }, - "name": "fragment_derived_view", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - }, - { - "id": "nested" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", - "viewLanguage": "sql" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD),customer_id)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),customer_id)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD),sale_price)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),lifetime_spend)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "customer_facts", - "platform": "urn:li:dataPlatform:looker", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "customer_id", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "unknown", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "lifetime_spend", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "unknown", - "recursive": false, - "isPartOfKey": false - } - ], - "primaryKeys": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "liquid.view.lkml", - "looker.model": "data" - }, - "name": "customer_facts", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "view: ability {\n sql_table_name: \"ECOMMERCE\".\"ABILITY\"\n ;;\n\n dimension: pk {\n type: number\n sql: ${TABLE}.\"PK\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n", - "viewLanguage": "lookml" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.ecommerce.ability,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.ecommerce.ability,PROD),pk)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD),pk)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.ecommerce.ability,PROD),count)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD),count)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "ability", - "platform": "urn:li:dataPlatform:looker", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "pk", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "number", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "count", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "count", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Measure" - } - ] - }, - "isPartOfKey": false - } - ], - "primaryKeys": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "ability.view.lkml", - "looker.model": "data" - }, - "name": "ability", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "view: owners {\n dimension: id {\n primary_key: yes\n sql: ${TABLE}.id ;;\n }\n dimension: owner_name {\n sql: ${TABLE}.owner_name ;;\n }\n}", - "viewLanguage": "lookml" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.owners,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.owners,PROD),id)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD),id)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.owners,PROD),owner_name)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD),owner_name)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "owners", - "platform": "urn:li:dataPlatform:looker", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": true - }, - { - "fieldPath": "owner_name", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": false - } - ], - "primaryKeys": [ - "id" - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "owners.view.lkml", - "looker.model": "data" - }, - "name": "owners", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "explore_source: my_view_explore {\n bind_all_filters: yes\n\n column: country {\n field: my_view_explore.country\n }\n\n column: city {\n field: my_view_explore.city\n }\n\n column: is_latest {\n field: my_view_explore.is_latest\n }\n\n derived_column: derived_col {\n sql: coalesce(country, 'US') ;;\n }\n}", - "viewLanguage": "lookml" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view_explore,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view_explore,PROD),my_view_explore.country)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD),country)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view_explore,PROD),my_view_explore.city)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD),city)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view_explore,PROD),my_view_explore.country)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD),unique_countries)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view_explore,PROD),my_view_explore.is_latest)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD),derived_col)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "view_derived_explore", - "platform": "urn:li:dataPlatform:looker", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "country", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "city", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "unique_countries", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "count_distinct", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Measure" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "derived_col", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "sum", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Measure" - } - ] - }, - "isPartOfKey": false - } - ], - "primaryKeys": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "native_derived_table.view.lkml", - "looker.model": "data" - }, - "name": "view_derived_explore", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "view: flights {\n sql_table_name: flightstats.accidents ;;\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n}\n\n# override type of id parameter\nview: +flights {\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: string\n sql: ${TABLE}.id ;;\n }\n}", - "viewLanguage": "lookml" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.flightstats.accidents,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.flightstats.accidents,PROD),id)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD),id)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "flights", - "platform": "urn:li:dataPlatform:looker", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "nullable": false, - "description": "", - "label": "id", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "number", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": true - } - ], - "primaryKeys": [ - "id" - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "flights.view.lkml", - "looker.model": "data" - }, - "name": "flights", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Dimension", - "aspects": [ - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Dimension", - "description": "A tag that is applied to all dimension fields." - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Temporal", - "aspects": [ - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Temporal", - "description": "A tag that is applied to all time-based (temporal) fields such as timestamps or durations." - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Measure", - "aspects": [ - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Measure", - "description": "A tag that is applied to all measures (metrics). Measures are typically the columns that you aggregate on" - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:Dimension", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:Measure", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:Temporal", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index e4eb564e3e86b7..94b3b103d0548c 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -10,7 +10,6 @@ from freezegun import freeze_time from looker_sdk.sdk.api40.models import DBConnection -from datahub.configuration.common import PipelineExecutionError from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.source.file import read_metadata_file from datahub.ingestion.source.looker.looker_template_language import ( @@ -518,53 +517,6 @@ def ingestion_test( ) -@freeze_time(FROZEN_TIME) -def test_lookml_bad_sql_parser(pytestconfig, tmp_path, mock_time): - """Incorrect specification of sql parser should not fail ingestion""" - test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" - mce_out = "lookml_mces_badsql_parser.json" - pipeline = Pipeline.create( - { - "run_id": "lookml-test", - "source": { - "type": "lookml", - "config": { - "base_folder": str(test_resources_dir / "lkml_samples"), - "connection_to_platform_map": { - "my_connection": { - "platform": "snowflake", - "default_db": "default_db", - "default_schema": "default_schema", - } - }, - "parse_table_names_from_sql": True, - "project_name": "lkml_samples", - "sql_parser": "bad.sql.Parser", - "emit_reachable_views_only": False, - "process_refinements": False, - }, - }, - "sink": { - "type": "file", - "config": { - "filename": f"{tmp_path}/{mce_out}", - }, - }, - } - ) - pipeline.run() - pipeline.pretty_print_summary() - pipeline.raise_from_status(raise_warnings=False) - with pytest.raises(PipelineExecutionError): # we expect the source to have warnings - pipeline.raise_from_status(raise_warnings=True) - - mce_helpers.check_golden_file( - pytestconfig, - output_path=tmp_path / mce_out, - golden_path=test_resources_dir / mce_out, - ) - - @freeze_time(FROZEN_TIME) def test_lookml_git_info(pytestconfig, tmp_path, mock_time): """Add github info to config""" From 6b09346ca554783906afd8901ba9f89e1ef7310d Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 17 Oct 2024 08:14:06 -0700 Subject: [PATCH 31/40] feat(ingest): add offline flag to SQL parser CLI (#11635) --- .../src/datahub/cli/check_cli.py | 35 +++++++++++++++---- .../datahub/sql_parsing/sqlglot_lineage.py | 4 ++- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/src/datahub/cli/check_cli.py b/metadata-ingestion/src/datahub/cli/check_cli.py index 6e9bfddd350f92..39ed1b2bfea087 100644 --- a/metadata-ingestion/src/datahub/cli/check_cli.py +++ b/metadata-ingestion/src/datahub/cli/check_cli.py @@ -188,9 +188,13 @@ def sql_format(sql: str, platform: str) -> None: @click.option( "--sql", type=str, - required=True, help="The SQL query to parse", ) +@click.option( + "--sql-file", + type=click.Path(exists=True, dir_okay=False, readable=True), + help="The SQL file to parse", +) @click.option( "--platform", type=str, @@ -218,25 +222,44 @@ def sql_format(sql: str, platform: str) -> None: type=str, help="The default schema to use for unqualified table names", ) +@click.option( + "--online/--offline", + type=bool, + is_flag=True, + default=True, + help="Run in offline mode and disable schema-aware parsing.", +) @telemetry.with_telemetry() def sql_lineage( - sql: str, + sql: Optional[str], + sql_file: Optional[str], platform: str, default_db: Optional[str], default_schema: Optional[str], platform_instance: Optional[str], env: str, + online: bool, ) -> None: """Parse the lineage of a SQL query. - This performs schema-aware parsing in order to generate column-level lineage. - If the relevant tables are not in DataHub, this will be less accurate. + In online mode (the default), we perform schema-aware parsing in order to generate column-level lineage. + If offline mode is enabled or if the relevant tables are not in DataHub, this will be less accurate. """ - graph = get_default_graph() + from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result + + if sql is None: + if sql_file is None: + raise click.UsageError("Either --sql or --sql-file must be provided") + sql = pathlib.Path(sql_file).read_text() + + graph = None + if online: + graph = get_default_graph() - lineage = graph.parse_sql_lineage( + lineage = create_lineage_sql_parsed_result( sql, + graph=graph, platform=platform, platform_instance=platform_instance, env=env, diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py index 0806d0ec774fe7..273e9d0f9f0b1d 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py @@ -656,7 +656,9 @@ def _get_direct_raw_col_upstreams( # Parse the column name out of the node name. # Sqlglot calls .sql(), so we have to do the inverse. normalized_col = sqlglot.parse_one(node.name).this.name - if node.subfield: + if hasattr(node, "subfield") and node.subfield: + # The hasattr check is necessary, since it lets us be compatible with + # sqlglot versions that don't have the subfield attribute. normalized_col = f"{normalized_col}.{node.subfield}" direct_raw_col_upstreams.add( From 68cd17b34eb404421898cad4ab26ecab2564a637 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 17 Oct 2024 10:08:37 -0700 Subject: [PATCH 32/40] fix(ingest/redshift): reduce sequence limit for LISTAGG (#11621) Co-authored-by: treff7es Co-authored-by: Aseem Bansal --- .../ingestion/source/redshift/query.py | 281 +++++++++--------- .../unit/redshift/redshift_query_mocker.py | 40 +-- 2 files changed, 143 insertions(+), 178 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py index 39370b93b561c5..f7fad574f7fbe7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py @@ -4,6 +4,12 @@ redshift_datetime_format = "%Y-%m-%d %H:%M:%S" +# See https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext +# for why we need to limit the size of the query text. +# We use 290 instead instead of the standard 320, because escape characters can add to the length. +_QUERY_SEQUENCE_LIMIT = 290 + + class RedshiftCommonQuery: CREATE_TEMP_TABLE_CLAUSE = "create temp table" CREATE_TEMPORARY_TABLE_CLAUSE = "create temporary table" @@ -487,71 +493,70 @@ def list_unload_commands_sql( def list_insert_create_queries_sql( db_name: str, start_time: datetime, end_time: datetime ) -> str: - return """ - with query_txt as - ( - select - query, - pid, - LISTAGG(case - when LEN(RTRIM(text)) = 0 then text - else RTRIM(text) - end) within group ( - order by - sequence) as ddl - from - ( - select - query, - pid, - text, - sequence - from - STL_QUERYTEXT - where - sequence < 320 - order by - sequence - ) - group by - query, - pid - ) - select - distinct tbl as target_table_id, - sti.schema as target_schema, - sti.table as target_table, - sti.database as cluster, - usename as username, - ddl, - sq.query as query_id, - min(si.starttime) as timestamp, - ANY_VALUE(pid) as session_id - from - stl_insert as si - left join SVV_TABLE_INFO sti on - sti.table_id = tbl - left join svl_user_info sui on - si.userid = sui.usesysid - left join query_txt sq on - si.query = sq.query - left join stl_load_commits slc on - slc.query = si.query - where - sui.usename <> 'rdsdb' - and cluster = '{db_name}' - and slc.query IS NULL - and si.starttime >= '{start_time}' - and si.starttime < '{end_time}' - group by - target_table_id, - target_schema, - target_table, - cluster, - username, - ddl, - sq.query + return """\ +with query_txt as ( + select + query, + pid, + LISTAGG(case + when LEN(RTRIM(text)) = 0 then text + else RTRIM(text) + end) within group ( + order by sequence + ) as ddl + from ( + select + query, + pid, + text, + sequence + from + STL_QUERYTEXT + where + sequence < {_QUERY_SEQUENCE_LIMIT} + order by + sequence + ) + group by + query, + pid +) +select + distinct tbl as target_table_id, + sti.schema as target_schema, + sti.table as target_table, + sti.database as cluster, + usename as username, + ddl, + sq.query as query_id, + min(si.starttime) as timestamp, + ANY_VALUE(pid) as session_id +from + stl_insert as si +left join SVV_TABLE_INFO sti on + sti.table_id = tbl +left join svl_user_info sui on + si.userid = sui.usesysid +left join query_txt sq on + si.query = sq.query +left join stl_load_commits slc on + slc.query = si.query +where + sui.usename <> 'rdsdb' + and cluster = '{db_name}' + and slc.query IS NULL + and si.starttime >= '{start_time}' + and si.starttime < '{end_time}' +group by + target_table_id, + target_schema, + target_table, + cluster, + username, + ddl, + sq.query """.format( + _QUERY_SEQUENCE_LIMIT=_QUERY_SEQUENCE_LIMIT, # We need the original database name for filtering db_name=db_name, start_time=start_time.strftime(redshift_datetime_format), @@ -564,84 +569,82 @@ def temp_table_ddl_query(start_time: datetime, end_time: datetime) -> str: end_time_str: str = end_time.strftime(redshift_datetime_format) - return rf"""-- DataHub Redshift Source temp table DDL query + return rf"""\ +-- DataHub Redshift Source temp table DDL query +select + * +from ( + select + session_id, + transaction_id, + start_time, + userid, + REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\n','\\n'), '(CREATE(?:[\\n\\s\\t]+(?:temp|temporary))?(?:[\\n\\s\\t]+)table(?:[\\n\\s\\t]+)[^\\n\\s\\t()-]+)', 0, 1, 'ipe'),'[\\n\\s\\t]+',' ',1,'p') as create_command, + query_text, + row_number() over ( + partition by session_id, TRIM(query_text) + order by start_time desc + ) rn + from ( + select + pid as session_id, + xid as transaction_id, + starttime as start_time, + type, + query_text, + userid + from ( select - * + starttime, + pid, + xid, + type, + userid, + LISTAGG(case + when LEN(RTRIM(text)) = 0 then text + else RTRIM(text) + end, + '') within group ( + order by sequence + ) as query_text from - ( - select - session_id, - transaction_id, - start_time, - userid, - REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\n','\\n'), '(CREATE(?:[\\n\\s\\t]+(?:temp|temporary))?(?:[\\n\\s\\t]+)table(?:[\\n\\s\\t]+)[^\\n\\s\\t()-]+)', 0, 1, 'ipe'),'[\\n\\s\\t]+',' ',1,'p') as create_command, - query_text, - row_number() over ( - partition by session_id, TRIM(query_text) - order by start_time desc - ) rn - from - ( - select - pid as session_id, - xid as transaction_id, - starttime as start_time, - type, - query_text, - userid - from - ( - select - starttime, - pid, - xid, - type, - userid, - LISTAGG(case - when LEN(RTRIM(text)) = 0 then text - else RTRIM(text) - end, - '') within group ( - order by sequence - ) as query_text - from - SVL_STATEMENTTEXT - where - type in ('DDL', 'QUERY') - AND starttime >= '{start_time_str}' - AND starttime < '{end_time_str}' - -- See https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext - AND sequence < 320 - group by - starttime, - pid, - xid, - type, - userid - order by - starttime, - pid, - xid, - type, - userid - asc) - where - type in ('DDL', 'QUERY') - ) - where - (create_command ilike 'create temp table %' - or create_command ilike 'create temporary table %' - -- we want to get all the create table statements and not just temp tables if non temp table is created and dropped in the same transaction - or create_command ilike 'create table %') - -- Redshift creates temp tables with the following names: volt_tt_%. We need to filter them out. - and query_text not ilike 'CREATE TEMP TABLE volt_tt_%' - and create_command not like 'CREATE TEMP TABLE volt_tt_' - -- We need to filter out our query and it was not possible earlier when we did not have any comment in the query - and query_text not ilike '%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext%' - - ) + SVL_STATEMENTTEXT where - rn = 1 + type in ('DDL', 'QUERY') + AND starttime >= '{start_time_str}' + AND starttime < '{end_time_str}' + AND sequence < {_QUERY_SEQUENCE_LIMIT} + group by + starttime, + pid, + xid, + type, + userid + order by + starttime, + pid, + xid, + type, + userid + asc + ) + where + type in ('DDL', 'QUERY') + ) + where + (create_command ilike 'create temp table %' + or create_command ilike 'create temporary table %' + -- we want to get all the create table statements and not just temp tables if non temp table is created and dropped in the same transaction + or create_command ilike 'create table %') + -- Redshift creates temp tables with the following names: volt_tt_%. We need to filter them out. + and query_text not ilike 'CREATE TEMP TABLE volt_tt_%' + and create_command not like 'CREATE TEMP TABLE volt_tt_' + -- We need to filter out our query and it was not possible earlier when we did not have any comment in the query + and query_text not ilike '%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext%' + +) +where + rn = 1 """ # Add this join to the sql query for more metrics on completed queries diff --git a/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py b/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py index ada76e624032ba..06b592d42914bf 100644 --- a/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py +++ b/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py @@ -56,45 +56,7 @@ def mock_stl_insert_table_cursor(cursor: MagicMock) -> None: query_vs_cursor_mocker = { ( - "-- DataHub Redshift Source temp table DDL query\n select\n *\n " - "from\n (\n select\n session_id,\n " - " transaction_id,\n start_time,\n userid,\n " - " REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\\\\\n','\\\\n'), '(CREATE(?:[" - "\\\\n\\\\s\\\\t]+(?:temp|temporary))?(?:[\\\\n\\\\s\\\\t]+)table(?:[\\\\n\\\\s\\\\t]+)[" - "^\\\\n\\\\s\\\\t()-]+)', 0, 1, 'ipe'),'[\\\\n\\\\s\\\\t]+',' ',1,'p') as create_command,\n " - " query_text,\n row_number() over (\n partition " - "by session_id, TRIM(query_text)\n order by start_time desc\n ) rn\n " - " from\n (\n select\n pid " - "as session_id,\n xid as transaction_id,\n starttime " - "as start_time,\n type,\n query_text,\n " - " userid\n from\n (\n " - "select\n starttime,\n pid,\n " - " xid,\n type,\n userid,\n " - " LISTAGG(case\n when LEN(RTRIM(text)) = 0 then text\n " - " else RTRIM(text)\n end,\n " - " '') within group (\n order by sequence\n " - " ) as query_text\n from\n " - "SVL_STATEMENTTEXT\n where\n type in ('DDL', " - "'QUERY')\n AND starttime >= '2024-01-01 12:00:00'\n " - " AND starttime < '2024-01-10 12:00:00'\n -- See " - "https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl" - "-statementtext\n AND sequence < 320\n group by\n " - " starttime,\n pid,\n " - "xid,\n type,\n userid\n " - " order by\n starttime,\n pid,\n " - " xid,\n type,\n userid\n " - " asc)\n where\n type in ('DDL', " - "'QUERY')\n )\n where\n (create_command ilike " - "'create temp table %'\n or create_command ilike 'create temporary table %'\n " - " -- we want to get all the create table statements and not just temp tables " - "if non temp table is created and dropped in the same transaction\n or " - "create_command ilike 'create table %')\n -- Redshift creates temp tables with " - "the following names: volt_tt_%. We need to filter them out.\n and query_text not " - "ilike 'CREATE TEMP TABLE volt_tt_%'\n and create_command not like 'CREATE TEMP " - "TABLE volt_tt_'\n -- We need to filter out our query and it was not possible " - "earlier when we did not have any comment in the query\n and query_text not ilike " - "'%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl" - "-statementtext%'\n\n )\n where\n rn = 1\n " + "\\\n-- DataHub Redshift Source temp table DDL query\nselect\n *\nfrom (\n select\n session_id,\n transaction_id,\n start_time,\n userid,\n REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\\\\\n','\\\\n'), '(CREATE(?:[\\\\n\\\\s\\\\t]+(?:temp|temporary))?(?:[\\\\n\\\\s\\\\t]+)table(?:[\\\\n\\\\s\\\\t]+)[^\\\\n\\\\s\\\\t()-]+)', 0, 1, 'ipe'),'[\\\\n\\\\s\\\\t]+',' ',1,'p') as create_command,\n query_text,\n row_number() over (\n partition by session_id, TRIM(query_text)\n order by start_time desc\n ) rn\n from (\n select\n pid as session_id,\n xid as transaction_id,\n starttime as start_time,\n type,\n query_text,\n userid\n from (\n select\n starttime,\n pid,\n xid,\n type,\n userid,\n LISTAGG(case\n when LEN(RTRIM(text)) = 0 then text\n else RTRIM(text)\n end,\n '') within group (\n order by sequence\n ) as query_text\n from\n SVL_STATEMENTTEXT\n where\n type in ('DDL', 'QUERY')\n AND starttime >= '2024-01-01 12:00:00'\n AND starttime < '2024-01-10 12:00:00'\n AND sequence < 290\n group by\n starttime,\n pid,\n xid,\n type,\n userid\n order by\n starttime,\n pid,\n xid,\n type,\n userid\n asc\n )\n where\n type in ('DDL', 'QUERY')\n )\n where\n (create_command ilike 'create temp table %'\n or create_command ilike 'create temporary table %'\n -- we want to get all the create table statements and not just temp tables if non temp table is created and dropped in the same transaction\n or create_command ilike 'create table %')\n -- Redshift creates temp tables with the following names: volt_tt_%. We need to filter them out.\n and query_text not ilike 'CREATE TEMP TABLE volt_tt_%'\n and create_command not like 'CREATE TEMP TABLE volt_tt_'\n -- We need to filter out our query and it was not possible earlier when we did not have any comment in the query\n and query_text not ilike '%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext%'\n\n)\nwhere\n rn = 1\n " ): mock_temp_table_cursor, "select * from test_collapse_temp_lineage": mock_stl_insert_table_cursor, } From cf3634a6a9233b98d45560beb8cbabf6f17ff166 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Thu, 17 Oct 2024 19:57:02 +0200 Subject: [PATCH 33/40] fix(ingest/bigquery): Not setting platform instance for bigquery platform resources (#11659) --- .../ingestion/source/bigquery_v2/bigquery.py | 3 - .../bigquery_platform_resource_helper.py | 2 +- .../bigquery_v2/bigquery_mcp_golden.json | 90 +++++++++---------- 3 files changed, 45 insertions(+), 50 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index c30dade921d257..3c6202cc7cbfaf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -65,9 +65,6 @@ # We can't use close as it is not called if the ingestion is not successful def cleanup(config: BigQueryV2Config) -> None: if config._credentials_path is not None: - logger.debug( - f"Deleting temporary credential file at {config._credentials_path}" - ) os.unlink(config._credentials_path) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py index d2da895be985dc..9da2aceb19220a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py @@ -42,7 +42,7 @@ def platform_resource_key(self) -> PlatformResourceKey: return PlatformResourceKey( platform="bigquery", resource_type="BigQueryLabelInfo", - platform_instance=self.project, + platform_instance=None, primary_key=self.label.primary_key(), ) diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json index b268926f155b74..640ee1bf436b03 100644 --- a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json +++ b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json @@ -201,7 +201,7 @@ }, { "entityType": "platformResource", - "entityUrn": "urn:li:platformResource:79d443a7956814fdab2168e11392bbf2", + "entityUrn": "urn:li:platformResource:11f75a60f0dbd414676852e46a45e39b", "changeType": "UPSERT", "aspectName": "platformResourceInfo", "aspect": { @@ -221,30 +221,29 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "bigquery-2022_02_03-07_00_00", + "runId": "bigquery-2022_02_03-07_00_00-2j2qqv", "lastRunId": "no-run-id-provided" } }, { "entityType": "platformResource", - "entityUrn": "urn:li:platformResource:79d443a7956814fdab2168e11392bbf2", + "entityUrn": "urn:li:platformResource:11f75a60f0dbd414676852e46a45e39b", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:bigquery", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:bigquery,project-id-1)" + "platform": "urn:li:dataPlatform:bigquery" } }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "bigquery-2022_02_03-07_00_00", + "runId": "bigquery-2022_02_03-07_00_00-2j2qqv", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", + "entityType": "platformResource", + "entityUrn": "urn:li:platformResource:11f75a60f0dbd414676852e46a45e39b", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -254,13 +253,13 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "bigquery-2022_02_03-07_00_00", + "runId": "bigquery-2022_02_03-07_00_00-2j2qqv", "lastRunId": "no-run-id-provided" } }, { "entityType": "platformResource", - "entityUrn": "urn:li:platformResource:0a8c87e84bd90486c4fd57bbae6557e3", + "entityUrn": "urn:li:platformResource:99b34051bd90d28d922b0e107277a916", "changeType": "UPSERT", "aspectName": "platformResourceInfo", "aspect": { @@ -280,19 +279,50 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "bigquery-2022_02_03-07_00_00", + "runId": "bigquery-2022_02_03-07_00_00-2j2qqv", "lastRunId": "no-run-id-provided" } }, { "entityType": "platformResource", - "entityUrn": "urn:li:platformResource:0a8c87e84bd90486c4fd57bbae6557e3", + "entityUrn": "urn:li:platformResource:99b34051bd90d28d922b0e107277a916", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:bigquery", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:bigquery,project-id-1)" + "platform": "urn:li:dataPlatform:bigquery" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00-2j2qqv", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "platformResource", + "entityUrn": "urn:li:platformResource:99b34051bd90d28d922b0e107277a916", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00-2j2qqv", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } }, "systemMetadata": { @@ -395,38 +425,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "platformResource", - "entityUrn": "urn:li:platformResource:79d443a7956814fdab2168e11392bbf2", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "bigquery-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "platformResource", - "entityUrn": "urn:li:platformResource:0a8c87e84bd90486c4fd57bbae6557e3", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "bigquery-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", From 6f193221c957b4adcc0f15b1e0af0679c1665272 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 17 Oct 2024 11:50:42 -0700 Subject: [PATCH 34/40] fix(ingest/dbt): fix bug in CLL pruning (#11614) --- .../src/datahub/ingestion/source/dbt/dbt_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index c15f1deb43a3af..4cd3c934ce6348 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -1074,8 +1074,8 @@ def add_node_to_cll_list(dbt_name: str) -> None: for upstream in all_nodes_map[dbt_name].upstream_nodes: schema_nodes.add(upstream) - upstream_node = all_nodes_map[upstream] - if upstream_node.is_ephemeral_model(): + upstream_node = all_nodes_map.get(upstream) + if upstream_node and upstream_node.is_ephemeral_model(): add_node_to_cll_list(upstream) cll_nodes.add(dbt_name) From 523a456f5c9f166c5e99a9cdb33d9b421e199505 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 17 Oct 2024 12:33:04 -0700 Subject: [PATCH 35/40] fix(ingest/redshift): fix syntax error in temp sql (#11661) --- .../src/datahub/ingestion/source/redshift/exception.py | 8 ++++---- .../src/datahub/ingestion/source/redshift/query.py | 5 ++--- .../tests/unit/redshift/redshift_query_mocker.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/exception.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/exception.py index 43ad5bfcefdf1b..ed0856fc1e2924 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/exception.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/exception.py @@ -40,25 +40,25 @@ def report_redshift_failure( error_message = str(e).lower() if "permission denied" in error_message: if "svv_table_info" in error_message: - report.report_failure( + report.failure( title="Permission denied", message="Failed to extract metadata due to insufficient permission to access 'svv_table_info' table. Please ensure the provided database user has access.", exc=e, ) elif "svl_user_info" in error_message: - report.report_failure( + report.failure( title="Permission denied", message="Failed to extract metadata due to insufficient permission to access 'svl_user_info' table. Please ensure the provided database user has access.", exc=e, ) else: - report.report_failure( + report.failure( title="Permission denied", message="Failed to extract metadata due to insufficient permissions.", exc=e, ) else: - report.report_failure( + report.failure( title="Failed to extract some metadata", message="Failed to extract some metadata from Redshift.", exc=e, diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py index f7fad574f7fbe7..b18b526ef30fce 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py @@ -569,8 +569,7 @@ def temp_table_ddl_query(start_time: datetime, end_time: datetime) -> str: end_time_str: str = end_time.strftime(redshift_datetime_format) - return rf"""\ --- DataHub Redshift Source temp table DDL query + return rf"""-- DataHub Redshift Source temp table DDL query select * from ( @@ -645,7 +644,7 @@ def temp_table_ddl_query(start_time: datetime, end_time: datetime) -> str: ) where rn = 1 - """ +""" # Add this join to the sql query for more metrics on completed queries # LEFT JOIN svl_query_metrics_summary sqms ON ss.query = sqms.query diff --git a/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py b/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py index 06b592d42914bf..10c0250e37e37f 100644 --- a/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py +++ b/metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py @@ -56,7 +56,7 @@ def mock_stl_insert_table_cursor(cursor: MagicMock) -> None: query_vs_cursor_mocker = { ( - "\\\n-- DataHub Redshift Source temp table DDL query\nselect\n *\nfrom (\n select\n session_id,\n transaction_id,\n start_time,\n userid,\n REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\\\\\n','\\\\n'), '(CREATE(?:[\\\\n\\\\s\\\\t]+(?:temp|temporary))?(?:[\\\\n\\\\s\\\\t]+)table(?:[\\\\n\\\\s\\\\t]+)[^\\\\n\\\\s\\\\t()-]+)', 0, 1, 'ipe'),'[\\\\n\\\\s\\\\t]+',' ',1,'p') as create_command,\n query_text,\n row_number() over (\n partition by session_id, TRIM(query_text)\n order by start_time desc\n ) rn\n from (\n select\n pid as session_id,\n xid as transaction_id,\n starttime as start_time,\n type,\n query_text,\n userid\n from (\n select\n starttime,\n pid,\n xid,\n type,\n userid,\n LISTAGG(case\n when LEN(RTRIM(text)) = 0 then text\n else RTRIM(text)\n end,\n '') within group (\n order by sequence\n ) as query_text\n from\n SVL_STATEMENTTEXT\n where\n type in ('DDL', 'QUERY')\n AND starttime >= '2024-01-01 12:00:00'\n AND starttime < '2024-01-10 12:00:00'\n AND sequence < 290\n group by\n starttime,\n pid,\n xid,\n type,\n userid\n order by\n starttime,\n pid,\n xid,\n type,\n userid\n asc\n )\n where\n type in ('DDL', 'QUERY')\n )\n where\n (create_command ilike 'create temp table %'\n or create_command ilike 'create temporary table %'\n -- we want to get all the create table statements and not just temp tables if non temp table is created and dropped in the same transaction\n or create_command ilike 'create table %')\n -- Redshift creates temp tables with the following names: volt_tt_%. We need to filter them out.\n and query_text not ilike 'CREATE TEMP TABLE volt_tt_%'\n and create_command not like 'CREATE TEMP TABLE volt_tt_'\n -- We need to filter out our query and it was not possible earlier when we did not have any comment in the query\n and query_text not ilike '%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext%'\n\n)\nwhere\n rn = 1\n " + "-- DataHub Redshift Source temp table DDL query\nselect\n *\nfrom (\n select\n session_id,\n transaction_id,\n start_time,\n userid,\n REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\\\\\n','\\\\n'), '(CREATE(?:[\\\\n\\\\s\\\\t]+(?:temp|temporary))?(?:[\\\\n\\\\s\\\\t]+)table(?:[\\\\n\\\\s\\\\t]+)[^\\\\n\\\\s\\\\t()-]+)', 0, 1, 'ipe'),'[\\\\n\\\\s\\\\t]+',' ',1,'p') as create_command,\n query_text,\n row_number() over (\n partition by session_id, TRIM(query_text)\n order by start_time desc\n ) rn\n from (\n select\n pid as session_id,\n xid as transaction_id,\n starttime as start_time,\n type,\n query_text,\n userid\n from (\n select\n starttime,\n pid,\n xid,\n type,\n userid,\n LISTAGG(case\n when LEN(RTRIM(text)) = 0 then text\n else RTRIM(text)\n end,\n '') within group (\n order by sequence\n ) as query_text\n from\n SVL_STATEMENTTEXT\n where\n type in ('DDL', 'QUERY')\n AND starttime >= '2024-01-01 12:00:00'\n AND starttime < '2024-01-10 12:00:00'\n AND sequence < 290\n group by\n starttime,\n pid,\n xid,\n type,\n userid\n order by\n starttime,\n pid,\n xid,\n type,\n userid\n asc\n )\n where\n type in ('DDL', 'QUERY')\n )\n where\n (create_command ilike 'create temp table %'\n or create_command ilike 'create temporary table %'\n -- we want to get all the create table statements and not just temp tables if non temp table is created and dropped in the same transaction\n or create_command ilike 'create table %')\n -- Redshift creates temp tables with the following names: volt_tt_%. We need to filter them out.\n and query_text not ilike 'CREATE TEMP TABLE volt_tt_%'\n and create_command not like 'CREATE TEMP TABLE volt_tt_'\n -- We need to filter out our query and it was not possible earlier when we did not have any comment in the query\n and query_text not ilike '%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext%'\n\n)\nwhere\n rn = 1\n" ): mock_temp_table_cursor, "select * from test_collapse_temp_lineage": mock_stl_insert_table_cursor, } From d7ccf3b2c34fa53085678ab635c733edf8d9fc31 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 17 Oct 2024 16:36:18 -0700 Subject: [PATCH 36/40] docs(airflow): add known limitations for automatic lineage (#11652) --- docs/lineage/airflow.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/lineage/airflow.md b/docs/lineage/airflow.md index 35f2ff862e6958..829c048a8f8e24 100644 --- a/docs/lineage/airflow.md +++ b/docs/lineage/airflow.md @@ -132,7 +132,7 @@ conn_id = datahub_rest_default # or datahub_kafka_default ``` | Name | Default value | Description | -|----------------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -------------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | enabled | true | If the plugin should be enabled. | | conn_id | datahub_rest_default | The name of the datahub connection you set in step 1. | | cluster | prod | name of the airflow cluster | @@ -191,6 +191,10 @@ These operators are supported by OpenLineage, but we haven't tested them yet: There's also a few operators (e.g. BashOperator, PythonOperator) that have custom extractors, but those extractors don't generate lineage. --> +Known limitations: + +- We do not fully support operators that run multiple SQL statements at once. In these cases, we'll only capture lineage from the first SQL statement. + ## Manual Lineage Annotation ### Using `inlets` and `outlets` From 6e3724b2daaeeebd1895fc43fda4956827112af3 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 17 Oct 2024 17:50:59 -0700 Subject: [PATCH 37/40] perf(ingest): streamline CLL generation (#11645) --- metadata-ingestion/setup.py | 2 +- .../ingestion/source/redshift/lineage.py | 2 +- .../ingestion/source/tableau/tableau.py | 2 +- .../datahub/sql_parsing/sqlglot_lineage.py | 51 ++++++++++------- .../tableau/test_tableau_ingest.py | 17 ++++-- ...st_bigquery_subquery_column_inference.json | 57 +++++++++++++++++++ .../unit/sql_parsing/test_sqlglot_lineage.py | 15 +++++ 7 files changed, 118 insertions(+), 28 deletions(-) create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_subquery_column_inference.json diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index bfec2c00cb8647..365da21208ecce 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -101,7 +101,7 @@ sqlglot_lib = { # Using an Acryl fork of sqlglot. # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:main?expand=1 - "acryl-sqlglot[rs]==25.20.2.dev6", + "acryl-sqlglot[rs]==25.25.2.dev9", } classification_lib = { diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py index 0e4cb6f1599e52..fe491ccb318505 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py @@ -133,7 +133,7 @@ def parse_alter_table_rename(default_schema: str, query: str) -> Tuple[str, str, assert isinstance(parsed_query, sqlglot.exp.Alter) prev_name = parsed_query.this.name rename_clause = parsed_query.args["actions"][0] - assert isinstance(rename_clause, sqlglot.exp.RenameTable) + assert isinstance(rename_clause, sqlglot.exp.AlterRename) new_name = rename_clause.this.name schema = parsed_query.this.db or default_schema diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 9f011790990ec2..2c17a5a322f052 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -2131,7 +2131,7 @@ def _create_lineage_from_unsupported_csql( fine_grained_lineages: List[FineGrainedLineage] = [] if self.config.extract_column_level_lineage: - logger.info("Extracting CLL from custom sql") + logger.debug("Extracting CLL from custom sql") fine_grained_lineages = make_fine_grained_lineage_class( parsed_result, csql_urn, out_columns ) diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py index 273e9d0f9f0b1d..6a7ff5be6d1ea8 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py @@ -1,6 +1,5 @@ import dataclasses import functools -import itertools import logging import traceback from collections import defaultdict @@ -14,6 +13,8 @@ import sqlglot.optimizer.annotate_types import sqlglot.optimizer.optimizer import sqlglot.optimizer.qualify +import sqlglot.optimizer.qualify_columns +import sqlglot.optimizer.unnest_subqueries from datahub.cli.env_utils import get_boolean_env_variable from datahub.ingestion.graph.client import DataHubGraph @@ -63,24 +64,30 @@ SQL_LINEAGE_TIMEOUT_SECONDS = 10 -RULES_BEFORE_TYPE_ANNOTATION: tuple = tuple( - filter( - lambda func: func.__name__ - not in { - # Skip pushdown_predicates because it sometimes throws exceptions, and we - # don't actually need it for anything. - "pushdown_predicates", - # Skip normalize because it can sometimes be expensive. - "normalize", - }, - itertools.takewhile( - lambda func: func != sqlglot.optimizer.annotate_types.annotate_types, - sqlglot.optimizer.optimizer.RULES, - ), - ) +# These rules are a subset of the rules in sqlglot.optimizer.optimizer.RULES. +# If there's a change in their rules, we probably need to re-evaluate our list as well. +assert len(sqlglot.optimizer.optimizer.RULES) == 14 + +_OPTIMIZE_RULES = ( + sqlglot.optimizer.optimizer.qualify, + # We need to enable this in order for annotate types to work. + sqlglot.optimizer.optimizer.pushdown_projections, + # sqlglot.optimizer.optimizer.normalize, # causes perf issues + sqlglot.optimizer.optimizer.unnest_subqueries, + # sqlglot.optimizer.optimizer.pushdown_predicates, # causes perf issues + # sqlglot.optimizer.optimizer.optimize_joins, + # sqlglot.optimizer.optimizer.eliminate_subqueries, + # sqlglot.optimizer.optimizer.merge_subqueries, + # sqlglot.optimizer.optimizer.eliminate_joins, + # sqlglot.optimizer.optimizer.eliminate_ctes, + sqlglot.optimizer.optimizer.quote_identifiers, + # These three are run separately or not run at all. + # sqlglot.optimizer.optimizer.annotate_types, + # sqlglot.optimizer.canonicalize.canonicalize, + # sqlglot.optimizer.simplify.simplify, ) -# Quick check that the rules were loaded correctly. -assert 0 < len(RULES_BEFORE_TYPE_ANNOTATION) < len(sqlglot.optimizer.optimizer.RULES) + +_DEBUG_TYPE_ANNOTATIONS = False class _ColumnRef(_FrozenModel): @@ -385,11 +392,12 @@ def _sqlglot_force_column_normalizer( schema=sqlglot_db_schema, qualify_columns=True, validate_qualify_columns=False, + allow_partial_qualification=True, identify=True, # sqlglot calls the db -> schema -> table hierarchy "catalog", "db", "table". catalog=default_db, db=default_schema, - rules=RULES_BEFORE_TYPE_ANNOTATION, + rules=_OPTIMIZE_RULES, ) except (sqlglot.errors.OptimizeError, ValueError) as e: raise SqlUnderstandingError( @@ -408,6 +416,10 @@ def _sqlglot_force_column_normalizer( except (sqlglot.errors.OptimizeError, sqlglot.errors.ParseError) as e: # This is not a fatal error, so we can continue. logger.debug("sqlglot failed to annotate or parse types: %s", e) + if _DEBUG_TYPE_ANNOTATIONS and logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Type annotated sql %s", statement.sql(pretty=True, dialect=dialect) + ) return statement, _ColumnResolver( sqlglot_db_schema=sqlglot_db_schema, @@ -907,6 +919,7 @@ def _sqlglot_lineage_inner( # At this stage we only want to qualify the table names. The columns will be dealt with later. qualify_columns=False, validate_qualify_columns=False, + allow_partial_qualification=True, # Only insert quotes where necessary. identify=False, ) diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index 5a5552a78c56fa..3798df07000c8a 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -19,6 +19,7 @@ from datahub.configuration.source_common import DEFAULT_ENV from datahub.emitter.mce_builder import make_schema_field_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.run.pipeline import Pipeline, PipelineContext from datahub.ingestion.source.tableau.tableau import ( TableauConfig, @@ -37,7 +38,7 @@ FineGrainedLineageUpstreamType, UpstreamLineage, ) -from datahub.metadata.schema_classes import MetadataChangeProposalClass, UpstreamClass +from datahub.metadata.schema_classes import UpstreamClass from tests.test_helpers import mce_helpers, test_connection_helpers from tests.test_helpers.state_helpers import ( get_current_checkpoint_from_pipeline, @@ -939,11 +940,12 @@ def test_tableau_unsupported_csql(): database_override_map={"production database": "prod"} ) - def test_lineage_metadata( + def check_lineage_metadata( lineage, expected_entity_urn, expected_upstream_table, expected_cll ): - mcp = cast(MetadataChangeProposalClass, next(iter(lineage)).metadata) - assert mcp.aspect == UpstreamLineage( + mcp = cast(MetadataChangeProposalWrapper, list(lineage)[0].metadata) + + expected = UpstreamLineage( upstreams=[ UpstreamClass( dataset=expected_upstream_table, @@ -966,6 +968,9 @@ def test_lineage_metadata( ) assert mcp.entityUrn == expected_entity_urn + actual_aspect = mcp.aspect + assert actual_aspect == expected + csql_urn = "urn:li:dataset:(urn:li:dataPlatform:tableau,09988088-05ad-173c-a2f1-f33ba3a13d1a,PROD)" expected_upstream_table = "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_bigquery_project.invent_dw.UserDetail,PROD)" expected_cll = { @@ -996,7 +1001,7 @@ def test_lineage_metadata( }, out_columns=[], ) - test_lineage_metadata( + check_lineage_metadata( lineage=lineage, expected_entity_urn=csql_urn, expected_upstream_table=expected_upstream_table, @@ -1014,7 +1019,7 @@ def test_lineage_metadata( }, out_columns=[], ) - test_lineage_metadata( + check_lineage_metadata( lineage=lineage, expected_entity_urn=csql_urn, expected_upstream_table=expected_upstream_table, diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_subquery_column_inference.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_subquery_column_inference.json new file mode 100644 index 00000000000000..32b3830bf1412a --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_subquery_column_inference.json @@ -0,0 +1,57 @@ +{ + "query_type": "SELECT", + "query_type_props": {}, + "query_fingerprint": "4094ebd230c1d47c7e6879b05ab927e550923b1986eb58c5f3814396cf401d18", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,invent_dw.UserDetail,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "user_id", + "column_type": null, + "native_column_type": null + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,invent_dw.UserDetail,PROD)", + "column": "user_id" + } + ] + }, + { + "downstream": { + "table": null, + "column": "source", + "column_type": null, + "native_column_type": null + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,invent_dw.UserDetail,PROD)", + "column": "source" + } + ] + }, + { + "downstream": { + "table": null, + "column": "user_source", + "column_type": null, + "native_column_type": null + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,invent_dw.UserDetail,PROD)", + "column": "user_source" + } + ] + } + ], + "debug_info": { + "confidence": 0.2, + "generalized_statement": "SELECT user_id, source, user_source FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY __partition_day DESC) AS rank_ FROM invent_dw.UserDetail) AS source_user WHERE rank_ = ?" + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py index fb1d2a0bc50110..90cc863d6bd231 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py @@ -1253,3 +1253,18 @@ def test_snowflake_drop_schema() -> None: dialect="snowflake", expected_file=RESOURCE_DIR / "test_snowflake_drop_schema.json", ) + + +def test_bigquery_subquery_column_inference() -> None: + assert_sql_result( + """\ +SELECT user_id, source, user_source +FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY __partition_day DESC) AS rank_ + FROM invent_dw.UserDetail +) source_user +WHERE rank_ = 1 +""", + dialect="bigquery", + expected_file=RESOURCE_DIR / "test_bigquery_subquery_column_inference.json", + ) From 179a6714a619a4990c9845eb1997649d94ab53cb Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Fri, 18 Oct 2024 10:33:28 +0530 Subject: [PATCH 38/40] feat(ingest): ensure sqlite file delete on clean exit (#11612) --- .../ingestion/source/bigquery_v2/bigquery.py | 9 +- .../source/bigquery_v2/bigquery_queries.py | 4 + .../source/bigquery_v2/queries_extractor.py | 18 +++- .../ingestion/source/redshift/lineage_v2.py | 7 +- .../ingestion/source/redshift/redshift.py | 23 +++-- .../source/snowflake/snowflake_connection.py | 3 +- .../source/snowflake/snowflake_queries.py | 64 ++++++++----- .../source/snowflake/snowflake_v2.py | 94 ++++++++++--------- .../sql_parsing/sql_parsing_aggregator.py | 7 ++ .../bigquery_v2/test_bigquery_queries.py | 16 ++++ .../snowflake/test_snowflake_queries.py | 24 +++++ .../unit/sql_parsing/test_sql_aggregator.py | 22 +++++ 12 files changed, 203 insertions(+), 88 deletions(-) create mode 100644 metadata-ingestion/tests/integration/snowflake/test_snowflake_queries.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index 3c6202cc7cbfaf..0e986acc81add8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -272,7 +272,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.report.set_ingestion_stage("*", QUERIES_EXTRACTION) - queries_extractor = BigQueryQueriesExtractor( + with BigQueryQueriesExtractor( connection=self.config.get_bigquery_client(), schema_api=self.bq_schema_extractor.schema_api, config=BigQueryQueriesExtractorConfig( @@ -288,9 +288,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: identifiers=self.identifiers, schema_resolver=self.sql_parser_schema_resolver, discovered_tables=self.bq_schema_extractor.table_refs, - ) - self.report.queries_extractor = queries_extractor.report - yield from queries_extractor.get_workunits_internal() + ) as queries_extractor: + self.report.queries_extractor = queries_extractor.report + yield from queries_extractor.get_workunits_internal() + else: if self.config.include_usage_statistics: yield from self.usage_extractor.get_usage_workunits( diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_queries.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_queries.py index ed27aae19ce963..47f21c9f32353a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_queries.py @@ -88,3 +88,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: def get_report(self) -> BigQueryQueriesSourceReport: return self.report + + def close(self) -> None: + self.queries_extractor.close() + self.connection.close() diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py index b4a443673b9a97..afaaaf51964f8e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py @@ -13,6 +13,7 @@ BaseTimeWindowConfig, get_time_bucket, ) +from datahub.ingestion.api.closeable import Closeable from datahub.ingestion.api.source import SourceReport from datahub.ingestion.api.source_helpers import auto_workunit from datahub.ingestion.api.workunit import MetadataWorkUnit @@ -114,7 +115,7 @@ class BigQueryQueriesExtractorConfig(BigQueryBaseConfig): ) -class BigQueryQueriesExtractor: +class BigQueryQueriesExtractor(Closeable): """ Extracts query audit log and generates usage/lineage/operation workunits. @@ -181,6 +182,7 @@ def __init__( is_allowed_table=self.is_allowed_table, format_queries=False, ) + self.report.sql_aggregator = self.aggregator.report self.report.num_discovered_tables = ( len(self.discovered_tables) if self.discovered_tables else None @@ -273,12 +275,14 @@ def get_workunits_internal( self.report.num_unique_queries = len(queries_deduped) logger.info(f"Found {self.report.num_unique_queries} unique queries") - with self.report.audit_log_load_timer: + with self.report.audit_log_load_timer, queries_deduped: i = 0 for _, query_instances in queries_deduped.items(): for query in query_instances.values(): if i > 0 and i % 10000 == 0: - logger.info(f"Added {i} query log entries to SQL aggregator") + logger.info( + f"Added {i} query log equeries_dedupedntries to SQL aggregator" + ) if self.report.sql_aggregator: logger.info(self.report.sql_aggregator.as_string()) @@ -287,6 +291,11 @@ def get_workunits_internal( yield from auto_workunit(self.aggregator.gen_metadata()) + if not use_cached_audit_log: + queries.close() + shared_connection.close() + audit_log_file.unlink(missing_ok=True) + def deduplicate_queries( self, queries: FileBackedList[ObservedQuery] ) -> FileBackedDict[Dict[int, ObservedQuery]]: @@ -404,6 +413,9 @@ def _parse_audit_log_row(self, row: BigQueryJob) -> ObservedQuery: return entry + def close(self) -> None: + self.aggregator.close() + def _extract_query_text(row: BigQueryJob) -> str: # We wrap select statements in a CTE to make them parseable as DML statement. diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py index 4df64c80bad8a8..53f9383ec02a72 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py @@ -5,6 +5,7 @@ import redshift_connector from datahub.emitter import mce_builder +from datahub.ingestion.api.closeable import Closeable from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.redshift.config import LineageMode, RedshiftConfig @@ -39,7 +40,7 @@ logger = logging.getLogger(__name__) -class RedshiftSqlLineageV2: +class RedshiftSqlLineageV2(Closeable): # does lineage and usage based on SQL parsing. def __init__( @@ -56,6 +57,7 @@ def __init__( self.context = context self.database = database + self.aggregator = SqlParsingAggregator( platform=self.platform, platform_instance=self.config.platform_instance, @@ -436,3 +438,6 @@ def generate(self) -> Iterable[MetadataWorkUnit]: message="Unexpected error(s) while attempting to extract lineage from SQL queries. See the full logs for more details.", context=f"Query Parsing Failures: {self.aggregator.report.observed_query_parse_failures}", ) + + def close(self) -> None: + self.aggregator.close() diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py index a9fc9ab8f3e993..76030cea984946 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py @@ -451,24 +451,23 @@ def _extract_metadata( ) if self.config.use_lineage_v2: - lineage_extractor = RedshiftSqlLineageV2( + with RedshiftSqlLineageV2( config=self.config, report=self.report, context=self.ctx, database=database, redundant_run_skip_handler=self.redundant_lineage_run_skip_handler, - ) - - yield from lineage_extractor.aggregator.register_schemas_from_stream( - self.process_schemas(connection, database) - ) + ) as lineage_extractor: + yield from lineage_extractor.aggregator.register_schemas_from_stream( + self.process_schemas(connection, database) + ) - self.report.report_ingestion_stage_start(LINEAGE_EXTRACTION) - yield from self.extract_lineage_v2( - connection=connection, - database=database, - lineage_extractor=lineage_extractor, - ) + self.report.report_ingestion_stage_start(LINEAGE_EXTRACTION) + yield from self.extract_lineage_v2( + connection=connection, + database=database, + lineage_extractor=lineage_extractor, + ) all_tables = self.get_all_tables() else: diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py index d39e95a884dbc2..a9f454cfd3cdb3 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py @@ -18,6 +18,7 @@ from datahub.configuration.connection_resolver import auto_connection_resolver from datahub.configuration.oauth import OAuthConfiguration, OAuthIdentityProvider from datahub.configuration.validate_field_rename import pydantic_renamed_field +from datahub.ingestion.api.closeable import Closeable from datahub.ingestion.source.snowflake.constants import ( CLIENT_PREFETCH_THREADS, CLIENT_SESSION_KEEP_ALIVE, @@ -364,7 +365,7 @@ def get_connection(self) -> "SnowflakeConnection": ) from e -class SnowflakeConnection: +class SnowflakeConnection(Closeable): _connection: NativeSnowflakeConnection def __init__(self, connection: NativeSnowflakeConnection): diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py index 1445d02aa49dbd..e11073d77b46eb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py @@ -1,3 +1,4 @@ +import contextlib import dataclasses import functools import json @@ -17,6 +18,7 @@ BaseTimeWindowConfig, BucketDuration, ) +from datahub.ingestion.api.closeable import Closeable from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.report import Report from datahub.ingestion.api.source import Source, SourceReport @@ -121,7 +123,7 @@ class SnowflakeQueriesSourceReport(SourceReport): queries_extractor: Optional[SnowflakeQueriesExtractorReport] = None -class SnowflakeQueriesExtractor(SnowflakeStructuredReportMixin): +class SnowflakeQueriesExtractor(SnowflakeStructuredReportMixin, Closeable): def __init__( self, connection: SnowflakeConnection, @@ -143,28 +145,33 @@ def __init__( self._structured_report = structured_report - self.aggregator = SqlParsingAggregator( - platform=self.identifiers.platform, - platform_instance=self.identifiers.identifier_config.platform_instance, - env=self.identifiers.identifier_config.env, - schema_resolver=schema_resolver, - graph=graph, - eager_graph_load=False, - generate_lineage=self.config.include_lineage, - generate_queries=self.config.include_queries, - generate_usage_statistics=self.config.include_usage_statistics, - generate_query_usage_statistics=self.config.include_query_usage_statistics, - usage_config=BaseUsageConfig( - bucket_duration=self.config.window.bucket_duration, - start_time=self.config.window.start_time, - end_time=self.config.window.end_time, - user_email_pattern=self.config.user_email_pattern, - # TODO make the rest of the fields configurable - ), - generate_operations=self.config.include_operations, - is_temp_table=self.is_temp_table, - is_allowed_table=self.is_allowed_table, - format_queries=False, + # The exit stack helps ensure that we close all the resources we open. + self._exit_stack = contextlib.ExitStack() + + self.aggregator: SqlParsingAggregator = self._exit_stack.enter_context( + SqlParsingAggregator( + platform=self.identifiers.platform, + platform_instance=self.identifiers.identifier_config.platform_instance, + env=self.identifiers.identifier_config.env, + schema_resolver=schema_resolver, + graph=graph, + eager_graph_load=False, + generate_lineage=self.config.include_lineage, + generate_queries=self.config.include_queries, + generate_usage_statistics=self.config.include_usage_statistics, + generate_query_usage_statistics=self.config.include_query_usage_statistics, + usage_config=BaseUsageConfig( + bucket_duration=self.config.window.bucket_duration, + start_time=self.config.window.start_time, + end_time=self.config.window.end_time, + user_email_pattern=self.config.user_email_pattern, + # TODO make the rest of the fields configurable + ), + generate_operations=self.config.include_operations, + is_temp_table=self.is_temp_table, + is_allowed_table=self.is_allowed_table, + format_queries=False, + ) ) self.report.sql_aggregator = self.aggregator.report @@ -248,6 +255,10 @@ def get_workunits_internal( self.aggregator.add(query) yield from auto_workunit(self.aggregator.gen_metadata()) + if not use_cached_audit_log: + queries.close() + shared_connection.close() + audit_log_file.unlink(missing_ok=True) def fetch_copy_history(self) -> Iterable[KnownLineageMapping]: # Derived from _populate_external_lineage_from_copy_history. @@ -426,6 +437,9 @@ def _parse_audit_log_row(self, row: Dict[str, Any]) -> PreparsedQuery: ) return entry + def close(self) -> None: + self._exit_stack.close() + class SnowflakeQueriesSource(Source): def __init__(self, ctx: PipelineContext, config: SnowflakeQueriesSourceConfig): @@ -468,6 +482,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: def get_report(self) -> SnowflakeQueriesSourceReport: return self.report + def close(self) -> None: + self.connection.close() + self.queries_extractor.close() + # Make sure we don't try to generate too much info for a single query. _MAX_TABLES_PER_QUERY = 20 diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index 0d7881f36554d1..dd7f73268fdc4f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -1,3 +1,4 @@ +import contextlib import functools import json import logging @@ -149,7 +150,12 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): cached_domains=[k for k in self.config.domain], graph=self.ctx.graph ) - self.connection = self.config.get_connection() + # The exit stack helps ensure that we close all the resources we open. + self._exit_stack = contextlib.ExitStack() + + self.connection: SnowflakeConnection = self._exit_stack.enter_context( + self.config.get_connection() + ) # For database, schema, tables, views, etc self.data_dictionary = SnowflakeDataDictionary(connection=self.connection) @@ -157,25 +163,27 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): self.aggregator: Optional[SqlParsingAggregator] = None if self.config.use_queries_v2 or self.config.include_table_lineage: - self.aggregator = SqlParsingAggregator( - platform=self.identifiers.platform, - platform_instance=self.config.platform_instance, - env=self.config.env, - graph=self.ctx.graph, - eager_graph_load=( - # If we're ingestion schema metadata for tables/views, then we will populate - # schemas into the resolver as we go. We only need to do a bulk fetch - # if we're not ingesting schema metadata as part of ingestion. - not ( - self.config.include_technical_schema - and self.config.include_tables - and self.config.include_views - ) - and not self.config.lazy_schema_resolver - ), - generate_usage_statistics=False, - generate_operations=False, - format_queries=self.config.format_sql_queries, + self.aggregator = self._exit_stack.enter_context( + SqlParsingAggregator( + platform=self.identifiers.platform, + platform_instance=self.config.platform_instance, + env=self.config.env, + graph=self.ctx.graph, + eager_graph_load=( + # If we're ingestion schema metadata for tables/views, then we will populate + # schemas into the resolver as we go. We only need to do a bulk fetch + # if we're not ingesting schema metadata as part of ingestion. + not ( + self.config.include_technical_schema + and self.config.include_tables + and self.config.include_views + ) + and not self.config.lazy_schema_resolver + ), + generate_usage_statistics=False, + generate_operations=False, + format_queries=self.config.format_sql_queries, + ) ) self.report.sql_aggregator = self.aggregator.report @@ -191,14 +199,16 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): pipeline_name=self.ctx.pipeline_name, run_id=self.ctx.run_id, ) - self.lineage_extractor = SnowflakeLineageExtractor( - config, - self.report, - connection=self.connection, - filters=self.filters, - identifiers=self.identifiers, - redundant_run_skip_handler=redundant_lineage_run_skip_handler, - sql_aggregator=self.aggregator, + self.lineage_extractor = self._exit_stack.enter_context( + SnowflakeLineageExtractor( + config, + self.report, + connection=self.connection, + filters=self.filters, + identifiers=self.identifiers, + redundant_run_skip_handler=redundant_lineage_run_skip_handler, + sql_aggregator=self.aggregator, + ) ) self.usage_extractor: Optional[SnowflakeUsageExtractor] = None @@ -213,13 +223,15 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): pipeline_name=self.ctx.pipeline_name, run_id=self.ctx.run_id, ) - self.usage_extractor = SnowflakeUsageExtractor( - config, - self.report, - connection=self.connection, - filter=self.filters, - identifiers=self.identifiers, - redundant_run_skip_handler=redundant_usage_run_skip_handler, + self.usage_extractor = self._exit_stack.enter_context( + SnowflakeUsageExtractor( + config, + self.report, + connection=self.connection, + filter=self.filters, + identifiers=self.identifiers, + redundant_run_skip_handler=redundant_usage_run_skip_handler, + ) ) self.profiling_state_handler: Optional[ProfilingHandler] = None @@ -444,10 +456,6 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self._snowflake_clear_ocsp_cache() - self.connection = self.config.get_connection() - if self.connection is None: - return - self.inspect_session_metadata(self.connection) snowsight_url_builder = None @@ -513,7 +521,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: schema_resolver = self.aggregator._schema_resolver - queries_extractor = SnowflakeQueriesExtractor( + queries_extractor: SnowflakeQueriesExtractor = SnowflakeQueriesExtractor( connection=self.connection, config=SnowflakeQueriesExtractorConfig( window=self.config, @@ -535,6 +543,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: # it should be pretty straightforward to refactor this and only initialize the aggregator once. self.report.queries_extractor = queries_extractor.report yield from queries_extractor.get_workunits_internal() + queries_extractor.close() else: if self.config.include_table_lineage and self.lineage_extractor: @@ -723,7 +732,4 @@ def _snowflake_clear_ocsp_cache(self) -> None: def close(self) -> None: super().close() StatefulIngestionSourceBase.close(self) - if self.lineage_extractor: - self.lineage_extractor.close() - if self.usage_extractor: - self.usage_extractor.close() + self._exit_stack.close() diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py index 5f2709fe426605..0b7ad14a8c1b41 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py @@ -275,6 +275,8 @@ class SqlAggregatorReport(Report): tool_meta_report: Optional[ToolMetaExtractorReport] = None def compute_stats(self) -> None: + if self._aggregator._closed: + return self.schema_resolver_count = self._aggregator._schema_resolver.schema_count() self.num_unique_query_fingerprints = len(self._aggregator._query_map) @@ -345,6 +347,7 @@ def __init__( # The exit stack helps ensure that we close all the resources we open. self._exit_stack = contextlib.ExitStack() + self._closed: bool = False # Set up the schema resolver. self._schema_resolver: SchemaResolver @@ -456,12 +459,16 @@ def __init__( shared_connection=self._shared_connection, tablename="query_usage_counts", ) + self._exit_stack.push(self._query_usage_counts) # Tool Extractor self._tool_meta_extractor = ToolMetaExtractor() self.report.tool_meta_report = self._tool_meta_extractor.report def close(self) -> None: + # Compute stats once before closing connections + self.report.compute_stats() + self._closed = True self._exit_stack.close() @property diff --git a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py index 9290100b0c521c..ef846f698f156e 100644 --- a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py +++ b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py @@ -1,4 +1,5 @@ import json +import os from datetime import datetime from pathlib import Path from unittest.mock import patch @@ -6,7 +7,9 @@ import pytest from freezegun import freeze_time +from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.source.bigquery_v2.bigquery_queries import ( + BigQueryQueriesSource, BigQueryQueriesSourceReport, ) from datahub.metadata.urns import CorpUserUrn @@ -93,3 +96,16 @@ def test_queries_ingestion(project_client, client, pytestconfig, monkeypatch, tm output_path=mcp_output_path, golden_path=mcp_golden_path, ) + + +@patch("google.cloud.bigquery.Client") +@patch("google.cloud.resourcemanager_v3.ProjectsClient") +def test_source_close_cleans_tmp(projects_client, client, tmp_path): + with patch("tempfile.tempdir", str(tmp_path)): + source = BigQueryQueriesSource.create( + {"project_ids": ["project1"]}, PipelineContext("run-id") + ) + assert len(os.listdir(tmp_path)) > 0 + # This closes QueriesExtractor which in turn closes SqlParsingAggregator + source.close() + assert len(os.listdir(tmp_path)) == 0 diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_queries.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_queries.py new file mode 100644 index 00000000000000..82f5691bcee3de --- /dev/null +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_queries.py @@ -0,0 +1,24 @@ +import os +from unittest.mock import patch + +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.source.snowflake.snowflake_queries import SnowflakeQueriesSource + + +@patch("snowflake.connector.connect") +def test_source_close_cleans_tmp(snowflake_connect, tmp_path): + with patch("tempfile.tempdir", str(tmp_path)): + source = SnowflakeQueriesSource.create( + { + "connection": { + "account_id": "ABC12345.ap-south-1.aws", + "username": "TST_USR", + "password": "TST_PWD", + } + }, + PipelineContext("run-id"), + ) + assert len(os.listdir(tmp_path)) > 0 + # This closes QueriesExtractor which in turn closes SqlParsingAggregator + source.close() + assert len(os.listdir(tmp_path)) == 0 diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py b/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py index 0d21936a74d072..849d550ef69c57 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py @@ -1,5 +1,7 @@ +import os import pathlib from datetime import datetime, timezone +from unittest.mock import patch import pytest from freezegun import freeze_time @@ -661,3 +663,23 @@ def test_basic_usage(pytestconfig: pytest.Config) -> None: outputs=mcps, golden_path=RESOURCE_DIR / "test_basic_usage.json", ) + + +def test_sql_aggreator_close_cleans_tmp(tmp_path): + frozen_timestamp = parse_user_datetime(FROZEN_TIME) + with patch("tempfile.tempdir", str(tmp_path)): + aggregator = SqlParsingAggregator( + platform="redshift", + generate_lineage=False, + generate_usage_statistics=True, + generate_operations=False, + usage_config=BaseUsageConfig( + start_time=get_time_bucket(frozen_timestamp, BucketDuration.DAY), + end_time=frozen_timestamp, + ), + generate_queries=True, + generate_query_usage_statistics=True, + ) + assert len(os.listdir(tmp_path)) > 0 + aggregator.close() + assert len(os.listdir(tmp_path)) == 0 From 011e5b97e7c0f0d330edae59368cc5f8c8512126 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Fri, 18 Oct 2024 01:12:05 -0700 Subject: [PATCH 39/40] =?UTF-8?q?fix(sdk):=20platform=20resource=20-=20sup?= =?UTF-8?q?port=20indexed=20queries=20when=20urns=20are=20i=E2=80=A6=20(#1?= =?UTF-8?q?1660)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platformresource/platform_resource.py | 41 +++++++++- .../test_platform_resource.py | 74 +++++++++++++++++++ 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py b/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py index 1556a67a9e5555..349b0ff11d84f7 100644 --- a/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py +++ b/metadata-ingestion/src/datahub/api/entities/platformresource/platform_resource.py @@ -85,6 +85,7 @@ def scroll_urns_by_filter( self, entity_type: str, extra_or_filters: List[Dict[str, str]], + extra_and_filters: List[Dict[str, str]] = [], ) -> Iterable[str]: """ Scroll through all urns that match the given filters @@ -92,10 +93,26 @@ def scroll_urns_by_filter( key_aspect = self.ENTITY_KEY_ASPECT_MAP.get(entity_type) assert key_aspect, f"No key aspect found for entity type {entity_type}" + if extra_or_filters and extra_and_filters: + raise ValueError( + "Only one of extra_or_filters and extra_and_filters should be provided" + ) count = 1000 - query = " OR ".join( - [f"{filter['field']}:{filter['value']}" for filter in extra_or_filters] + query = ( + " OR ".join( + [ + f"{filter['field']}:\"{filter['value']}\"" + for filter in extra_or_filters + ] + ) + if extra_or_filters + else " AND ".join( + [ + f"{filter['field']}:\"{filter['value']}\"" + for filter in extra_and_filters + ] + ) ) scroll_id = None while True: @@ -252,3 +269,23 @@ def search_by_key( def delete(self, graph_client: DataHubGraph, hard: bool = True) -> None: graph_client.delete_entity(str(PlatformResourceUrn(self.id)), hard=hard) + + @staticmethod + def search_by_filters( + graph_client: DataHubGraph, + and_filters: List[Dict[str, str]] = [], + or_filters: List[Dict[str, str]] = [], + ) -> Iterable["PlatformResource"]: + if and_filters and or_filters: + raise ValueError( + "Only one of and_filters and or_filters should be provided" + ) + openapi_client = OpenAPIGraphClient(graph_client) + for urn in openapi_client.scroll_urns_by_filter( + entity_type="platformResource", + extra_or_filters=or_filters if or_filters else [], + extra_and_filters=and_filters if and_filters else [], + ): + platform_resource = PlatformResource.from_datahub(graph_client, urn) + if platform_resource: + yield platform_resource diff --git a/smoke-test/tests/platform_resources/test_platform_resource.py b/smoke-test/tests/platform_resources/test_platform_resource.py index 7c53f72d843c93..7ebfd4d6ea15b4 100644 --- a/smoke-test/tests/platform_resources/test_platform_resource.py +++ b/smoke-test/tests/platform_resources/test_platform_resource.py @@ -112,3 +112,77 @@ def test_platform_resource_non_existent(graph_client, test_id): graph_client=graph_client, ) assert platform_resource is None + + +def test_platform_resource_urn_secondary_key(graph_client, test_id): + key = PlatformResourceKey( + platform=f"test_platform_{test_id}", + resource_type=f"test_resource_type_{test_id}", + primary_key=f"test_primary_key_{test_id}", + ) + dataset_urn = ( + f"urn:li:dataset:(urn:li:dataPlatform:test,test_secondary_key_{test_id},PROD)" + ) + platform_resource = PlatformResource.create( + key=key, + value={"test_key": f"test_value_{test_id}"}, + secondary_keys=[dataset_urn], + ) + platform_resource.to_datahub(graph_client) + wait_for_writes_to_sync() + + read_platform_resources = [ + r + for r in PlatformResource.search_by_key( + graph_client, dataset_urn, primary=False + ) + ] + assert len(read_platform_resources) == 1 + assert read_platform_resources[0] == platform_resource + + +def test_platform_resource_listing_by_resource_type(graph_client, test_id): + # Generate two resources with the same resource type + key1 = PlatformResourceKey( + platform=f"test_platform_{test_id}", + resource_type=f"test_resource_type_{test_id}", + primary_key=f"test_primary_key_1_{test_id}", + ) + platform_resource1 = PlatformResource.create( + key=key1, + value={"test_key": f"test_value_1_{test_id}"}, + ) + platform_resource1.to_datahub(graph_client) + + key2 = PlatformResourceKey( + platform=f"test_platform_{test_id}", + resource_type=f"test_resource_type_{test_id}", + primary_key=f"test_primary_key_2_{test_id}", + ) + platform_resource2 = PlatformResource.create( + key=key2, + value={"test_key": f"test_value_2_{test_id}"}, + ) + platform_resource2.to_datahub(graph_client) + + wait_for_writes_to_sync() + + search_results = [ + r + for r in PlatformResource.search_by_filters( + graph_client, + and_filters=[ + { + "field": "resourceType", + "condition": "EQUAL", + "value": key1.resource_type, + } + ], + ) + ] + assert len(search_results) == 2 + + read_platform_resource_1 = next(r for r in search_results if r.id == key1.id) + read_platform_resource_2 = next(r for r in search_results if r.id == key2.id) + assert read_platform_resource_1 == platform_resource1 + assert read_platform_resource_2 == platform_resource2 From dcf4793c3d8fb192b7ae3e3248f50445d59e8912 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Fri, 18 Oct 2024 03:13:18 -0700 Subject: [PATCH 40/40] fix(ingest/dbt): Prevent lineage cycles when parsing sql of dbt models (#11666) --- .../ingestion/source/dbt/dbt_common.py | 6 +++ .../tests/unit/test_dbt_source.py | 42 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index 4cd3c934ce6348..c95d0e545c5989 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -1989,6 +1989,11 @@ def _translate_dbt_name_to_upstream_urn(dbt_name: str) -> str: time=mce_builder.get_sys_time(), actor=_DEFAULT_ACTOR, ) + sibling_urn = node.get_urn( + self.config.target_platform, + self.config.env, + self.config.target_platform_instance, + ) return UpstreamLineageClass( upstreams=[ UpstreamClass( @@ -1997,6 +2002,7 @@ def _translate_dbt_name_to_upstream_urn(dbt_name: str) -> str: auditStamp=auditStamp, ) for upstream in upstream_urns + if not (node.node_type == "model" and upstream == sibling_urn) ], fineGrainedLineages=( (cll or None) if self.config.include_column_lineage else None diff --git a/metadata-ingestion/tests/unit/test_dbt_source.py b/metadata-ingestion/tests/unit/test_dbt_source.py index 7d01ecd034523d..f0d4c3408271f7 100644 --- a/metadata-ingestion/tests/unit/test_dbt_source.py +++ b/metadata-ingestion/tests/unit/test_dbt_source.py @@ -10,6 +10,7 @@ from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.source.dbt import dbt_cloud from datahub.ingestion.source.dbt.dbt_cloud import DBTCloudConfig +from datahub.ingestion.source.dbt.dbt_common import DBTNode from datahub.ingestion.source.dbt.dbt_core import ( DBTCoreConfig, DBTCoreSource, @@ -253,6 +254,47 @@ def test_dbt_config_prefer_sql_parser_lineage(): assert config.prefer_sql_parser_lineage is True +def test_dbt_prefer_sql_parser_lineage_no_self_reference(): + ctx = PipelineContext(run_id="test-run-id") + config = DBTCoreConfig.parse_obj( + { + **create_base_dbt_config(), + "skip_sources_in_lineage": True, + "prefer_sql_parser_lineage": True, + } + ) + source: DBTCoreSource = DBTCoreSource(config, ctx, "dbt") + all_nodes_map = { + "model1": DBTNode( + name="model1", + database=None, + schema=None, + alias=None, + comment="", + description="", + language=None, + raw_code=None, + dbt_adapter="postgres", + dbt_name="model1", + dbt_file_path=None, + dbt_package_name=None, + node_type="model", + materialization="table", + max_loaded_at=None, + catalog_type=None, + missing_from_catalog=False, + owner=None, + compiled_code="SELECT d FROM results WHERE d > (SELECT MAX(d) FROM model1)", + ), + } + source._infer_schemas_and_update_cll(all_nodes_map) + upstream_lineage = source._create_lineage_aspect_for_dbt_node( + all_nodes_map["model1"], all_nodes_map + ) + assert upstream_lineage is not None + assert len(upstream_lineage.upstreams) == 1 + + def test_dbt_s3_config(): # test missing aws config config_dict: dict = {