From d953718ab7b2090b3a03bdc578863f865e60759f Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 10 Dec 2024 18:32:52 -0500 Subject: [PATCH 01/47] feat(ingest): allow max_workers=1 with ASYNC_BATCH rest sink (#12088) --- .../src/datahub/ingestion/sink/datahub_rest.py | 6 +++--- .../src/datahub/utilities/partition_executor.py | 2 +- .../tests/unit/utilities/test_partition_executor.py | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py b/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py index 5b4d3fe38ecd9..1bb07ea846227 100644 --- a/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py +++ b/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py @@ -65,11 +65,11 @@ class DatahubRestSinkConfig(DatahubClientConfig): mode: RestSinkMode = _DEFAULT_REST_SINK_MODE # These only apply in async modes. - max_threads: int = _DEFAULT_REST_SINK_MAX_THREADS - max_pending_requests: int = 2000 + max_threads: pydantic.PositiveInt = _DEFAULT_REST_SINK_MAX_THREADS + max_pending_requests: pydantic.PositiveInt = 2000 # Only applies in async batch mode. - max_per_batch: int = 100 + max_per_batch: pydantic.PositiveInt = 100 @dataclasses.dataclass diff --git a/metadata-ingestion/src/datahub/utilities/partition_executor.py b/metadata-ingestion/src/datahub/utilities/partition_executor.py index 4d873d8f74bd8..542889f2f90e2 100644 --- a/metadata-ingestion/src/datahub/utilities/partition_executor.py +++ b/metadata-ingestion/src/datahub/utilities/partition_executor.py @@ -268,7 +268,7 @@ def __init__( self.process_batch = process_batch self.min_process_interval = min_process_interval self.read_from_pending_interval = read_from_pending_interval - assert self.max_workers > 1 + assert self.max_workers >= 1 self._state_lock = threading.Lock() self._executor = ThreadPoolExecutor( diff --git a/metadata-ingestion/tests/unit/utilities/test_partition_executor.py b/metadata-ingestion/tests/unit/utilities/test_partition_executor.py index eba79eafce473..ce211c2d61806 100644 --- a/metadata-ingestion/tests/unit/utilities/test_partition_executor.py +++ b/metadata-ingestion/tests/unit/utilities/test_partition_executor.py @@ -80,7 +80,8 @@ def task(id: str) -> str: assert len(done_tasks) == 16 -def test_batch_partition_executor_sequential_key_execution(): +@pytest.mark.parametrize("max_workers", [1, 2, 10]) +def test_batch_partition_executor_sequential_key_execution(max_workers: int) -> None: executing_tasks = set() done_tasks = set() done_task_batches = set() @@ -99,7 +100,7 @@ def process_batch(batch): done_task_batches.add(tuple(id for _, id in batch)) with BatchPartitionExecutor( - max_workers=2, + max_workers=max_workers, max_pending=10, max_per_batch=2, process_batch=process_batch, From 59dd8be457087a73068dd0bfb7386034326884e2 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Tue, 10 Dec 2024 18:33:53 -0600 Subject: [PATCH 02/47] fix(openapi): fix sort criteria parameter (#12090) --- docs/how/updating-datahub.md | 1 + .../controller/GenericEntitiesController.java | 20 +++++++++++++------ .../openapi/v3/OpenAPIV3Generator.java | 8 ++------ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 073f68db23063..8ba83768512a5 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -40,6 +40,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #12056: The DataHub Airflow plugin no longer supports Airflow 2.1 and Airflow 2.2. - #12056: The DataHub Airflow plugin now defaults to the v2 plugin implementation. - OpenAPI Update: PIT Keep Alive parameter added to scroll. NOTE: This parameter requires the `pointInTimeCreationEnabled` feature flag to be enabled and the `elasticSearch.implementation` configuration to be `elasticsearch`. This feature is not supported for OpenSearch at this time and the parameter will not be respected without both of these set. +- OpenAPI Update 2: Previously there was an incorrectly marked parameter named `sort` on the generic list entities endpoint for v3. This parameter is deprecated and only supports a single string value while the documentation indicates it supports a list of strings. This documentation error has been fixed and the correct field, `sortCriteria`, is now documented which supports a list of strings. ### Breaking Changes diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java index 425646d428214..32252e8033064 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java @@ -57,6 +57,7 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.MediaType; @@ -190,7 +191,8 @@ public ResponseEntity getEntities( @RequestParam(value = "count", defaultValue = "10") Integer count, @RequestParam(value = "query", defaultValue = "*") String query, @RequestParam(value = "scrollId", required = false) String scrollId, - @RequestParam(value = "sort", required = false, defaultValue = "urn") String sortField, + @RequestParam(value = "sort", required = false, defaultValue = "urn") @Deprecated + String sortField, @RequestParam(value = "sortCriteria", required = false) List sortFields, @RequestParam(value = "sortOrder", required = false, defaultValue = "ASCENDING") String sortOrder, @@ -222,14 +224,20 @@ public ResponseEntity getEntities( authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); } + SortOrder finalSortOrder = + SortOrder.valueOf(Optional.ofNullable(sortOrder).orElse("ASCENDING")); + List sortCriteria; - if (!CollectionUtils.isEmpty(sortFields)) { + if (!CollectionUtils.isEmpty(sortFields) + && sortFields.stream().anyMatch(StringUtils::isNotBlank)) { sortCriteria = new ArrayList<>(); - sortFields.forEach( - field -> sortCriteria.add(SearchUtil.sortBy(field, SortOrder.valueOf(sortOrder)))); + sortFields.stream() + .filter(StringUtils::isNotBlank) + .forEach(field -> sortCriteria.add(SearchUtil.sortBy(field, finalSortOrder))); + } else if (StringUtils.isNotBlank(sortField)) { + sortCriteria = Collections.singletonList(SearchUtil.sortBy(sortField, finalSortOrder)); } else { - sortCriteria = - Collections.singletonList(SearchUtil.sortBy(sortField, SortOrder.valueOf(sortOrder))); + sortCriteria = Collections.singletonList(SearchUtil.sortBy("urn", finalSortOrder)); } ScrollResult result = diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index 68ed316573f77..3c35a5c1984c1 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -570,7 +570,7 @@ private static void addExtraParameters(final Components components) { "SortBy" + MODEL_VERSION, new Parameter() .in(NAME_QUERY) - .name("sort") + .name("sortCriteria") .explode(true) .description("Sort fields for pagination.") .example(PROPERTY_URN) @@ -578,11 +578,7 @@ private static void addExtraParameters(final Components components) { new Schema() .type(TYPE_ARRAY) ._default(List.of(PROPERTY_URN)) - .items( - new Schema<>() - .type(TYPE_STRING) - ._enum(List.of(PROPERTY_URN)) - ._default(PROPERTY_URN)))); + .items(new Schema<>().type(TYPE_STRING)._default(PROPERTY_URN)))); components.addParameters( "SortOrder" + MODEL_VERSION, new Parameter() From 355a7e69aa1a444e497121f2c3f6b4dffa5beb36 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:08:39 +0530 Subject: [PATCH 03/47] feat(ingest/snowflake): allow option for incremental properties (#12080) --- .../api/incremental_properties_helper.py | 69 ++ .../datahub/ingestion/api/source_helpers.py | 4 +- .../ingestion/source/redshift/config.py | 1 + .../ingestion/source/redshift/redshift.py | 2 + .../source/snowflake/snowflake_config.py | 4 + .../source/snowflake/snowflake_v2.py | 6 + .../datahub/ingestion/source/unity/source.py | 2 + .../snowflake_privatelink_golden.json | 590 +++++++++++++----- .../integration/snowflake/test_snowflake.py | 5 +- 9 files changed, 513 insertions(+), 170 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/api/incremental_properties_helper.py diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_properties_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_properties_helper.py new file mode 100644 index 0000000000000..151b0c72a6c2d --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_properties_helper.py @@ -0,0 +1,69 @@ +import logging +from typing import Iterable, Optional + +from pydantic.fields import Field + +from datahub.configuration.common import ConfigModel +from datahub.emitter.mce_builder import set_aspect +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.source_helpers import create_dataset_props_patch_builder +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.metadata.schema_classes import ( + DatasetPropertiesClass, + MetadataChangeEventClass, + SystemMetadataClass, +) + +logger = logging.getLogger(__name__) + + +def convert_dataset_properties_to_patch( + urn: str, + aspect: DatasetPropertiesClass, + system_metadata: Optional[SystemMetadataClass], +) -> MetadataWorkUnit: + patch_builder = create_dataset_props_patch_builder(urn, aspect, system_metadata) + mcp = next(iter(patch_builder.build())) + return MetadataWorkUnit(id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp) + + +def auto_incremental_properties( + incremental_properties: bool, + stream: Iterable[MetadataWorkUnit], +) -> Iterable[MetadataWorkUnit]: + if not incremental_properties: + yield from stream + return # early exit + + for wu in stream: + urn = wu.get_urn() + + if isinstance(wu.metadata, MetadataChangeEventClass): + properties_aspect = wu.get_aspect_of_type(DatasetPropertiesClass) + set_aspect(wu.metadata, None, DatasetPropertiesClass) + if len(wu.metadata.proposedSnapshot.aspects) > 0: + yield wu + + if properties_aspect: + yield convert_dataset_properties_to_patch( + urn, properties_aspect, wu.metadata.systemMetadata + ) + elif isinstance(wu.metadata, MetadataChangeProposalWrapper) and isinstance( + wu.metadata.aspect, DatasetPropertiesClass + ): + properties_aspect = wu.metadata.aspect + if properties_aspect: + yield convert_dataset_properties_to_patch( + urn, properties_aspect, wu.metadata.systemMetadata + ) + else: + yield wu + + +# TODO: Use this in SQLCommonConfig. Currently only used in snowflake +class IncrementalPropertiesConfigMixin(ConfigModel): + incremental_properties: bool = Field( + default=False, + description="When enabled, emits dataset properties as incremental to existing dataset properties " + "in DataHub. When disabled, re-states dataset properties on each run.", + ) diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py index 8511f8529ac12..0c86e1cf47203 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py @@ -32,6 +32,7 @@ SchemaFieldClass, SchemaMetadataClass, StatusClass, + SystemMetadataClass, TimeWindowSizeClass, ) from datahub.metadata.urns import DatasetUrn, GlossaryTermUrn, TagUrn, Urn @@ -65,9 +66,10 @@ def auto_workunit( def create_dataset_props_patch_builder( dataset_urn: str, dataset_properties: DatasetPropertiesClass, + system_metadata: Optional[SystemMetadataClass] = None, ) -> DatasetPatchBuilder: """Creates a patch builder with a table's or view's attributes and dataset properties""" - patch_builder = DatasetPatchBuilder(dataset_urn) + patch_builder = DatasetPatchBuilder(dataset_urn, system_metadata) patch_builder.set_display_name(dataset_properties.name) patch_builder.set_description(dataset_properties.description) patch_builder.set_created(dataset_properties.created) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py index 2ff73323a14e3..cad48eaf1c237 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py @@ -159,6 +159,7 @@ class RedshiftConfig( description="Whether to extract column level lineage. This config works with rest-sink only.", ) + # TODO - use DatasetPropertiesConfigMixin instead patch_custom_properties: bool = Field( default=True, description="Whether to patch custom properties on existing datasets rather than replace.", diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py index 06cbb7fbae27c..49f7941563c1a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py @@ -831,6 +831,8 @@ def gen_dataset_workunits( customProperties=custom_properties, ) if self.config.patch_custom_properties: + # TODO: use auto_incremental_properties workunit processor instead + # Deprecate use of patch_custom_properties patch_builder = create_dataset_props_patch_builder( dataset_urn, dataset_properties ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index c30a26fbbd02c..1d1cc3c2af4f0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -16,6 +16,9 @@ from datahub.configuration.time_window_config import BaseTimeWindowConfig from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.configuration.validate_field_rename import pydantic_renamed_field +from datahub.ingestion.api.incremental_properties_helper import ( + IncrementalPropertiesConfigMixin, +) from datahub.ingestion.glossary.classification_mixin import ( ClassificationSourceConfigMixin, ) @@ -188,6 +191,7 @@ class SnowflakeV2Config( StatefulUsageConfigMixin, StatefulProfilingConfigMixin, ClassificationSourceConfigMixin, + IncrementalPropertiesConfigMixin, ): include_usage_stats: bool = Field( default=True, diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index 538841018067e..c3a7912c40e8e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -17,6 +17,9 @@ support_status, ) from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage +from datahub.ingestion.api.incremental_properties_helper import ( + auto_incremental_properties, +) from datahub.ingestion.api.source import ( CapabilityReport, MetadataWorkUnitProcessor, @@ -446,6 +449,9 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: functools.partial( auto_incremental_lineage, self.config.incremental_lineage ), + functools.partial( + auto_incremental_properties, self.config.incremental_properties + ), StaleEntityRemovalHandler.create( self, self.config, self.ctx ).workunit_processor, diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py index f758746193cd8..9d9a746580f93 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py @@ -556,6 +556,8 @@ def process_table(self, table: Table, schema: Schema) -> Iterable[MetadataWorkUn ) if table_props: + # TODO: use auto_incremental_properties workunit processor instead + # Consider enabling incremental_properties by default patch_builder = create_dataset_props_patch_builder(dataset_urn, table_props) for patch_mcp in patch_builder.build(): yield MetadataWorkUnit( diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json index d232ae710e891..3040c6c4e9196 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json @@ -138,27 +138,49 @@ { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_3,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/", - "name": "TABLE_3", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3", - "description": "Comment for Table", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "TABLE_3" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for Table" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.TABLE_3" + }, + { + "op": "add", + "path": "/customProperties/CLUSTERING_KEY", + "value": "LINEAR(COL_1)" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, @@ -567,27 +589,44 @@ { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/", - "name": "VIEW_1", - "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1", - "description": "Comment for View", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "VIEW_1" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for View" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.VIEW_1" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, @@ -808,27 +847,49 @@ { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/", - "name": "TABLE_1", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1", - "description": "Comment for Table", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "TABLE_1" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for Table" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.TABLE_1" + }, + { + "op": "add", + "path": "/customProperties/CLUSTERING_KEY", + "value": "LINEAR(COL_1)" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, @@ -1473,27 +1534,49 @@ { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_10,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/", - "name": "TABLE_10", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10", - "description": "Comment for Table", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "TABLE_10" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for Table" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.TABLE_10" + }, + { + "op": "add", + "path": "/customProperties/CLUSTERING_KEY", + "value": "LINEAR(COL_1)" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, @@ -1712,54 +1795,98 @@ { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/", - "name": "TABLE_5", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5", - "description": "Comment for Table", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "TABLE_5" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for Table" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.TABLE_5" + }, + { + "op": "add", + "path": "/customProperties/CLUSTERING_KEY", + "value": "LINEAR(COL_1)" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/", - "name": "TABLE_2", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2", - "description": "Comment for Table", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "TABLE_2" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for Table" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.TABLE_2" + }, + { + "op": "add", + "path": "/customProperties/CLUSTERING_KEY", + "value": "LINEAR(COL_1)" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, @@ -2301,27 +2428,49 @@ { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/", - "name": "TABLE_6", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6", - "description": "Comment for Table", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "TABLE_6" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for Table" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.TABLE_6" + }, + { + "op": "add", + "path": "/customProperties/CLUSTERING_KEY", + "value": "LINEAR(COL_1)" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, @@ -2621,27 +2770,49 @@ { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/", - "name": "TABLE_7", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7", - "description": "Comment for Table", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "TABLE_7" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for Table" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.TABLE_7" + }, + { + "op": "add", + "path": "/customProperties/CLUSTERING_KEY", + "value": "LINEAR(COL_1)" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, @@ -2664,27 +2835,49 @@ { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/", - "name": "TABLE_4", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4", - "description": "Comment for Table", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "TABLE_4" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for Table" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.TABLE_4" + }, + { + "op": "add", + "path": "/customProperties/CLUSTERING_KEY", + "value": "LINEAR(COL_1)" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, @@ -3162,27 +3355,49 @@ { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/", - "name": "TABLE_8", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8", - "description": "Comment for Table", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "TABLE_8" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for Table" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.TABLE_8" + }, + { + "op": "add", + "path": "/customProperties/CLUSTERING_KEY", + "value": "LINEAR(COL_1)" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, @@ -3302,27 +3517,49 @@ { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_9,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/", - "name": "TABLE_9", - "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9", - "description": "Comment for Table", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "TABLE_9" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for Table" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.TABLE_9" + }, + { + "op": "add", + "path": "/customProperties/CLUSTERING_KEY", + "value": "LINEAR(COL_1)" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, @@ -3607,27 +3844,44 @@ { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", + "changeType": "PATCH", "aspectName": "datasetProperties", "aspect": { - "json": { - "customProperties": {}, - "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/", - "name": "VIEW_2", - "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_2", - "description": "Comment for View", - "created": { - "time": 1623110400000 + "json": [ + { + "op": "add", + "path": "/name", + "value": "VIEW_2" }, - "lastModified": { - "time": 1623110400000 + { + "op": "add", + "path": "/description", + "value": "Comment for View" }, - "tags": [] - } + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.VIEW_2" + } + ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2022_06_07-17_00_00-ad3hnf", "lastRunId": "no-run-id-provided" } }, diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py index ca694b02cff01..1d7470d24f768 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py @@ -187,7 +187,9 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph): @freeze_time(FROZEN_TIME) -def test_snowflake_private_link(pytestconfig, tmp_path, mock_time, mock_datahub_graph): +def test_snowflake_private_link_and_incremental_mcps( + pytestconfig, tmp_path, mock_time, mock_datahub_graph +): test_resources_dir = pytestconfig.rootpath / "tests/integration/snowflake" # Run the metadata ingestion pipeline. @@ -218,6 +220,7 @@ def test_snowflake_private_link(pytestconfig, tmp_path, mock_time, mock_datahub_ include_usage_stats=False, format_sql_queries=True, incremental_lineage=False, + incremental_properties=True, include_operational_stats=False, platform_instance="instance1", start_time=datetime(2022, 6, 6, 0, 0, 0, 0).replace( From ff7ac480210f50b674ee9f864ad59bb4f5085cd9 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 11 Dec 2024 16:11:08 +0530 Subject: [PATCH 04/47] fix(cli): don't use /api in gms url (#12083) --- metadata-ingestion/src/datahub/cli/cli_utils.py | 2 ++ metadata-ingestion/tests/unit/cli/test_cli_utils.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/metadata-ingestion/src/datahub/cli/cli_utils.py b/metadata-ingestion/src/datahub/cli/cli_utils.py index 1b9cccb1cbc21..f80181192ba58 100644 --- a/metadata-ingestion/src/datahub/cli/cli_utils.py +++ b/metadata-ingestion/src/datahub/cli/cli_utils.py @@ -327,6 +327,8 @@ def _ensure_valid_gms_url_acryl_cloud(url: str) -> str: url = f"{url}/gms" elif url.endswith("acryl.io/"): url = f"{url}gms" + if url.endswith("acryl.io/api/gms"): + url = url.replace("acryl.io/api/gms", "acryl.io/gms") return url diff --git a/metadata-ingestion/tests/unit/cli/test_cli_utils.py b/metadata-ingestion/tests/unit/cli/test_cli_utils.py index af3a184d97e41..c9693c75d96fe 100644 --- a/metadata-ingestion/tests/unit/cli/test_cli_utils.py +++ b/metadata-ingestion/tests/unit/cli/test_cli_utils.py @@ -66,6 +66,10 @@ def test_fixup_gms_url(): assert cli_utils.fixup_gms_url("http://localhost:8080") == "http://localhost:8080" assert cli_utils.fixup_gms_url("http://localhost:8080/") == "http://localhost:8080" assert cli_utils.fixup_gms_url("http://abc.acryl.io") == "https://abc.acryl.io/gms" + assert ( + cli_utils.fixup_gms_url("http://abc.acryl.io/api/gms") + == "https://abc.acryl.io/gms" + ) def test_guess_frontend_url_from_gms_url(): From becf51575e3be279951a9a97801ac8998218c688 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Wed, 11 Dec 2024 18:21:58 +0530 Subject: [PATCH 05/47] docs(ingest/athena): update recipe with aws key pair example (#12076) --- datahub-web-react/src/app/ingest/source/builder/sources.json | 2 +- metadata-ingestion/docs/sources/athena/athena_recipe.yml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 776b6703895c3..102cce0f491e3 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -181,7 +181,7 @@ "displayName": "Athena", "description": "Import Schemas, Tables, Views, and lineage to S3 from Athena.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/athena/", - "recipe": "source:\n type: athena\n config:\n # Coordinates\n aws_region: my_aws_region\n work_group: primary\n\n # Options\n s3_staging_dir: \"s3://my_staging_athena_results_bucket/results/\"" + "recipe": "source:\n type: athena\n config:\n # AWS Keys (Optional - Required only if local aws credentials are not set)\n username: aws_access_key_id\n password: aws_secret_access_key\n # Coordinates\n aws_region: my_aws_region\n work_group: primary\n\n # Options\n s3_staging_dir: \"s3://my_staging_athena_results_bucket/results/\"" }, { "urn": "urn:li:dataPlatform:clickhouse", diff --git a/metadata-ingestion/docs/sources/athena/athena_recipe.yml b/metadata-ingestion/docs/sources/athena/athena_recipe.yml index 540d8101737a3..c93047ffed9ff 100644 --- a/metadata-ingestion/docs/sources/athena/athena_recipe.yml +++ b/metadata-ingestion/docs/sources/athena/athena_recipe.yml @@ -1,6 +1,11 @@ source: type: athena config: + + # AWS Keys (Optional - Required only if local aws credentials are not set) + username: my_aws_access_key_id + password: my_aws_secret_access_key + # Coordinates aws_region: my_aws_region work_group: primary From 54d53cfc625a1583fbd5e93b4b6bfff55b9897c6 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 11 Dec 2024 18:24:18 +0530 Subject: [PATCH 06/47] fix(ingest/gc): minor tweak gc source (#12093) --- .../src/datahub/ingestion/source/gc/datahub_gc.py | 10 +++++----- .../ingestion/source/gc/soft_deleted_entity_cleanup.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py index 52807ca2a3f02..814f65ecb45cf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py @@ -153,11 +153,6 @@ def get_workunits_internal( self.truncate_indices() except Exception as e: self.report.failure("While trying to truncate indices ", exc=e) - if self.dataprocess_cleanup: - try: - yield from self.dataprocess_cleanup.get_workunits_internal() - except Exception as e: - self.report.failure("While trying to cleanup data process ", exc=e) if self.soft_deleted_entities_cleanup: try: self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities() @@ -170,6 +165,11 @@ def get_workunits_internal( self.execution_request_cleanup.run() except Exception as e: self.report.failure("While trying to cleanup execution request ", exc=e) + if self.dataprocess_cleanup: + try: + yield from self.dataprocess_cleanup.get_workunits_internal() + except Exception as e: + self.report.failure("While trying to cleanup data process ", exc=e) yield from [] def truncate_indices(self) -> None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py index 3b367cdea5813..bb4ab753543b7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py @@ -60,7 +60,7 @@ class SoftDeletedEntitiesCleanupConfig(ConfigModel): description="Query to filter entities", ) limit_entities_delete: Optional[int] = Field( - 10000, description="Max number of entities to delete." + 25000, description="Max number of entities to delete." ) runtime_limit_seconds: Optional[int] = Field( From e6cc676b23e06da184c3324b8331acd994b112b1 Mon Sep 17 00:00:00 2001 From: Jonny Dixon <45681293+acrylJonny@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:54:47 +0000 Subject: [PATCH 07/47] fix(ingest/abs): detect jsonl schema (#11775) --- metadata-ingestion/src/datahub/ingestion/source/abs/source.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/abs/source.py b/metadata-ingestion/src/datahub/ingestion/source/abs/source.py index 66f268799b2f1..ad2bc36cf558b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/abs/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/abs/source.py @@ -201,6 +201,10 @@ def get_fields(self, table_data: TableData, path_spec: PathSpec) -> List: ).infer_schema(file) elif extension == ".json": fields = json.JsonInferrer().infer_schema(file) + elif extension == ".jsonl": + fields = json.JsonInferrer( + max_rows=self.source_config.max_rows, format="jsonl" + ).infer_schema(file) elif extension == ".avro": fields = avro.AvroInferrer().infer_schema(file) else: From b091e4615d915783818c1750599723b933d41267 Mon Sep 17 00:00:00 2001 From: skrydal Date: Wed, 11 Dec 2024 17:02:31 +0100 Subject: [PATCH 08/47] feat(ingest/kafka): Flag for optional schemas ingestion (#12077) --- docs/how/updating-datahub.md | 3 +- .../datahub/ingestion/source/kafka/kafka.py | 29 +- .../tests/integration/kafka/kafka_to_file.yml | 1 + .../kafka_without_schemas_mces_golden.json | 575 ++++++++++++++++++ .../kafka/kafka_without_schemas_to_file.yml | 16 + .../tests/integration/kafka/test_kafka.py | 9 +- .../tests/unit/test_kafka_source.py | 5 +- 7 files changed, 620 insertions(+), 18 deletions(-) create mode 100644 metadata-ingestion/tests/integration/kafka/kafka_without_schemas_mces_golden.json create mode 100644 metadata-ingestion/tests/integration/kafka/kafka_without_schemas_to_file.yml diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 8ba83768512a5..5bc0e66fa2ff1 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -31,7 +31,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe urn:li:dataset:(urn:li:dataPlatform:powerbi,[.]...,) ``` - The config `include_workspace_name_in_dataset_urn` is default to `false` for backward compatiblity, However, we recommend enabling this flag after performing the necessary cleanup. + The config `include_workspace_name_in_dataset_urn` is default to `false` for backward compatibility, However, we recommend enabling this flag after performing the necessary cleanup. If stateful ingestion is enabled, running ingestion with the latest CLI version will handle the cleanup automatically. Otherwise, we recommend soft deleting all powerbi data via the DataHub CLI: `datahub delete --platform powerbi --soft` and then re-ingest with the latest CLI version, ensuring the `include_workspace_name_in_dataset_urn` configuration is set to true. @@ -39,6 +39,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #11742: For PowerBi ingestion, `use_powerbi_email` is now enabled by default when extracting ownership information. - #12056: The DataHub Airflow plugin no longer supports Airflow 2.1 and Airflow 2.2. - #12056: The DataHub Airflow plugin now defaults to the v2 plugin implementation. +- #12077: `Kafka` source no longer ingests schemas from schema registry as separate entities by default, set `ingest_schemas_as_entities` to `true` to ingest them - OpenAPI Update: PIT Keep Alive parameter added to scroll. NOTE: This parameter requires the `pointInTimeCreationEnabled` feature flag to be enabled and the `elasticSearch.implementation` configuration to be `elasticsearch`. This feature is not supported for OpenSearch at this time and the parameter will not be respected without both of these set. - OpenAPI Update 2: Previously there was an incorrectly marked parameter named `sort` on the generic list entities endpoint for v3. This parameter is deprecated and only supports a single string value while the documentation indicates it supports a list of strings. This documentation error has been fixed and the correct field, `sortCriteria`, is now documented which supports a list of strings. diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py index 709ba431f0f87..fa842a15ba732 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py @@ -141,6 +141,10 @@ class KafkaSourceConfig( default=False, description="Disables the utilization of the TopicRecordNameStrategy for Schema Registry subjects. For more information, visit: https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#handling-differences-between-preregistered-and-client-derived-schemas:~:text=io.confluent.kafka.serializers.subject.TopicRecordNameStrategy", ) + ingest_schemas_as_entities: bool = pydantic.Field( + default=False, + description="Enables ingesting schemas from schema registry as separate entities, in addition to the topics", + ) def get_kafka_consumer( @@ -343,17 +347,20 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: else: self.report.report_dropped(topic) - # Get all subjects from schema registry and ingest them as SCHEMA DatasetSubTypes - for subject in self.schema_registry_client.get_subjects(): - try: - yield from self._extract_record( - subject, True, topic_detail=None, extra_topic_config=None - ) - except Exception as e: - logger.warning(f"Failed to extract subject {subject}", exc_info=True) - self.report.report_warning( - "subject", f"Exception while extracting topic {subject}: {e}" - ) + if self.source_config.ingest_schemas_as_entities: + # Get all subjects from schema registry and ingest them as SCHEMA DatasetSubTypes + for subject in self.schema_registry_client.get_subjects(): + try: + yield from self._extract_record( + subject, True, topic_detail=None, extra_topic_config=None + ) + except Exception as e: + logger.warning( + f"Failed to extract subject {subject}", exc_info=True + ) + self.report.report_warning( + "subject", f"Exception while extracting topic {subject}: {e}" + ) def _extract_record( self, diff --git a/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml b/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml index 380df845e737c..cde21d85ed2d9 100644 --- a/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml +++ b/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml @@ -3,6 +3,7 @@ run_id: kafka-test source: type: kafka config: + ingest_schemas_as_entities: true connection: bootstrap: "localhost:29092" schema_registry_url: "http://localhost:28081" diff --git a/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_mces_golden.json b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_mces_golden.json new file mode 100644 index 0000000000000..7810c8077b31d --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_mces_golden.json @@ -0,0 +1,575 @@ +[ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "key_topic", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "44fd7a7b325d6fdd4275b1f02a79c1a8", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "", + "keySchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}", + "keySchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=long].id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "id", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=string].namespace", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "namespace", + "recursive": false, + "isPartOfKey": true + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "Partitions": "1", + "Replication Factor": "1", + "min.insync.replicas": "1", + "retention.bytes": "-1", + "retention.ms": "604800000", + "cleanup.policy": "delete", + "max.message.bytes": "1048588", + "unclean.leader.election.enable": "false" + }, + "name": "key_topic", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Topic" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "key_value_topic", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "a79a2fe3adab60b21d272a9cc3e93595", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\",\"tags\":[\"Email\"]},{\"name\":\"firstName\",\"type\":\"string\",\"tags\":[\"Name\"]},{\"name\":\"lastName\",\"type\":\"string\",\"tags\":[\"Name\"]}],\"tags\":[\"PII\"]}", + "documentSchemaType": "AVRO", + "keySchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}", + "keySchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=long].id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "id", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=string].namespace", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "namespace", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "email", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Email" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Email\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].firstName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "firstName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].lastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "lastName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:PII" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "Partitions": "1", + "Replication Factor": "1", + "min.insync.replicas": "1", + "retention.bytes": "-1", + "retention.ms": "604800000", + "cleanup.policy": "delete", + "max.message.bytes": "1048588", + "unclean.leader.election.enable": "false", + "Schema Name": "key_value_topic-value" + }, + "name": "key_value_topic", + "description": "Value schema for kafka topic", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Topic" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "json": { + "domains": [ + "urn:li:domain:sales" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "value_topic", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "62c7c400ec5760797a59c45e59c2f2dc", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\",\"tags\":[\"Email\"]},{\"name\":\"firstName\",\"type\":\"string\",\"tags\":[\"Name\"]},{\"name\":\"lastName\",\"type\":\"string\",\"tags\":[\"Name\"]}],\"tags\":[\"PII\"]}", + "documentSchemaType": "AVRO", + "keySchema": "\"string\"", + "keySchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[key=True].[type=string]", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "email", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Email" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Email\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].firstName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "firstName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].lastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "lastName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:PII" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "Partitions": "1", + "Replication Factor": "1", + "min.insync.replicas": "1", + "retention.bytes": "-1", + "retention.ms": "604800000", + "cleanup.policy": "delete", + "max.message.bytes": "1048588", + "unclean.leader.election.enable": "false", + "Schema Name": "value_topic-value" + }, + "name": "value_topic", + "description": "Value schema for kafka topic", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Topic" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Email", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Email" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Name", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Name" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:PII", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "PII" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_to_file.yml b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_to_file.yml new file mode 100644 index 0000000000000..7f44e43c3c490 --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_to_file.yml @@ -0,0 +1,16 @@ +run_id: kafka-test + +source: + type: kafka + config: + connection: + bootstrap: "localhost:29092" + schema_registry_url: "http://localhost:28081" + domain: + "urn:li:domain:sales": + allow: + - "key_value_topic" +sink: + type: file + config: + filename: "./kafka_without_schemas_mces.json" diff --git a/metadata-ingestion/tests/integration/kafka/test_kafka.py b/metadata-ingestion/tests/integration/kafka/test_kafka.py index bf0ec1845a66c..0d9a714625e96 100644 --- a/metadata-ingestion/tests/integration/kafka/test_kafka.py +++ b/metadata-ingestion/tests/integration/kafka/test_kafka.py @@ -43,20 +43,21 @@ def mock_kafka_service(docker_compose_runner, test_resources_dir): yield docker_compose_runner +@pytest.mark.parametrize("approach", ["kafka_without_schemas", "kafka"]) @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_kafka_ingest( - mock_kafka_service, test_resources_dir, pytestconfig, tmp_path, mock_time + mock_kafka_service, test_resources_dir, pytestconfig, tmp_path, mock_time, approach ): # Run the metadata ingestion pipeline. - config_file = (test_resources_dir / "kafka_to_file.yml").resolve() + config_file = (test_resources_dir / f"{approach}_to_file.yml").resolve() run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path) # Verify the output. mce_helpers.check_golden_file( pytestconfig, - output_path=tmp_path / "kafka_mces.json", - golden_path=test_resources_dir / "kafka_mces_golden.json", + output_path=tmp_path / f"{approach}_mces.json", + golden_path=test_resources_dir / f"{approach}_mces_golden.json", ignore_paths=[], ) diff --git a/metadata-ingestion/tests/unit/test_kafka_source.py b/metadata-ingestion/tests/unit/test_kafka_source.py index cab0a2bce7ba8..1a8afe1b956fa 100644 --- a/metadata-ingestion/tests/unit/test_kafka_source.py +++ b/metadata-ingestion/tests/unit/test_kafka_source.py @@ -330,6 +330,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: "topic2-key": "test.acryl.Topic2Key", "topic2-value": "test.acryl.Topic2Value", }, + "ingest_schemas_as_entities": True, } ctx = PipelineContext(run_id="test") kafka_source = KafkaSource.create(source_config, ctx) @@ -478,8 +479,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: kafka_source = KafkaSource.create(source_config, ctx) workunits = list(kafka_source.get_workunits()) - - assert len(workunits) == 6 + assert len(workunits) == 2 if ignore_warnings_on_schema_type: assert not kafka_source.report.warnings else: @@ -622,6 +622,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: kafka_source = KafkaSource.create( { "connection": {"bootstrap": "localhost:9092"}, + "ingest_schemas_as_entities": True, "meta_mapping": { "owner": { "match": "^@(.*)", From f1ef4f8e5f7e4facee5d80e71c7ce6c8051bafcc Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 11 Dec 2024 13:59:14 -0500 Subject: [PATCH 09/47] feat(structuredProperties) Add new settings aspect plus graphql changes for structured props (#12052) --- .../CreateStructuredPropertyResolver.java | 119 ++++++++--- .../DeleteStructuredPropertyResolver.java | 18 ++ .../RemoveStructuredPropertiesResolver.java | 2 +- .../UpdateStructuredPropertyResolver.java | 196 +++++++++++++++--- .../UpsertStructuredPropertiesResolver.java | 2 +- .../types/chart/mappers/ChartMapper.java | 3 +- .../container/mappers/ContainerMapper.java | 4 +- .../corpgroup/mappers/CorpGroupMapper.java | 3 +- .../corpuser/mappers/CorpUserMapper.java | 3 +- .../dashboard/mappers/DashboardMapper.java | 3 +- .../dataflow/mappers/DataFlowMapper.java | 3 +- .../types/datajob/mappers/DataJobMapper.java | 3 +- .../mappers/DataProductMapper.java | 3 +- .../types/dataset/mappers/DatasetMapper.java | 3 +- .../graphql/types/domain/DomainMapper.java | 4 +- .../glossary/mappers/GlossaryNodeMapper.java | 3 +- .../glossary/mappers/GlossaryTermMapper.java | 3 +- .../mlmodel/mappers/MLFeatureMapper.java | 3 +- .../mlmodel/mappers/MLFeatureTableMapper.java | 3 +- .../mlmodel/mappers/MLModelGroupMapper.java | 3 +- .../types/mlmodel/mappers/MLModelMapper.java | 3 +- .../mlmodel/mappers/MLPrimaryKeyMapper.java | 3 +- .../types/schemafield/SchemaFieldMapper.java | 3 +- .../StructuredPropertiesMapper.java | 17 +- .../StructuredPropertyMapper.java | 18 ++ .../StructuredPropertyType.java | 3 +- .../src/main/resources/properties.graphql | 84 +++++++- .../CreateStructuredPropertyResolverTest.java | 120 ++++++++++- .../DeleteStructuredPropertyResolverTest.java | 91 ++++++++ .../StructuredPropertyUtilsTest.java | 42 ++++ .../UpdateStructuredPropertyResolverTest.java | 102 ++++++++- .../models/StructuredPropertyUtils.java | 50 +++++ .../java/com/linkedin/metadata/Constants.java | 2 + .../structuredproperties.py | 12 +- .../StructuredPropertyDefinition.pdl | 38 ++-- .../structured/StructuredPropertySettings.pdl | 64 ++++++ .../src/main/resources/entity-registry.yml | 1 + 37 files changed, 913 insertions(+), 124 deletions(-) create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolverTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/StructuredPropertyUtilsTest.java create mode 100644 metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertySettings.pdl diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java index 328f63b893d06..7d232748f0d93 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java @@ -1,7 +1,8 @@ package com.linkedin.datahub.graphql.resolvers.structuredproperties; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; -import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.buildMetadataChangeProposalWithUrn; +import static com.linkedin.metadata.Constants.*; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.SetMode; @@ -12,20 +13,24 @@ import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.CreateStructuredPropertyInput; import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.StructuredPropertySettingsInput; import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertyMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder; +import com.linkedin.metadata.models.StructuredPropertyUtils; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.structured.PrimitivePropertyValue; import com.linkedin.structured.PropertyCardinality; import com.linkedin.structured.PropertyValue; import com.linkedin.structured.StructuredPropertyKey; +import com.linkedin.structured.StructuredPropertySettings; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; -import java.util.UUID; import java.util.concurrent.CompletableFuture; import javax.annotation.Nonnull; @@ -54,40 +59,28 @@ public CompletableFuture get(final DataFetchingEnviron "Unable to create structured property. Please contact your admin."); } final StructuredPropertyKey key = new StructuredPropertyKey(); - final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); + final String id = + StructuredPropertyUtils.getPropertyId(input.getId(), input.getQualifiedName()); key.setId(id); final Urn propertyUrn = EntityKeyUtils.convertEntityKeyToUrn(key, STRUCTURED_PROPERTY_ENTITY_NAME); - StructuredPropertyDefinitionPatchBuilder builder = - new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); - - builder.setQualifiedName(input.getQualifiedName()); - builder.setValueType(input.getValueType()); - input.getEntityTypes().forEach(builder::addEntityType); - if (input.getDisplayName() != null) { - builder.setDisplayName(input.getDisplayName()); - } - if (input.getDescription() != null) { - builder.setDescription(input.getDescription()); - } - if (input.getImmutable() != null) { - builder.setImmutable(input.getImmutable()); - } - if (input.getTypeQualifier() != null) { - buildTypeQualifier(input, builder); - } - if (input.getAllowedValues() != null) { - buildAllowedValues(input, builder); + + if (_entityClient.exists(context.getOperationContext(), propertyUrn)) { + throw new IllegalArgumentException( + "A structured property already exists with this urn"); } - if (input.getCardinality() != null) { - builder.setCardinality( - PropertyCardinality.valueOf(input.getCardinality().toString())); + + List mcps = new ArrayList<>(); + + // first, create the property definition itself + mcps.add(createPropertyDefinition(context, propertyUrn, id, input)); + + // then add the settings aspect if we're adding any settings inputs + if (input.getSettings() != null) { + mcps.add(createPropertySettings(context, propertyUrn, input.getSettings())); } - builder.setCreated(context.getOperationContext().getAuditStamp()); - builder.setLastModified(context.getOperationContext().getAuditStamp()); - MetadataChangeProposal mcp = builder.build(); - _entityClient.ingestProposal(context.getOperationContext(), mcp, false); + _entityClient.batchIngestProposals(context.getOperationContext(), mcps, false); EntityResponse response = _entityClient.getV2( @@ -103,6 +96,72 @@ public CompletableFuture get(final DataFetchingEnviron }); } + private MetadataChangeProposal createPropertySettings( + @Nonnull final QueryContext context, + @Nonnull final Urn propertyUrn, + final StructuredPropertySettingsInput settingsInput) + throws Exception { + StructuredPropertySettings settings = new StructuredPropertySettings(); + + if (settingsInput.getIsHidden() != null) { + settings.setIsHidden(settingsInput.getIsHidden()); + } + if (settingsInput.getShowInSearchFilters() != null) { + settings.setShowInSearchFilters(settingsInput.getShowInSearchFilters()); + } + if (settingsInput.getShowInAssetSummary() != null) { + settings.setShowInAssetSummary(settingsInput.getShowInAssetSummary()); + } + if (settingsInput.getShowAsAssetBadge() != null) { + settings.setShowAsAssetBadge(settingsInput.getShowAsAssetBadge()); + } + if (settingsInput.getShowInColumnsTable() != null) { + settings.setShowInColumnsTable(settingsInput.getShowInColumnsTable()); + } + settings.setLastModified(context.getOperationContext().getAuditStamp()); + + StructuredPropertyUtils.validatePropertySettings(settings, true); + + return buildMetadataChangeProposalWithUrn( + propertyUrn, STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME, settings); + } + + private MetadataChangeProposal createPropertyDefinition( + @Nonnull final QueryContext context, + @Nonnull final Urn propertyUrn, + @Nonnull final String id, + final CreateStructuredPropertyInput input) + throws Exception { + StructuredPropertyDefinitionPatchBuilder builder = + new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); + + builder.setQualifiedName(id); + builder.setValueType(input.getValueType()); + input.getEntityTypes().forEach(builder::addEntityType); + if (input.getDisplayName() != null) { + builder.setDisplayName(input.getDisplayName()); + } + if (input.getDescription() != null) { + builder.setDescription(input.getDescription()); + } + if (input.getImmutable() != null) { + builder.setImmutable(input.getImmutable()); + } + if (input.getTypeQualifier() != null) { + buildTypeQualifier(input, builder); + } + if (input.getAllowedValues() != null) { + buildAllowedValues(input, builder); + } + if (input.getCardinality() != null) { + builder.setCardinality(PropertyCardinality.valueOf(input.getCardinality().toString())); + } + builder.setCreated(context.getOperationContext().getAuditStamp()); + builder.setLastModified(context.getOperationContext().getAuditStamp()); + + return builder.build(); + } + private void buildTypeQualifier( @Nonnull final CreateStructuredPropertyInput input, @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java index e7d59494654fd..58f8d340fcc07 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java @@ -6,6 +6,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.DeleteStructuredPropertyInput; import com.linkedin.entity.client.EntityClient; @@ -42,6 +43,23 @@ public CompletableFuture get(final DataFetchingEnvironment environment) "Unable to delete structured property. Please contact your admin."); } _entityClient.deleteEntity(context.getOperationContext(), propertyUrn); + // Asynchronously Delete all references to the entity (to return quickly) + GraphQLConcurrencyUtils.supplyAsync( + () -> { + try { + _entityClient.deleteEntityReferences( + context.getOperationContext(), propertyUrn); + } catch (Exception e) { + log.error( + String.format( + "Caught exception while attempting to clear all entity references for Structured Property with urn %s", + propertyUrn), + e); + } + return null; + }, + this.getClass().getSimpleName(), + "get"); return true; } catch (Exception e) { throw new RuntimeException( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java index ea8c6dac36a4a..313e0a16d8916 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java @@ -93,7 +93,7 @@ public CompletableFuture get(final DataFetchingEnviron "Unable to update structured property. Please contact your admin."); } final Urn propertyUrn = UrnUtils.getUrn(input.getUrn()); - StructuredPropertyDefinition existingDefinition = + final EntityResponse entityResponse = getExistingStructuredProperty(context, propertyUrn); - StructuredPropertyDefinitionPatchBuilder builder = - new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); - if (input.getDisplayName() != null) { - builder.setDisplayName(input.getDisplayName()); - } - if (input.getDescription() != null) { - builder.setDescription(input.getDescription()); - } - if (input.getImmutable() != null) { - builder.setImmutable(input.getImmutable()); - } - if (input.getTypeQualifier() != null) { - buildTypeQualifier(input, builder, existingDefinition); - } - if (input.getNewAllowedValues() != null) { - buildAllowedValues(input, builder); - } - if (input.getSetCardinalityAsMultiple() != null) { - builder.setCardinality(PropertyCardinality.MULTIPLE); + List mcps = new ArrayList<>(); + + // first update the definition aspect if we need to + MetadataChangeProposal definitionMcp = + updateDefinition(input, context, propertyUrn, entityResponse); + if (definitionMcp != null) { + mcps.add(definitionMcp); } - if (input.getNewEntityTypes() != null) { - input.getNewEntityTypes().forEach(builder::addEntityType); + + // then update the settings aspect if we need to + if (input.getSettings() != null) { + mcps.add(updateSettings(context, input.getSettings(), propertyUrn, entityResponse)); } - builder.setLastModified(context.getOperationContext().getAuditStamp()); - MetadataChangeProposal mcp = builder.build(); - _entityClient.ingestProposal(context.getOperationContext(), mcp, false); + _entityClient.batchIngestProposals(context.getOperationContext(), mcps, false); EntityResponse response = _entityClient.getV2( @@ -102,6 +95,120 @@ public CompletableFuture get(final DataFetchingEnviron }); } + private boolean hasSettingsChanged( + StructuredPropertySettings existingSettings, StructuredPropertySettingsInput settingsInput) { + if (settingsInput.getIsHidden() != null + && !existingSettings.isIsHidden().equals(settingsInput.getIsHidden())) { + return true; + } + if (settingsInput.getShowInSearchFilters() != null + && !existingSettings + .isShowInSearchFilters() + .equals(settingsInput.getShowInSearchFilters())) { + return true; + } + if (settingsInput.getShowInAssetSummary() != null + && !existingSettings.isShowInAssetSummary().equals(settingsInput.getShowInAssetSummary())) { + return true; + } + if (settingsInput.getShowAsAssetBadge() != null + && !existingSettings.isShowAsAssetBadge().equals(settingsInput.getShowAsAssetBadge())) { + return true; + } + if (settingsInput.getShowInColumnsTable() != null + && !existingSettings.isShowInColumnsTable().equals(settingsInput.getShowInColumnsTable())) { + return true; + } + return false; + } + + private MetadataChangeProposal updateSettings( + @Nonnull final QueryContext context, + @Nonnull final StructuredPropertySettingsInput settingsInput, + @Nonnull final Urn propertyUrn, + @Nonnull final EntityResponse entityResponse) + throws Exception { + StructuredPropertySettings existingSettings = + getExistingStructuredPropertySettings(entityResponse); + // check if settings has changed to determine if we should update the timestamp + boolean hasChanged = hasSettingsChanged(existingSettings, settingsInput); + if (hasChanged) { + existingSettings.setLastModified(context.getOperationContext().getAuditStamp()); + } + + if (settingsInput.getIsHidden() != null) { + existingSettings.setIsHidden(settingsInput.getIsHidden()); + } + if (settingsInput.getShowInSearchFilters() != null) { + existingSettings.setShowInSearchFilters(settingsInput.getShowInSearchFilters()); + } + if (settingsInput.getShowInAssetSummary() != null) { + existingSettings.setShowInAssetSummary(settingsInput.getShowInAssetSummary()); + } + if (settingsInput.getShowAsAssetBadge() != null) { + existingSettings.setShowAsAssetBadge(settingsInput.getShowAsAssetBadge()); + } + if (settingsInput.getShowInColumnsTable() != null) { + existingSettings.setShowInColumnsTable(settingsInput.getShowInColumnsTable()); + } + + StructuredPropertyUtils.validatePropertySettings(existingSettings, true); + + return buildMetadataChangeProposalWithUrn( + propertyUrn, STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME, existingSettings); + } + + private MetadataChangeProposal updateDefinition( + @Nonnull final UpdateStructuredPropertyInput input, + @Nonnull final QueryContext context, + @Nonnull final Urn propertyUrn, + @Nonnull final EntityResponse entityResponse) + throws Exception { + StructuredPropertyDefinition existingDefinition = + getExistingStructuredPropertyDefinition(entityResponse); + StructuredPropertyDefinitionPatchBuilder builder = + new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); + + boolean hasUpdatedDefinition = false; + + if (input.getDisplayName() != null) { + builder.setDisplayName(input.getDisplayName()); + hasUpdatedDefinition = true; + } + if (input.getDescription() != null) { + builder.setDescription(input.getDescription()); + hasUpdatedDefinition = true; + } + if (input.getImmutable() != null) { + builder.setImmutable(input.getImmutable()); + hasUpdatedDefinition = true; + } + if (input.getTypeQualifier() != null) { + buildTypeQualifier(input, builder, existingDefinition); + hasUpdatedDefinition = true; + } + if (input.getNewAllowedValues() != null) { + buildAllowedValues(input, builder); + hasUpdatedDefinition = true; + } + if (input.getSetCardinalityAsMultiple() != null + && input.getSetCardinalityAsMultiple().equals(true)) { + builder.setCardinality(PropertyCardinality.MULTIPLE); + hasUpdatedDefinition = true; + } + if (input.getNewEntityTypes() != null) { + input.getNewEntityTypes().forEach(builder::addEntityType); + hasUpdatedDefinition = true; + } + + if (hasUpdatedDefinition) { + builder.setLastModified(context.getOperationContext().getAuditStamp()); + + return builder.build(); + } + return null; + } + private void buildTypeQualifier( @Nonnull final UpdateStructuredPropertyInput input, @Nonnull final StructuredPropertyDefinitionPatchBuilder builder, @@ -141,17 +248,40 @@ private void buildAllowedValues( }); } - private StructuredPropertyDefinition getExistingStructuredProperty( + private EntityResponse getExistingStructuredProperty( @Nonnull final QueryContext context, @Nonnull final Urn propertyUrn) throws Exception { - EntityResponse response = - _entityClient.getV2( - context.getOperationContext(), STRUCTURED_PROPERTY_ENTITY_NAME, propertyUrn, null); + return _entityClient.getV2( + context.getOperationContext(), STRUCTURED_PROPERTY_ENTITY_NAME, propertyUrn, null); + } + private StructuredPropertyDefinition getExistingStructuredPropertyDefinition( + EntityResponse response) throws Exception { if (response != null && response.getAspects().containsKey(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)) { return new StructuredPropertyDefinition( - response.getAspects().get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME).getValue().data()); + response + .getAspects() + .get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME) + .getValue() + .data() + .copy()); } - return null; + throw new IllegalArgumentException( + "Attempting to update a structured property with no definition aspect."); + } + + private StructuredPropertySettings getExistingStructuredPropertySettings(EntityResponse response) + throws Exception { + if (response != null + && response.getAspects().containsKey(STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME)) { + return new StructuredPropertySettings( + response + .getAspects() + .get(STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME) + .getValue() + .data() + .copy()); + } + return new StructuredPropertySettings(); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java index 770c8a0d749c3..6c1d7949332fb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java @@ -103,7 +103,7 @@ public CompletableFuture chart.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java index 02357b3ddc349..7ac00c46475bc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java @@ -161,7 +161,9 @@ public static Container map( if (envelopedStructuredProps != null) { result.setStructuredProperties( StructuredPropertiesMapper.map( - context, new StructuredProperties(envelopedStructuredProps.getValue().data()))); + context, + new StructuredProperties(envelopedStructuredProps.getValue().data()), + entityUrn)); } final EnvelopedAspect envelopedForms = aspects.get(FORMS_ASPECT_NAME); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java index 6246cf64bbf7f..010816431f54d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java @@ -59,7 +59,8 @@ public CorpGroup apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java index 4fa278983399b..a94b555daebdf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java @@ -88,7 +88,8 @@ public CorpUser apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java index 4fa52b1136564..fd1c7a5db2a79 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java @@ -142,7 +142,8 @@ public Dashboard apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((dashboard, dataMap) -> dashboard.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java index 9e2612f60abda..44bc6a99eae4b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java @@ -114,7 +114,8 @@ public DataFlow apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java index d7da875bc2a29..772871d77f217 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java @@ -135,7 +135,8 @@ public DataJob apply( result.setSubTypes(SubTypesMapper.map(context, new SubTypes(data))); } else if (STRUCTURED_PROPERTIES_ASPECT_NAME.equals(name)) { result.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(data))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(data), entityUrn)); } else if (FORMS_ASPECT_NAME.equals(name)) { result.setForms(FormsMapper.map(new Forms(data), entityUrn.toString())); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java index 08637dbfd01ed..8693ec97f1a2e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java @@ -92,7 +92,8 @@ public DataProduct apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index 0869463ba73ac..e411014c23c89 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -173,7 +173,8 @@ public Dataset apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((dataset, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java index 7d05e0862a96d..ffcb94a0b7e29 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java @@ -71,7 +71,9 @@ public static Domain map(@Nullable QueryContext context, final EntityResponse en if (envelopedStructuredProps != null) { result.setStructuredProperties( StructuredPropertiesMapper.map( - context, new StructuredProperties(envelopedStructuredProps.getValue().data()))); + context, + new StructuredProperties(envelopedStructuredProps.getValue().data()), + entityUrn)); } final EnvelopedAspect envelopedForms = aspects.get(FORMS_ASPECT_NAME); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java index 4912d18614f41..a694b62999080 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java @@ -59,7 +59,8 @@ public GlossaryNode apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java index 1274646f45ec4..e309ffad84df5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java @@ -90,7 +90,8 @@ public GlossaryTerm apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java index a4f3aa7a0e226..d5eb1a15624dc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java @@ -115,7 +115,8 @@ public MLFeature apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((mlFeature, dataMap) -> mlFeature.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java index 30bf4dda1cf4f..51d3004d97a61 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java @@ -117,7 +117,8 @@ public MLFeatureTable apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((mlFeatureTable, dataMap) -> mlFeatureTable.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java index 7e99040e44c82..6e3da1c153392 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java @@ -112,7 +112,8 @@ public MLModelGroup apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((mlModelGroup, dataMap) -> mlModelGroup.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java index a3bc5c663c89a..7102fd4aed974 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java @@ -174,7 +174,8 @@ public MLModel apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((dataset, dataMap) -> dataset.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java index 36784f96ea30e..c446c892cb223 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java @@ -112,7 +112,8 @@ public MLPrimaryKey apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java index b1f27357d4550..30eac54aede9b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java @@ -41,7 +41,8 @@ public SchemaFieldEntity apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((schemaField, dataMap) -> schemaField.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( BUSINESS_ATTRIBUTE_ASPECT, (((schemaField, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java index dc1ff7ca32971..4f155903c055b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java @@ -25,23 +25,29 @@ public class StructuredPropertiesMapper { public static final StructuredPropertiesMapper INSTANCE = new StructuredPropertiesMapper(); public static com.linkedin.datahub.graphql.generated.StructuredProperties map( - @Nullable QueryContext context, @Nonnull final StructuredProperties structuredProperties) { - return INSTANCE.apply(context, structuredProperties); + @Nullable QueryContext context, + @Nonnull final StructuredProperties structuredProperties, + @Nonnull final Urn entityUrn) { + return INSTANCE.apply(context, structuredProperties, entityUrn); } public com.linkedin.datahub.graphql.generated.StructuredProperties apply( - @Nullable QueryContext context, @Nonnull final StructuredProperties structuredProperties) { + @Nullable QueryContext context, + @Nonnull final StructuredProperties structuredProperties, + @Nonnull final Urn entityUrn) { com.linkedin.datahub.graphql.generated.StructuredProperties result = new com.linkedin.datahub.graphql.generated.StructuredProperties(); result.setProperties( structuredProperties.getProperties().stream() - .map(p -> mapStructuredProperty(context, p)) + .map(p -> mapStructuredProperty(context, p, entityUrn)) .collect(Collectors.toList())); return result; } private StructuredPropertiesEntry mapStructuredProperty( - @Nullable QueryContext context, StructuredPropertyValueAssignment valueAssignment) { + @Nullable QueryContext context, + StructuredPropertyValueAssignment valueAssignment, + @Nonnull final Urn entityUrn) { StructuredPropertiesEntry entry = new StructuredPropertiesEntry(); entry.setStructuredProperty(createStructuredPropertyEntity(valueAssignment)); final List values = new ArrayList<>(); @@ -58,6 +64,7 @@ private StructuredPropertiesEntry mapStructuredProperty( }); entry.setValues(values); entry.setValueEntities(entities); + entry.setAssociatedUrn(entityUrn.toString()); return entry; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java index c539c65118ac6..5dc73d9ad0938 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java @@ -17,6 +17,7 @@ import com.linkedin.datahub.graphql.generated.StringValue; import com.linkedin.datahub.graphql.generated.StructuredPropertyDefinition; import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.StructuredPropertySettings; import com.linkedin.datahub.graphql.generated.TypeQualifier; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; import com.linkedin.datahub.graphql.types.mappers.MapperUtils; @@ -55,6 +56,8 @@ public StructuredPropertyEntity apply( MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult( STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, (this::mapStructuredPropertyDefinition)); + mappingHelper.mapToResult( + STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME, (this::mapStructuredPropertySettings)); return mappingHelper.getResult(); } @@ -112,6 +115,21 @@ private List mapAllowedValues(@Nonnull PropertyValueArray gmsValue return allowedValues; } + private void mapStructuredPropertySettings( + @Nonnull StructuredPropertyEntity extendedProperty, @Nonnull DataMap dataMap) { + com.linkedin.structured.StructuredPropertySettings gmsSettings = + new com.linkedin.structured.StructuredPropertySettings(dataMap); + StructuredPropertySettings settings = new StructuredPropertySettings(); + + settings.setIsHidden(gmsSettings.isIsHidden()); + settings.setShowInSearchFilters(gmsSettings.isShowInSearchFilters()); + settings.setShowInAssetSummary(gmsSettings.isShowInAssetSummary()); + settings.setShowAsAssetBadge(gmsSettings.isShowAsAssetBadge()); + settings.setShowInColumnsTable(gmsSettings.isShowInColumnsTable()); + + extendedProperty.setSettings(settings); + } + private DataTypeEntity createDataTypeEntity(final Urn dataTypeUrn) { final DataTypeEntity dataType = new DataTypeEntity(); dataType.setUrn(dataTypeUrn.toString()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java index 22e161d320f21..e451e96a3e84d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java @@ -27,7 +27,8 @@ public class StructuredPropertyType implements com.linkedin.datahub.graphql.types.EntityType { public static final Set ASPECTS_TO_FETCH = - ImmutableSet.of(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME); + ImmutableSet.of( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME); private final EntityClient _entityClient; @Override diff --git a/datahub-graphql-core/src/main/resources/properties.graphql b/datahub-graphql-core/src/main/resources/properties.graphql index 292381d064f36..ff20caa50bf03 100644 --- a/datahub-graphql-core/src/main/resources/properties.graphql +++ b/datahub-graphql-core/src/main/resources/properties.graphql @@ -49,6 +49,11 @@ type StructuredPropertyEntity implements Entity { """ definition: StructuredPropertyDefinition! + """ + Definition of this structured property including its name + """ + settings: StructuredPropertySettings + """ Granular API for querying edges extending from this entity """ @@ -117,6 +122,36 @@ type StructuredPropertyDefinition { lastModified: ResolvedAuditStamp } +""" +Settings specific to a structured property entity +""" +type StructuredPropertySettings { + """ + Whether or not this asset should be hidden in the main application + """ + isHidden: Boolean! + + """ + Whether or not this asset should be displayed as a search filter + """ + showInSearchFilters: Boolean! + + """ + Whether or not this asset should be displayed in the asset sidebar + """ + showInAssetSummary: Boolean! + + """ + Whether or not this asset should be displayed as an asset badge on other asset's headers + """ + showAsAssetBadge: Boolean! + + """ + Whether or not this asset should be displayed as a column in the schema field table in a Dataset's "Columns" tab. + """ + showInColumnsTable: Boolean! +} + """ An entry for an allowed value for a structured property """ @@ -202,6 +237,11 @@ type StructuredPropertiesEntry { The optional entities associated with the values if the values are entity urns """ valueEntities: [Entity] + + """ + The urn of the entity this property came from for tracking purposes e.g. when sibling nodes are merged together + """ + associatedUrn: String! } """ @@ -330,8 +370,9 @@ input CreateStructuredPropertyInput { """ The unique fully qualified name of this structured property, dot delimited. + This will be required to match the ID of this structured property. """ - qualifiedName: String! + qualifiedName: String """ The optional display name for this property @@ -375,6 +416,11 @@ input CreateStructuredPropertyInput { For example: ["urn:li:entityType:datahub.dataset"] """ entityTypes: [String!]! + + """ + Settings for this structured property + """ + settings: StructuredPropertySettingsInput } """ @@ -455,6 +501,11 @@ input UpdateStructuredPropertyInput { For backwards compatibility, this is append only. """ newEntityTypes: [String!] + + """ + Settings for this structured property + """ + settings: StructuredPropertySettingsInput } """ @@ -477,3 +528,34 @@ input DeleteStructuredPropertyInput { """ urn: String! } + +""" +Settings for a structured property +""" +input StructuredPropertySettingsInput { + """ + Whether or not this asset should be hidden in the main application + """ + isHidden: Boolean + + """ + Whether or not this asset should be displayed as a search filter + """ + showInSearchFilters: Boolean + + """ + Whether or not this asset should be displayed in the asset sidebar + """ + showInAssetSummary: Boolean + + """ + Whether or not this asset should be displayed as an asset badge on other asset's headers + """ + showAsAssetBadge: Boolean + + """ + Whether or not this asset should be displayed as a column in the schema field table in a Dataset's "Columns" tab. + """ + showInColumnsTable: Boolean +} + diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java index 72cdb78542e41..fec2251f92b63 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java @@ -10,11 +10,11 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.CreateStructuredPropertyInput; import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.StructuredPropertySettingsInput; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; -import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; import java.util.ArrayList; @@ -36,7 +36,8 @@ public class CreateStructuredPropertyResolverTest { null, null, null, - new ArrayList<>()); + new ArrayList<>(), + null); @Test public void testGetSuccess() throws Exception { @@ -56,7 +57,40 @@ public void testGetSuccess() throws Exception { // Validate that we called ingest Mockito.verify(mockEntityClient, Mockito.times(1)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testGetMismatchIdAndQualifiedName() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + CreateStructuredPropertyInput testInput = + new CreateStructuredPropertyInput( + "mismatched", + "io.acryl.test", + "Display Name", + "description", + true, + null, + null, + null, + null, + new ArrayList<>(), + null); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate ingest is not called + Mockito.verify(mockEntityClient, Mockito.times(0)) + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } @Test @@ -75,7 +109,7 @@ public void testGetUnauthorized() throws Exception { // Validate that we did NOT call ingest Mockito.verify(mockEntityClient, Mockito.times(0)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } @Test @@ -94,7 +128,83 @@ public void testGetFailure() throws Exception { // Validate that ingest was called, but that caused a failure Mockito.verify(mockEntityClient, Mockito.times(1)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testGetInvalidSettingsInput() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + // if isHidden is true, other fields should not be true + StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput(); + settingsInput.setIsHidden(true); + settingsInput.setShowAsAssetBadge(true); + + CreateStructuredPropertyInput testInput = + new CreateStructuredPropertyInput( + null, + "io.acryl.test", + "Display Name", + "description", + true, + null, + null, + null, + null, + new ArrayList<>(), + settingsInput); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate ingest is not called + Mockito.verify(mockEntityClient, Mockito.times(0)) + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testGetSuccessWithSettings() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput(); + settingsInput.setShowAsAssetBadge(true); + + CreateStructuredPropertyInput testInput = + new CreateStructuredPropertyInput( + null, + "io.acryl.test", + "Display Name", + "description", + true, + null, + null, + null, + null, + new ArrayList<>(), + settingsInput); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + StructuredPropertyEntity prop = resolver.get(mockEnv).get(); + + assertEquals(prop.getUrn(), TEST_STRUCTURED_PROPERTY_URN); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolverTest.java new file mode 100644 index 0000000000000..7ecec25708f2d --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolverTest.java @@ -0,0 +1,91 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DeleteStructuredPropertyInput; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class DeleteStructuredPropertyResolverTest { + private static final String TEST_PROP_URN = "urn:li:structuredProperty:test"; + + private static final DeleteStructuredPropertyInput TEST_INPUT = + new DeleteStructuredPropertyInput(TEST_PROP_URN); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + DeleteStructuredPropertyResolver resolver = + new DeleteStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Boolean success = resolver.get(mockEnv).get(); + assertTrue(success); + + // Validate that we called delete + Mockito.verify(mockEntityClient, Mockito.times(1)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_PROP_URN))); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + DeleteStructuredPropertyResolver resolver = + new DeleteStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call delete + Mockito.verify(mockEntityClient, Mockito.times(0)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_PROP_URN))); + } + + @Test + public void testGetFailure() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(false); + DeleteStructuredPropertyResolver resolver = + new DeleteStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that deleteEntity was called, but since it's the thing that failed it was called + // once still + Mockito.verify(mockEntityClient, Mockito.times(1)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_PROP_URN))); + } + + private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + if (!shouldSucceed) { + Mockito.doThrow(new RemoteInvocationException()).when(client).deleteEntity(any(), any()); + } + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/StructuredPropertyUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/StructuredPropertyUtilsTest.java new file mode 100644 index 0000000000000..0e9d064b3c7af --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/StructuredPropertyUtilsTest.java @@ -0,0 +1,42 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static org.testng.Assert.*; + +import com.linkedin.metadata.models.StructuredPropertyUtils; +import java.util.UUID; +import org.testng.annotations.Test; + +public class StructuredPropertyUtilsTest { + + @Test + public void testGetIdMismatchedInput() throws Exception { + assertThrows( + IllegalArgumentException.class, + () -> StructuredPropertyUtils.getPropertyId("test1", "test2")); + } + + @Test + public void testGetIdConsistentInput() throws Exception { + assertEquals(StructuredPropertyUtils.getPropertyId("test1", "test1"), "test1"); + } + + @Test + public void testGetIdNullQualifiedName() throws Exception { + assertEquals(StructuredPropertyUtils.getPropertyId("test1", null), "test1"); + } + + @Test + public void testGetIdNullId() throws Exception { + assertEquals(StructuredPropertyUtils.getPropertyId(null, "test1"), "test1"); + } + + @Test + public void testGetIdNullForBoth() throws Exception { + try { + String id = StructuredPropertyUtils.getPropertyId(null, null); + UUID.fromString(id); + } catch (Exception e) { + fail("ID produced is not a UUID"); + } + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java index b818bcfb7d7f4..2b0e7fd83b7ce 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java @@ -2,20 +2,25 @@ import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; import static org.mockito.ArgumentMatchers.any; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertThrows; +import com.linkedin.common.UrnArray; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.StructuredPropertySettingsInput; import com.linkedin.datahub.graphql.generated.UpdateStructuredPropertyInput; +import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; -import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.structured.StructuredPropertyDefinition; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; import org.mockito.Mockito; @@ -33,6 +38,7 @@ public class UpdateStructuredPropertyResolverTest { null, null, null, + null, null); @Test @@ -53,7 +59,7 @@ public void testGetSuccess() throws Exception { // Validate that we called ingest Mockito.verify(mockEntityClient, Mockito.times(1)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } @Test @@ -72,7 +78,7 @@ public void testGetUnauthorized() throws Exception { // Validate that we did NOT call ingest Mockito.verify(mockEntityClient, Mockito.times(0)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } @Test @@ -91,7 +97,80 @@ public void testGetFailure() throws Exception { // Validate that ingest was not called since there was a get failure before ingesting Mockito.verify(mockEntityClient, Mockito.times(0)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testGetInvalidSettingsInput() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateStructuredPropertyResolver resolver = + new UpdateStructuredPropertyResolver(mockEntityClient); + + // if isHidden is true, other fields should not be true + StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput(); + settingsInput.setIsHidden(true); + settingsInput.setShowInSearchFilters(true); + + final UpdateStructuredPropertyInput testInput = + new UpdateStructuredPropertyInput( + TEST_STRUCTURED_PROPERTY_URN, + "New Display Name", + "new description", + true, + null, + null, + null, + null, + settingsInput); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that ingest was not called since there was a get failure before ingesting + Mockito.verify(mockEntityClient, Mockito.times(0)) + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testGetValidSettingsInput() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateStructuredPropertyResolver resolver = + new UpdateStructuredPropertyResolver(mockEntityClient); + + // if isHidden is true, other fields should not be true + StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput(); + settingsInput.setIsHidden(true); + + final UpdateStructuredPropertyInput testInput = + new UpdateStructuredPropertyInput( + TEST_STRUCTURED_PROPERTY_URN, + "New Display Name", + "new description", + true, + null, + null, + null, + null, + settingsInput); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + StructuredPropertyEntity prop = resolver.get(mockEnv).get(); + + assertEquals(prop.getUrn(), TEST_STRUCTURED_PROPERTY_URN); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { @@ -99,7 +178,11 @@ private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exceptio EntityResponse response = new EntityResponse(); response.setEntityName(Constants.STRUCTURED_PROPERTY_ENTITY_NAME); response.setUrn(UrnUtils.getUrn(TEST_STRUCTURED_PROPERTY_URN)); - response.setAspects(new EnvelopedAspectMap()); + final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + aspectMap.put( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(createDefinition().data()))); + response.setAspects(aspectMap); if (shouldSucceed) { Mockito.when( client.getV2( @@ -120,4 +203,13 @@ private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exceptio return client; } + + private StructuredPropertyDefinition createDefinition() { + StructuredPropertyDefinition definition = new StructuredPropertyDefinition(); + definition.setDisplayName("test"); + definition.setQualifiedName("test"); + definition.setValueType(UrnUtils.getUrn("urn:li:dataType:datahub.string")); + definition.setEntityTypes(new UrnArray()); + return definition; + } } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java index e9ee7789550c6..1b12f540badfb 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java @@ -20,6 +20,7 @@ import com.linkedin.structured.PrimitivePropertyValue; import com.linkedin.structured.StructuredProperties; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.structured.StructuredPropertySettings; import com.linkedin.structured.StructuredPropertyValueAssignment; import com.linkedin.structured.StructuredPropertyValueAssignmentArray; import com.linkedin.util.Pair; @@ -32,6 +33,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -45,6 +47,11 @@ private StructuredPropertyUtils() {} static final Date MIN_DATE = Date.valueOf("1000-01-01"); static final Date MAX_DATE = Date.valueOf("9999-12-31"); + public static final String INVALID_SETTINGS_MESSAGE = + "Cannot have property isHidden = true while other display location settings are also true."; + public static final String ONLY_ONE_BADGE = + "Cannot have more than one property set with show as badge. Property urns currently set: "; + public static LogicalValueType getLogicalValueType( StructuredPropertyDefinition structuredPropertyDefinition) { return getLogicalValueType(structuredPropertyDefinition.getValueType()); @@ -355,4 +362,47 @@ private static Pair filterValue true); } } + + /* + * We accept both ID and qualifiedName as inputs when creating a structured property. However, + * these two fields should ALWAYS be the same. If they don't provide either, use a UUID for both. + * If they provide both, ensure they are the same otherwise throw. Otherwise, use what is provided. + */ + public static String getPropertyId( + @Nullable final String inputId, @Nullable final String inputQualifiedName) { + if (inputId != null && inputQualifiedName != null && !inputId.equals(inputQualifiedName)) { + throw new IllegalArgumentException( + "Qualified name and the ID of a structured property must match"); + } + + String id = UUID.randomUUID().toString(); + + if (inputQualifiedName != null) { + id = inputQualifiedName; + } else if (inputId != null) { + id = inputId; + } + + return id; + } + + /* + * Ensure that a structured property settings aspect is valid by ensuring that if isHidden is true, + * the other fields concerning display locations are false; + */ + public static boolean validatePropertySettings( + StructuredPropertySettings settings, boolean shouldThrow) { + if (settings.isIsHidden()) { + if (settings.isShowInSearchFilters() + || settings.isShowInAssetSummary() + || settings.isShowAsAssetBadge()) { + if (shouldThrow) { + throw new IllegalArgumentException(INVALID_SETTINGS_MESSAGE); + } else { + return false; + } + } + } + return true; + } } diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 9c608187342e8..797055d5fb6a9 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -363,6 +363,8 @@ public class Constants { // Structured Property public static final String STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME = "propertyDefinition"; public static final String STRUCTURED_PROPERTY_KEY_ASPECT_NAME = "structuredPropertyKey"; + public static final String STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME = + "structuredPropertySettings"; // Form public static final String FORM_INFO_ASPECT_NAME = "formInfo"; diff --git a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py index 181c70adc640a..013efbdf6a2f6 100644 --- a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py +++ b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py @@ -118,11 +118,13 @@ def validate_entity_types(cls, v): @property def fqn(self) -> str: assert self.urn is not None - return ( - self.qualified_name - or self.id - or Urn.from_string(self.urn).get_entity_id()[0] - ) + id = Urn.create_from_string(self.urn).get_entity_id()[0] + if self.qualified_name is not None: + # ensure that qualified name and ID match + assert ( + self.qualified_name == id + ), "ID in the urn and the qualified_name must match" + return id @validator("urn", pre=True, always=True) def urn_must_be_present(cls, v, values): diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl index 3ddb2d2e571da..416e2c5c11e22 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl @@ -89,25 +89,25 @@ record StructuredPropertyDefinition { version: optional string /** - * Created Audit stamp - */ - @Searchable = { - "/time": { - "fieldName": "createdTime", - "fieldType": "DATETIME" - } - } - created: optional AuditStamp + * Created Audit stamp + */ + @Searchable = { + "/time": { + "fieldName": "createdTime", + "fieldType": "DATETIME" + } + } + created: optional AuditStamp - /** - * Created Audit stamp - */ - @Searchable = { - "/time": { - "fieldName": "lastModified", - "fieldType": "DATETIME" - } - } - lastModified: optional AuditStamp + /** + * Last Modified Audit stamp + */ + @Searchable = { + "/time": { + "fieldName": "lastModified", + "fieldType": "DATETIME" + } + } + lastModified: optional AuditStamp } diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertySettings.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertySettings.pdl new file mode 100644 index 0000000000000..fadcdfa5204e1 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertySettings.pdl @@ -0,0 +1,64 @@ +namespace com.linkedin.structured + +import com.linkedin.common.AuditStamp + +/** + * Settings specific to a structured property entity + */ +@Aspect = { + "name": "structuredPropertySettings" +} +record StructuredPropertySettings { + /** + * Whether or not this asset should be hidden in the main application + */ + @Searchable = { + "fieldType": "BOOLEAN" + } + isHidden: boolean = false + + /** + * Whether or not this asset should be displayed as a search filter + */ + @Searchable = { + "fieldType": "BOOLEAN" + } + showInSearchFilters: boolean = false + + /** + * Whether or not this asset should be displayed in the asset sidebar + */ + @Searchable = { + "fieldType": "BOOLEAN" + } + showInAssetSummary: boolean = false + + /** + * Whether or not this asset should be displayed as an asset badge on other + * asset's headers + */ + @Searchable = { + "fieldType": "BOOLEAN" + } + showAsAssetBadge: boolean = false + + /** + * Whether or not this asset should be displayed as a column in the schema field table + * in a Dataset's "Columns" tab. + */ + @Searchable = { + "fieldType": "BOOLEAN" + } + showInColumnsTable: boolean = false + + /** + * Last Modified Audit stamp + */ + @Searchable = { + "/time": { + "fieldName": "lastModifiedSettings", + "fieldType": "DATETIME" + } + } + lastModified: optional AuditStamp +} diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index ee1481f29f7e9..1c3eb5b574e20 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -602,6 +602,7 @@ entities: keyAspect: structuredPropertyKey aspects: - propertyDefinition + - structuredPropertySettings - institutionalMemory - status - name: form From d062411c6893d2d34c4cf14b48eb6f5acb7d1658 Mon Sep 17 00:00:00 2001 From: sid-acryl <155424659+sid-acryl@users.noreply.github.com> Date: Thu, 12 Dec 2024 01:08:52 +0530 Subject: [PATCH 10/47] fix(ingest/tableau): project_path_pattern use in _is_denied_project (#12010) --- .../src/datahub/configuration/common.py | 6 +-- .../ingestion/source/tableau/tableau.py | 51 +++++++++++++------ 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py index 4fdf564162410..7df007e087979 100644 --- a/metadata-ingestion/src/datahub/configuration/common.py +++ b/metadata-ingestion/src/datahub/configuration/common.py @@ -258,7 +258,7 @@ def allow_all(cls) -> "AllowDenyPattern": return AllowDenyPattern() def allowed(self, string: str) -> bool: - if self._denied(string): + if self.denied(string): return False return any( @@ -266,7 +266,7 @@ def allowed(self, string: str) -> bool: for allow_pattern in self.allow ) - def _denied(self, string: str) -> bool: + def denied(self, string: str) -> bool: for deny_pattern in self.deny: if re.match(deny_pattern, string, self.regex_flags): return True @@ -290,7 +290,7 @@ def get_allowed_list(self) -> List[str]: raise ValueError( "allow list must be fully specified to get list of allowed strings" ) - return [a for a in self.allow if not self._denied(a)] + return [a for a in self.allow if not self.denied(a)] def __eq__(self, other): # type: ignore return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 68c38d4d06461..6844b8a425a7b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -353,7 +353,7 @@ class TableauConfig( project_path_separator: str = Field( default="/", - description="The separator used for the project_pattern field between project names. By default, we use a slash. " + description="The separator used for the project_path_pattern field between project names. By default, we use a slash. " "You can change this if your Tableau projects contain slashes in their names, and you'd like to filter by project.", ) @@ -959,19 +959,36 @@ def _is_allowed_project(self, project: TableauProject) -> bool: return is_allowed def _is_denied_project(self, project: TableauProject) -> bool: - # Either project name or project path should exist in deny - for deny_pattern in self.config.project_pattern.deny: - # Either name or project path is denied - if re.match( - deny_pattern, project.name, self.config.project_pattern.regex_flags - ) or re.match( - deny_pattern, - self._get_project_path(project), - self.config.project_pattern.regex_flags, - ): - return True - logger.info(f"project({project.name}) is not denied as per project_pattern") - return False + """ + Why use an explicit denial check instead of the `AllowDenyPattern.allowed` method? + + Consider a scenario where a Tableau site contains four projects: A, B, C, and D, with the following hierarchical relationship: + + - **A** + - **B** (Child of A) + - **C** (Child of A) + - **D** + + In this setup: + + - `project_pattern` is configured with `allow: ["A"]` and `deny: ["B"]`. + - `extract_project_hierarchy` is set to `True`. + + The goal is to extract assets from project A and its children while explicitly denying the child project B. + + If we rely solely on the `project_pattern.allowed()` method, project C's assets will not be ingested. + This happens because project C is not explicitly included in the `allow` list, nor is it part of the `deny` list. + However, since `extract_project_hierarchy` is enabled, project C should ideally be included in the ingestion process unless explicitly denied. + + To address this, the function explicitly checks the deny regex to ensure that project C’s assets are ingested if it is not specifically denied in the deny list. This approach ensures that the hierarchy is respected while adhering to the configured allow/deny rules. + """ + + # Either project_pattern or project_path_pattern is set in a recipe + # TableauConfig.projects_backward_compatibility ensures that at least one of these properties is configured. + + return self.config.project_pattern.denied( + project.name + ) or self.config.project_path_pattern.denied(self._get_project_path(project)) def _init_tableau_project_registry(self, all_project_map: dict) -> None: list_of_skip_projects: List[TableauProject] = [] @@ -999,9 +1016,11 @@ def _init_tableau_project_registry(self, all_project_map: dict) -> None: for project in list_of_skip_projects: if ( project.parent_id in projects_to_ingest - and self._is_denied_project(project) is False + and not self._is_denied_project(project) ): - logger.debug(f"Project {project.name} is added in project registry") + logger.debug( + f"Project {project.name} is added in project registry as it's a child project and not explicitly denied in `deny` list" + ) projects_to_ingest[project.id] = project # We rely on automatic browse paths (v2) when creating containers. That's why we need to sort the projects here. From 11c49ec6be3c2145151a91924799138d4b56bb3b Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 11 Dec 2024 17:31:06 -0500 Subject: [PATCH 11/47] fix(ui) Add backwards compatibility to the UI for old policy filters (#12017) --- .../permissions/policy/PolicyDetailsModal.tsx | 8 +- .../policy/PolicyPrivilegeForm.tsx | 29 +++- .../policy/_tests_/policyUtils.test.tsx | 160 +++++++++++++++++- .../src/app/permissions/policy/constants.ts | 4 + .../src/app/permissions/policy/policyUtils.ts | 24 ++- 5 files changed, 209 insertions(+), 16 deletions(-) create mode 100644 datahub-web-react/src/app/permissions/policy/constants.ts diff --git a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx index 37349585fa4c9..1988fea349699 100644 --- a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx +++ b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx @@ -12,6 +12,7 @@ import { mapResourceTypeToDisplayName, } from './policyUtils'; import AvatarsGroup from '../AvatarsGroup'; +import { RESOURCE_TYPE, RESOURCE_URN, TYPE, URN } from './constants'; type PrivilegeOptionType = { type?: string; @@ -72,10 +73,11 @@ export default function PolicyDetailsModal({ policy, open, onClose, privileges } const isMetadataPolicy = policy?.type === PolicyType.Metadata; const resources = convertLegacyResourceFilter(policy?.resources); - const resourceTypes = getFieldValues(resources?.filter, 'TYPE') || []; + const resourceTypes = getFieldValues(resources?.filter, TYPE, RESOURCE_TYPE) || []; const dataPlatformInstances = getFieldValues(resources?.filter, 'DATA_PLATFORM_INSTANCE') || []; - const resourceEntities = getFieldValues(resources?.filter, 'URN') || []; - const resourceFilterCondition = getFieldCondition(resources?.filter, 'URN') || PolicyMatchCondition.Equals; + const resourceEntities = getFieldValues(resources?.filter, URN, RESOURCE_URN) || []; + const resourceFilterCondition = + getFieldCondition(resources?.filter, URN, RESOURCE_URN) || PolicyMatchCondition.Equals; const domains = getFieldValues(resources?.filter, 'DOMAIN') || []; const { diff --git a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx index 7a0de67f41419..414346c2776db 100644 --- a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx +++ b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx @@ -28,6 +28,7 @@ import ClickOutside from '../../shared/ClickOutside'; import { TagTermLabel } from '../../shared/tags/TagTermLabel'; import { ENTER_KEY_CODE } from '../../shared/constants'; import { useGetRecommendations } from '../../shared/recommendation'; +import { RESOURCE_TYPE, RESOURCE_URN, TYPE, URN } from './constants'; type Props = { policyType: PolicyType; @@ -102,8 +103,9 @@ export default function PolicyPrivilegeForm({ } = useAppConfig(); const resources: ResourceFilter = convertLegacyResourceFilter(maybeResources) || EMPTY_POLICY.resources; - const resourceTypes = getFieldValues(resources.filter, 'TYPE') || []; - const resourceEntities = getFieldValues(resources.filter, 'URN') || []; + // RESOURCE_TYPE and RESOURCE_URN are deprecated, but need to get them for backwards compatibility + const resourceTypes = getFieldValues(resources.filter, TYPE, RESOURCE_TYPE) || []; + const resourceEntities = getFieldValues(resources.filter, URN, RESOURCE_URN) || []; const getDisplayName = (entity) => { if (!entity) { @@ -178,9 +180,14 @@ export default function PolicyPrivilegeForm({ const filter = resources.filter || { criteria: [], }; + // remove the deprecated RESOURCE_TYPE field and replace with TYPE field + const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_TYPE, []); setResources({ ...resources, - filter: setFieldValues(filter, 'TYPE', [...resourceTypes, createCriterionValue(selectedResourceType)]), + filter: setFieldValues(filterWithoutDeprecatedField, TYPE, [ + ...resourceTypes, + createCriterionValue(selectedResourceType), + ]), }); }; @@ -188,11 +195,13 @@ export default function PolicyPrivilegeForm({ const filter = resources.filter || { criteria: [], }; + // remove the deprecated RESOURCE_TYPE field and replace with TYPE field + const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_TYPE, []); setResources({ ...resources, filter: setFieldValues( - filter, - 'TYPE', + filterWithoutDeprecatedField, + TYPE, resourceTypes?.filter((criterionValue) => criterionValue.value !== deselectedResourceType), ), }); @@ -203,9 +212,11 @@ export default function PolicyPrivilegeForm({ const filter = resources.filter || { criteria: [], }; + // remove the deprecated RESOURCE_URN field and replace with URN field + const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_URN, []); setResources({ ...resources, - filter: setFieldValues(filter, 'URN', [ + filter: setFieldValues(filterWithoutDeprecatedField, URN, [ ...resourceEntities, createCriterionValueWithEntity( resource, @@ -220,11 +231,13 @@ export default function PolicyPrivilegeForm({ const filter = resources.filter || { criteria: [], }; + // remove the deprecated RESOURCE_URN field and replace with URN field + const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_URN, []); setResources({ ...resources, filter: setFieldValues( - filter, - 'URN', + filterWithoutDeprecatedField, + URN, resourceEntities?.filter((criterionValue) => criterionValue.value !== resource), ), }); diff --git a/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx b/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx index 1c9884e5fcf09..eae735b3477f0 100644 --- a/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx +++ b/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx @@ -1,4 +1,12 @@ -import { addOrUpdatePoliciesInList, updateListPoliciesCache, removeFromListPoliciesCache } from '../policyUtils'; +import { PolicyMatchCondition } from '../../../../types.generated'; +import { + addOrUpdatePoliciesInList, + updateListPoliciesCache, + removeFromListPoliciesCache, + getFieldValues, + getFieldCondition, + setFieldValues, +} from '../policyUtils'; // Mock the Apollo Client readQuery and writeQuery methods const mockReadQuery = vi.fn(); @@ -103,3 +111,153 @@ describe('removeFromListPoliciesCache', () => { }); }); }); + +describe('getFieldValues', () => { + it('should get field values for a given field', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataset' }, { value: 'dataJob' }], + }, + ], + }; + + expect(getFieldValues(filter, 'TYPE')).toMatchObject([{ value: 'dataset' }, { value: 'dataJob' }]); + }); + + it('should get field values for a alternate field (for deprecated fields)', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'RESOURCE_TYPE', + values: [{ value: 'dataset' }, { value: 'dataJob' }], + }, + ], + }; + + expect(getFieldValues(filter, 'TYPE', 'RESOURCE_TYPE')).toMatchObject([ + { value: 'dataset' }, + { value: 'dataJob' }, + ]); + }); + + it('should get field values for main field with alternative field given and has values', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'RESOURCE_TYPE', + values: [{ value: 'container' }, { value: 'dataFlow' }], + }, + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataset' }, { value: 'dataJob' }], + }, + ], + }; + + // should only return values from main field + expect(getFieldValues(filter, 'TYPE', 'RESOURCE_TYPE')).toMatchObject([ + { value: 'dataset' }, + { value: 'dataJob' }, + ]); + }); +}); + +describe('getFieldCondition', () => { + it('should get field values for a given field', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataset' }], + }, + ], + }; + + expect(getFieldCondition(filter, 'TYPE')).toBe(PolicyMatchCondition.Equals); + }); + + it('should get field values for a alternate field (for deprecated fields)', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'RESOURCE_TYPE', + values: [{ value: 'dataset' }], + }, + ], + }; + + expect(getFieldCondition(filter, 'TYPE', 'RESOURCE_TYPE')).toBe(PolicyMatchCondition.Equals); + }); + + it('should get field values for main field with alternative field given and has values', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.StartsWith, + field: 'RESOURCE_TYPE', + values: [{ value: 'container' }, { value: 'dataFlow' }], + }, + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataset' }], + }, + ], + }; + + // should only return values from main field + expect(getFieldCondition(filter, 'TYPE', 'RESOURCE_TYPE')).toBe(PolicyMatchCondition.Equals); + }); +}); +describe('setFieldValues', () => { + it('should remove a field if you pass in an empty array', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'RESOURCE_TYPE', + values: [{ value: 'dataset' }], + }, + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataJob' }], + }, + ], + }; + + expect(setFieldValues(filter, 'RESOURCE_TYPE', [])).toMatchObject({ + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataJob' }], + }, + ], + }); + }); + + it('should set values for a field properly', () => { + const filter = { + criteria: [], + }; + + expect(setFieldValues(filter, 'TYPE', [{ value: 'dataFlow' }])).toMatchObject({ + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataFlow' }], + }, + ], + }); + }); +}); diff --git a/datahub-web-react/src/app/permissions/policy/constants.ts b/datahub-web-react/src/app/permissions/policy/constants.ts new file mode 100644 index 0000000000000..cdd20bf9b50d6 --- /dev/null +++ b/datahub-web-react/src/app/permissions/policy/constants.ts @@ -0,0 +1,4 @@ +export const TYPE = 'TYPE'; +export const RESOURCE_TYPE = 'RESOURCE_TYPE'; +export const URN = 'URN'; +export const RESOURCE_URN = 'RESOURCE_URN'; diff --git a/datahub-web-react/src/app/permissions/policy/policyUtils.ts b/datahub-web-react/src/app/permissions/policy/policyUtils.ts index b71a38f80fc25..d6221a0a9293a 100644 --- a/datahub-web-react/src/app/permissions/policy/policyUtils.ts +++ b/datahub-web-react/src/app/permissions/policy/policyUtils.ts @@ -114,12 +114,28 @@ export const convertLegacyResourceFilter = (resourceFilter: Maybe | undefined, resourceFieldType: string) => { - return filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.values || []; +export const getFieldValues = ( + filter: Maybe | undefined, + resourceFieldType: string, + alternateResourceFieldType?: string, +) => { + return ( + filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.values || + filter?.criteria?.find((criterion) => criterion.field === alternateResourceFieldType)?.values || + [] + ); }; -export const getFieldCondition = (filter: Maybe | undefined, resourceFieldType: string) => { - return filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.condition || null; +export const getFieldCondition = ( + filter: Maybe | undefined, + resourceFieldType: string, + alternateResourceFieldType?: string, +) => { + return ( + filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.condition || + filter?.criteria?.find((criterion) => criterion.field === alternateResourceFieldType)?.condition || + null + ); }; export const getFieldValuesOfTags = (filter: Maybe | undefined, resourceFieldType: string) => { From 8b5fb711929e465c45c58e6db8c657c2623cb5e9 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 11 Dec 2024 18:45:46 -0500 Subject: [PATCH 12/47] feat(structuredProps) Add frontend for managing structured props and filtering by them (#12097) --- .../datahub/graphql/GmsGraphQLEngine.java | 14 + .../authorization/AuthorizationUtils.java | 5 + .../datahub/graphql/resolvers/MeResolver.java | 4 + .../resolvers/config/AppConfigResolver.java | 1 + .../AggregateAcrossEntitiesResolver.java | 8 +- .../search/SearchAcrossEntitiesResolver.java | 80 ++- .../graphql/resolvers/search/SearchUtils.java | 22 + .../graphql/types/mappers/MapperUtils.java | 3 + .../src/main/resources/app.graphql | 14 + .../src/main/resources/search.graphql | 10 + .../SearchAcrossEntitiesResolverTest.java | 6 +- datahub-web-react/src/Mocks.tsx | 66 ++- .../SearchBar/SearchBar.stories.tsx | 99 ++++ .../components/SearchBar/SearchBar.tsx | 30 ++ .../components/SearchBar/components.ts | 33 ++ .../components/SearchBar/index.ts | 1 + .../components/SearchBar/types.ts | 7 + .../src/alchemy-components/index.ts | 1 + datahub-web-react/src/app/SearchRoutes.tsx | 18 +- datahub-web-react/src/app/analytics/event.ts | 86 +++- .../src/app/buildEntityRegistry.ts | 2 + .../src/app/entity/Access/RoleEntity.tsx | 4 + datahub-web-react/src/app/entity/Entity.tsx | 5 + .../src/app/entity/EntityRegistry.tsx | 4 + .../BusinessAttributeEntity.tsx | 2 + .../src/app/entity/chart/ChartEntity.tsx | 2 + .../app/entity/container/ContainerEntity.tsx | 2 + .../app/entity/dashboard/DashboardEntity.tsx | 2 + .../app/entity/dataFlow/DataFlowEntity.tsx | 2 + .../src/app/entity/dataJob/DataJobEntity.tsx | 2 + .../dataPlatform/DataPlatformEntity.tsx | 4 + .../DataPlatformInstanceEntity.tsx | 4 + .../entity/dataProduct/DataProductEntity.tsx | 4 + .../src/app/entity/dataset/DatasetEntity.tsx | 2 + .../src/app/entity/domain/DomainEntity.tsx | 2 + .../ERModelRelationshipEntity.tsx | 2 + .../glossaryNode/GlossaryNodeEntity.tsx | 2 + .../glossaryTerm/GlossaryTermEntity.tsx | 2 + .../src/app/entity/group/Group.tsx | 2 + .../app/entity/mlFeature/MLFeatureEntity.tsx | 2 + .../mlFeatureTable/MLFeatureTableEntity.tsx | 2 + .../src/app/entity/mlModel/MLModelEntity.tsx | 2 + .../mlModelGroup/MLModelGroupEntity.tsx | 2 + .../mlPrimaryKey/MLPrimaryKeyEntity.tsx | 2 + .../SchemaFieldPropertiesEntity.tsx | 2 + .../Properties/useStructuredProperties.tsx | 19 + .../StructuredPropertyEntity.tsx | 86 ++++ datahub-web-react/src/app/entity/tag/Tag.tsx | 2 + .../src/app/entity/user/User.tsx | 2 + .../structuredProperties/AdvancedOptions.tsx | 64 +++ .../AllowedValuesDrawer.tsx | 142 ++++++ .../AllowedValuesField.tsx | 77 +++ .../DisplayPreferences.tsx | 165 ++++++ .../EmptyStructuredProperties.tsx | 29 ++ .../structuredProperties/RequiredAsterisk.tsx | 12 + .../StructuredProperties.tsx | 115 +++++ .../StructuredPropsDrawer.tsx | 396 +++++++++++++++ .../StructuredPropsForm.tsx | 139 +++++ .../StructuredPropsFormSection.tsx | 176 +++++++ .../StructuredPropsTable.tsx | 376 ++++++++++++++ .../ViewAdvancedOptions.tsx | 43 ++ .../ViewDisplayPreferences.tsx | 86 ++++ .../ViewStructuredPropsDrawer.tsx | 142 ++++++ .../govern/structuredProperties/cacheUtils.ts | 102 ++++ .../structuredProperties/styledComponents.ts | 473 ++++++++++++++++++ .../structuredProperties/useStructuredProp.ts | 135 +++++ .../app/govern/structuredProperties/utils.ts | 249 +++++++++ .../src/app/search/SearchPage.tsx | 2 +- .../src/app/search/filters/ActiveFilter.tsx | 11 +- .../filters/DateRangeMenu/DateRangeMenu.tsx | 88 ++++ .../DateRangeMenu/useDateRangeFilterValues.ts | 48 ++ .../EntityTypeFilter/entityTypeFilterUtils.ts | 2 +- .../src/app/search/filters/FilterOption.tsx | 9 +- .../app/search/filters/MoreFilterOption.tsx | 17 +- .../search/filters/OptionsDropdownMenu.tsx | 15 + .../src/app/search/filters/SearchFilter.tsx | 8 +- .../app/search/filters/SearchFilterView.tsx | 12 +- .../app/search/filters/mapFilterOption.tsx | 2 +- .../src/app/search/filters/types.ts | 1 + .../filters/useSearchFilterDropdown.tsx | 18 +- .../src/app/search/filters/utils.tsx | 62 ++- .../src/app/search/utils/constants.ts | 5 + .../src/app/shared/admin/HeaderLinks.tsx | 25 + datahub-web-react/src/app/shared/constants.ts | 3 + .../app/sharedV2/modals/ConfirmationModal.tsx | 76 +++ .../src/app/sharedV2/toastMessageUtils.ts | 18 + datahub-web-react/src/appConfigContext.tsx | 1 + datahub-web-react/src/conf/Global.ts | 1 + datahub-web-react/src/graphql/app.graphql | 1 + .../src/graphql/fragments.graphql | 55 ++ datahub-web-react/src/graphql/lineage.graphql | 35 ++ datahub-web-react/src/graphql/me.graphql | 2 + datahub-web-react/src/graphql/search.graphql | 90 ++++ .../src/graphql/structuredProperties.graphql | 137 +++++ datahub-web-react/src/images/empty-forms.svg | 48 ++ datahub-web-react/src/images/table-icon.svg | 3 + .../request/AggregationQueryBuilder.java | 54 +- .../fixtures/SampleDataFixtureTestBase.java | 17 +- .../request/AggregationQueryBuilderTest.java | 117 +++-- .../request/SearchRequestHandlerTest.java | 3 +- .../linkedin/metadata/query/SearchFlags.pdl | 6 + .../metadata/search/AggregationMetadata.pdl | 5 + .../graphql/featureflags/FeatureFlags.java | 1 + .../src/main/resources/application.yaml | 1 + ...linkedin.analytics.analytics.snapshot.json | 2 +- .../com.linkedin.entity.aspects.snapshot.json | 54 +- ...com.linkedin.entity.entities.snapshot.json | 75 ++- .../com.linkedin.entity.runs.snapshot.json | 52 +- ...nkedin.operations.operations.snapshot.json | 52 +- ...m.linkedin.platform.platform.snapshot.json | 60 ++- .../datahubusage/DataHubUsageEventType.java | 10 +- .../war/src/main/resources/boot/policies.json | 6 +- .../authorization/PoliciesConfig.java | 6 + 113 files changed, 4694 insertions(+), 223 deletions(-) create mode 100644 datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/SearchBar/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/SearchBar/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/SearchBar/types.ts create mode 100644 datahub-web-react/src/app/entity/structuredProperty/StructuredPropertyEntity.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/AdvancedOptions.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/AllowedValuesDrawer.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/AllowedValuesField.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/DisplayPreferences.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/EmptyStructuredProperties.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/RequiredAsterisk.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/StructuredProperties.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/StructuredPropsDrawer.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/StructuredPropsForm.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/StructuredPropsFormSection.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/StructuredPropsTable.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/ViewAdvancedOptions.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/ViewDisplayPreferences.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/ViewStructuredPropsDrawer.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/cacheUtils.ts create mode 100644 datahub-web-react/src/app/govern/structuredProperties/styledComponents.ts create mode 100644 datahub-web-react/src/app/govern/structuredProperties/useStructuredProp.ts create mode 100644 datahub-web-react/src/app/govern/structuredProperties/utils.ts create mode 100644 datahub-web-react/src/app/search/filters/DateRangeMenu/DateRangeMenu.tsx create mode 100644 datahub-web-react/src/app/search/filters/DateRangeMenu/useDateRangeFilterValues.ts create mode 100644 datahub-web-react/src/app/sharedV2/modals/ConfirmationModal.tsx create mode 100644 datahub-web-react/src/app/sharedV2/toastMessageUtils.ts create mode 100644 datahub-web-react/src/images/empty-forms.svg create mode 100644 datahub-web-react/src/images/table-icon.svg diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index d1da55268a50d..079a20619d1ea 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -67,6 +67,7 @@ import com.linkedin.datahub.graphql.generated.EntityPath; import com.linkedin.datahub.graphql.generated.EntityRelationship; import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy; +import com.linkedin.datahub.graphql.generated.FacetMetadata; import com.linkedin.datahub.graphql.generated.ForeignKeyConstraint; import com.linkedin.datahub.graphql.generated.FormActorAssignment; import com.linkedin.datahub.graphql.generated.FreshnessContract; @@ -1474,6 +1475,19 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder "entity", new EntityTypeResolver( entityTypes, (env) -> ((BrowsePathEntry) env.getSource()).getEntity()))) + .type( + "FacetMetadata", + typeWiring -> + typeWiring.dataFetcher( + "entity", + new EntityTypeResolver( + entityTypes, + (env) -> { + FacetMetadata facetMetadata = env.getSource(); + return facetMetadata.getEntity() != null + ? facetMetadata.getEntity() + : null; + }))) .type( "LineageRelationship", typeWiring -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java index ca60acaa80538..c25d6af75fe76 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java @@ -339,6 +339,11 @@ public static boolean canManageStructuredProperties(@Nonnull QueryContext contex context.getOperationContext(), PoliciesConfig.MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE); } + public static boolean canViewStructuredPropertiesPage(@Nonnull QueryContext context) { + return AuthUtil.isAuthorized( + context.getOperationContext(), PoliciesConfig.VIEW_STRUCTURED_PROPERTIES_PAGE_PRIVILEGE); + } + public static boolean canManageForms(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( context.getOperationContext(), PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java index b1101ae3ee865..8297392e642d5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java @@ -93,6 +93,10 @@ public CompletableFuture get(DataFetchingEnvironment environm BusinessAttributeAuthorizationUtils.canCreateBusinessAttribute(context)); platformPrivileges.setManageBusinessAttributes( BusinessAttributeAuthorizationUtils.canManageBusinessAttribute(context)); + platformPrivileges.setManageStructuredProperties( + AuthorizationUtils.canManageStructuredProperties(context)); + platformPrivileges.setViewStructuredPropertiesPage( + AuthorizationUtils.canViewStructuredPropertiesPage(context)); // Construct and return authenticated user object. final AuthenticatedUser authUser = new AuthenticatedUser(); authUser.setCorpUser(corpUser); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index 259d05c631557..3647eb55b2583 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -188,6 +188,7 @@ public CompletableFuture get(final DataFetchingEnvironment environmen .setDataContractsEnabled(_featureFlags.isDataContractsEnabled()) .setEditableDatasetNameEnabled(_featureFlags.isEditableDatasetNameEnabled()) .setShowSeparateSiblings(_featureFlags.isShowSeparateSiblings()) + .setShowManageStructuredProperties(_featureFlags.isShowManageStructuredProperties()) .build(); appConfig.setFeatureFlags(featureFlagsConfig); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java index 29b71d95ad974..31ed2de7a6d51 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java @@ -66,11 +66,17 @@ public CompletableFuture get(DataFetchingEnvironment environme final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); - final SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags()); + final SearchFlags searchFlags = + input.getSearchFlags() != null + ? mapInputFlags(context, input.getSearchFlags()) + : new SearchFlags(); final List facets = input.getFacets() != null && input.getFacets().size() > 0 ? input.getFacets() : null; + // do not include default facets if we're requesting any facets specifically + searchFlags.setIncludeDefaultFacets(facets == null || facets.size() <= 0); + List finalEntities = maybeResolvedView != null ? SearchUtils.intersectEntityTypes( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java index d103704146d39..29bc3a82a1649 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java @@ -2,19 +2,28 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.*; +import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.getEntityNames; +import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; +import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.SearchAcrossEntitiesInput; import com.linkedin.datahub.graphql.generated.SearchResults; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.service.ViewService; +import com.linkedin.metadata.utils.CriterionUtils; import com.linkedin.view.DataHubViewInfo; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -64,24 +73,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags()); - List sortCriteria; - if (input.getSortInput() != null) { - if (input.getSortInput().getSortCriteria() != null) { - sortCriteria = - input.getSortInput().getSortCriteria().stream() - .map(SearchUtils::mapSortCriterion) - .collect(Collectors.toList()); - } else { - sortCriteria = - input.getSortInput().getSortCriterion() != null - ? Collections.singletonList( - mapSortCriterion(input.getSortInput().getSortCriterion())) - : Collections.emptyList(); - } - - } else { - sortCriteria = Collections.emptyList(); - } + List sortCriteria = SearchUtils.getSortCriteria(input.getSortInput()); try { log.debug( @@ -101,6 +93,14 @@ public CompletableFuture get(DataFetchingEnvironment environment) return SearchUtils.createEmptySearchResults(start, count); } + boolean shouldIncludeStructuredPropertyFacets = + input.getSearchFlags() != null + && input.getSearchFlags().getIncludeStructuredPropertyFacets() != null + ? input.getSearchFlags().getIncludeStructuredPropertyFacets() + : false; + List structuredPropertyFacets = + shouldIncludeStructuredPropertyFacets ? getStructuredPropertyFacets(context) : null; + return UrnSearchResultsMapper.map( context, _entityClient.searchAcrossEntities( @@ -113,7 +113,8 @@ public CompletableFuture get(DataFetchingEnvironment environment) : baseFilter, start, count, - sortCriteria)); + sortCriteria, + structuredPropertyFacets)); } catch (Exception e) { log.error( "Failed to execute search for multiple entities: entity types {}, query {}, filters: {}, start: {}, count: {}", @@ -133,4 +134,45 @@ public CompletableFuture get(DataFetchingEnvironment environment) this.getClass().getSimpleName(), "get"); } + + private List getStructuredPropertyFacets(final QueryContext context) { + try { + SearchFlags searchFlags = new SearchFlags().setSkipCache(true); + SearchResult result = + _entityClient.searchAcrossEntities( + context.getOperationContext().withSearchFlags(flags -> searchFlags), + getEntityNames(ImmutableList.of(EntityType.STRUCTURED_PROPERTY)), + "*", + createStructuredPropertyFilter(), + 0, + 100, + Collections.emptyList(), + null); + return result.getEntities().stream() + .map(entity -> String.format("structuredProperties.%s", entity.getEntity().getId())) + .collect(Collectors.toList()); + } catch (Exception e) { + log.error("Failed to get structured property facets to filter on", e); + return Collections.emptyList(); + } + } + + private Filter createStructuredPropertyFilter() { + return new Filter() + .setOr( + new ConjunctiveCriterionArray( + ImmutableList.of( + new ConjunctiveCriterion() + .setAnd( + new CriterionArray( + ImmutableList.of( + CriterionUtils.buildCriterion( + "filterStatus", Condition.EQUAL, "ENABLED")))), + new ConjunctiveCriterion() + .setAnd( + new CriterionArray( + ImmutableList.of( + CriterionUtils.buildCriterion( + "showInSearchFilters", Condition.EQUAL, "true"))))))); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java index 04777c3fcdb4e..a01b3aaec9c98 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java @@ -22,6 +22,7 @@ import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; +import com.linkedin.datahub.graphql.generated.SearchSortInput; import com.linkedin.datahub.graphql.types.common.mappers.SearchFlagsInputMapper; import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.query.SearchFlags; @@ -326,4 +327,25 @@ public static SearchResults createEmptySearchResults(final int start, final int result.setFacets(new ArrayList<>()); return result; } + + public static List getSortCriteria(@Nullable final SearchSortInput sortInput) { + List sortCriteria; + if (sortInput != null) { + if (sortInput.getSortCriteria() != null) { + sortCriteria = + sortInput.getSortCriteria().stream() + .map(SearchUtils::mapSortCriterion) + .collect(Collectors.toList()); + } else { + sortCriteria = + sortInput.getSortCriterion() != null + ? Collections.singletonList(mapSortCriterion(sortInput.getSortCriterion())) + : new ArrayList<>(); + } + } else { + sortCriteria = new ArrayList<>(); + } + + return sortCriteria; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java index 0d69e62c621a6..8fe58df2d2ede 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java @@ -70,6 +70,9 @@ public static FacetMetadata mapFacet( aggregationFacets.stream() .map(facet -> facet.equals("entity") || facet.contains("_entityType")) .collect(Collectors.toList()); + if (aggregationMetadata.getEntity() != null) { + facetMetadata.setEntity(UrnToEntityMapper.map(context, aggregationMetadata.getEntity())); + } facetMetadata.setField(aggregationMetadata.getName()); facetMetadata.setDisplayName( Optional.ofNullable(aggregationMetadata.getDisplayName()) diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index 262d2384d84ad..2868890368723 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -156,6 +156,15 @@ type PlatformPrivileges { """ manageBusinessAttributes: Boolean! + """ + Whether the user can create, edit, and delete structured properties. + """ + manageStructuredProperties: Boolean! + + """ + Whether the user can view the manage structured properties page. + """ + viewStructuredPropertiesPage: Boolean! } """ @@ -517,6 +526,11 @@ type FeatureFlagsConfig { If turned on, all siblings will be separated with no way to get to a "combined" sibling view """ showSeparateSiblings: Boolean! + + """ + If turned on, show the manage structured properties tab in the govern dropdown + """ + showManageStructuredProperties: Boolean! } """ diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql index d0f669f05f959..82bfb9ee26fc4 100644 --- a/datahub-graphql-core/src/main/resources/search.graphql +++ b/datahub-graphql-core/src/main/resources/search.graphql @@ -167,6 +167,11 @@ input SearchFlags { fields to include for custom Highlighting """ customHighlightingFields: [String!] + + """ + Whether or not to fetch and request for structured property facets when doing a search + """ + includeStructuredPropertyFacets: Boolean } """ @@ -872,6 +877,11 @@ type FacetMetadata { """ displayName: String + """ + Entity corresponding to the facet + """ + entity: Entity + """ Aggregated search result counts by value of the field """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java index 42768b8a2de21..89d218683e33e 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java @@ -471,7 +471,8 @@ private static EntityClient initMockEntityClient( Mockito.eq(filter), Mockito.eq(start), Mockito.eq(limit), - Mockito.eq(Collections.emptyList()))) + Mockito.eq(Collections.emptyList()), + Mockito.eq(null))) .thenReturn(result); return client; } @@ -496,7 +497,8 @@ private static void verifyMockEntityClient( Mockito.eq(filter), Mockito.eq(start), Mockito.eq(limit), - Mockito.eq(Collections.emptyList())); + Mockito.eq(Collections.emptyList()), + Mockito.eq(null)); } private static void verifyMockViewService(ViewService mockService, Urn viewUrn) { diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index 329d6250e576a..73a789030ce6f 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -2204,7 +2204,7 @@ export const mocks = [ count: 10, filters: [], orFilters: [], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -2244,6 +2244,7 @@ export const mocks = [ field: 'origin', displayName: 'origin', aggregations: [{ value: 'PROD', count: 3, entity: null }], + entity: null, }, { field: '_entityType', @@ -2252,6 +2253,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -2261,6 +2263,7 @@ export const mocks = [ { value: 'MySQL', count: 1, entity: null }, { value: 'Kafka', count: 1, entity: null }, ], + entity: null, }, ], suggestions: [], @@ -2290,7 +2293,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -2325,6 +2328,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -2333,6 +2337,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -2343,6 +2348,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], suggestions: [], @@ -2393,6 +2399,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -2401,6 +2408,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -2410,6 +2418,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -2464,7 +2473,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -2501,6 +2510,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -2510,6 +2520,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -2520,6 +2531,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null, __typename: 'AggregationMetadata' }, { value: 'kafka', count: 1, entity: null, __typename: 'AggregationMetadata' }, ], + entity: null, }, ], }, @@ -2669,6 +2681,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -2677,6 +2690,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -2686,6 +2700,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -2743,6 +2758,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -2751,6 +2767,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -2760,6 +2777,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -2809,6 +2827,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: 'platform', @@ -2822,6 +2841,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -2953,6 +2973,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: 'platform', @@ -2966,6 +2987,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -3013,7 +3035,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3050,6 +3072,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, // { // displayName: 'Domain', @@ -3071,6 +3094,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -3096,6 +3120,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, ], }, @@ -3181,7 +3206,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3215,6 +3240,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: 'platform', @@ -3228,6 +3254,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -3256,7 +3283,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3290,6 +3317,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -3298,6 +3326,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -3307,6 +3336,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -3335,7 +3365,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3377,6 +3407,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -3385,6 +3416,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -3394,6 +3426,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -3428,7 +3461,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3465,6 +3498,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -3474,6 +3508,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -3484,6 +3519,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null, __typename: 'AggregationMetadata' }, { value: 'kafka', count: 1, entity: null, __typename: 'AggregationMetadata' }, ], + entity: null, }, ], }, @@ -3518,7 +3554,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3555,6 +3591,7 @@ export const mocks = [ __typename: 'AggregationMetadata', }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -3564,6 +3601,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -3574,6 +3612,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null, __typename: 'AggregationMetadata' }, { value: 'kafka', count: 1, entity: null, __typename: 'AggregationMetadata' }, ], + entity: null, }, ], }, @@ -3635,6 +3674,8 @@ export const mocks = [ manageGlobalAnnouncements: true, createBusinessAttributes: true, manageBusinessAttributes: true, + manageStructuredProperties: true, + viewStructuredPropertiesPage: true, }, }, }, @@ -3722,7 +3763,7 @@ export const mocks = [ count: 10, filters: [], orFilters: [], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3821,6 +3862,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -3829,6 +3871,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -3838,6 +3881,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -3912,4 +3956,6 @@ export const platformPrivileges: PlatformPrivileges = { manageGlobalAnnouncements: true, createBusinessAttributes: true, manageBusinessAttributes: true, + manageStructuredProperties: true, + viewStructuredPropertiesPage: true, }; diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.stories.tsx b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.stories.tsx new file mode 100644 index 0000000000000..9ae34356a71d6 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.stories.tsx @@ -0,0 +1,99 @@ +import React, { useState } from 'react'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; +import type { Meta, StoryObj } from '@storybook/react'; +import { GridList } from '@components/.docs/mdx-components'; +import { SearchBar, searchBarDefaults } from './SearchBar'; +import { SearchBarProps } from './types'; + +const meta = { + title: 'Components / Search Bar', + component: SearchBar, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'A component that is used to get search bar', + }, + }, + + // Component-level argTypes + argTypes: { + placeholder: { + description: 'Placeholder of search bar.', + table: { + defaultValue: { summary: searchBarDefaults.placeholder }, + }, + control: { + type: 'text', + }, + }, + value: { + description: 'Value of the search bar.', + table: { + defaultValue: { summary: searchBarDefaults.value }, + }, + control: false, + }, + width: { + description: 'Width of the search bar.', + table: { + defaultValue: { summary: searchBarDefaults.width }, + }, + control: { + type: 'text', + }, + }, + allowClear: { + description: 'Whether clear button should be present.', + table: { + defaultValue: { summary: searchBarDefaults.allowClear?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + onChange: { + description: 'On change function for the search bar.', + }, + }, + + // Define defaults + args: { + placeholder: searchBarDefaults.placeholder, + value: searchBarDefaults.value, + allowClear: searchBarDefaults.allowClear, + width: searchBarDefaults.width, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +const SandboxWrapper = (props: SearchBarProps) => { + const [value, setValue] = useState(''); + + const handleChange = (newValue: string) => { + setValue(newValue); + }; + + return ; +}; + +export const sandbox: Story = { + tags: ['dev'], + render: (props) => { + return ; + }, +}; + +export const customWidths = () => ( + + + + + + +); diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.tsx b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.tsx new file mode 100644 index 0000000000000..f39f761058d8c --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.tsx @@ -0,0 +1,30 @@ +import { SearchOutlined } from '@ant-design/icons'; +import React from 'react'; +import { StyledSearchBar } from './components'; +import { SearchBarProps } from './types'; + +export const searchBarDefaults: SearchBarProps = { + placeholder: 'Search..', + value: '', + width: '272px', + allowClear: true, +}; + +export const SearchBar = ({ + placeholder = searchBarDefaults.placeholder, + value = searchBarDefaults.value, + width = searchBarDefaults.width, + allowClear = searchBarDefaults.allowClear, + onChange, +}: SearchBarProps) => { + return ( + onChange?.(e.target.value)} + value={value} + prefix={} + allowClear={allowClear} + $width={width} + /> + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/components.ts b/datahub-web-react/src/alchemy-components/components/SearchBar/components.ts new file mode 100644 index 0000000000000..7045801ddf092 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/SearchBar/components.ts @@ -0,0 +1,33 @@ +import { colors, typography } from '@src/alchemy-components/theme'; +import { Input } from 'antd'; +import styled from 'styled-components'; + +export const StyledSearchBar = styled(Input)<{ $width?: string }>` + height: 40px; + width: ${(props) => props.$width}; + display: flex; + align-items: center; + border-radius: 8px; + + input { + color: ${colors.gray[600]}; + font-size: ${typography.fontSizes.md} !important; + } + + .ant-input-prefix { + width: 20px; + color: ${colors.gray[1800]}; + + svg { + height: 16px; + width: 16px; + } + } + + &:hover, + &:focus, + &:focus-within { + border-color: ${colors.violet[300]} !important; + box-shadow: none !important; + } +`; diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/index.ts b/datahub-web-react/src/alchemy-components/components/SearchBar/index.ts new file mode 100644 index 0000000000000..8c1933163b29f --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/SearchBar/index.ts @@ -0,0 +1 @@ +export { SearchBar } from './SearchBar'; diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/types.ts b/datahub-web-react/src/alchemy-components/components/SearchBar/types.ts new file mode 100644 index 0000000000000..04ac218dc2cc6 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/SearchBar/types.ts @@ -0,0 +1,7 @@ +export interface SearchBarProps { + placeholder?: string; + value?: string; + width?: string; + onChange?: (value: string) => void; + allowClear?: boolean; +} diff --git a/datahub-web-react/src/alchemy-components/index.ts b/datahub-web-react/src/alchemy-components/index.ts index 8ef4f73f4408f..7e40d343e884d 100644 --- a/datahub-web-react/src/alchemy-components/index.ts +++ b/datahub-web-react/src/alchemy-components/index.ts @@ -15,6 +15,7 @@ export * from './components/LineChart'; export * from './components/PageTitle'; export * from './components/Pills'; export * from './components/Popover'; +export * from './components/SearchBar'; export * from './components/Select'; export * from './components/Switch'; export * from './components/Table'; diff --git a/datahub-web-react/src/app/SearchRoutes.tsx b/datahub-web-react/src/app/SearchRoutes.tsx index 3343260c72bcf..024b6e0add15a 100644 --- a/datahub-web-react/src/app/SearchRoutes.tsx +++ b/datahub-web-react/src/app/SearchRoutes.tsx @@ -11,23 +11,36 @@ import { AnalyticsPage } from './analyticsDashboard/components/AnalyticsPage'; import { ManageIngestionPage } from './ingest/ManageIngestionPage'; import GlossaryRoutes from './glossary/GlossaryRoutes'; import { SettingsPage } from './settings/SettingsPage'; +import { useUserContext } from './context/useUserContext'; import DomainRoutes from './domain/DomainRoutes'; -import { useBusinessAttributesFlag, useIsAppConfigContextLoaded, useIsNestedDomainsEnabled } from './useAppConfig'; +import { + useAppConfig, + useBusinessAttributesFlag, + useIsAppConfigContextLoaded, + useIsNestedDomainsEnabled, +} from './useAppConfig'; import { ManageDomainsPage } from './domain/ManageDomainsPage'; import { BusinessAttributes } from './businessAttribute/BusinessAttributes'; +import StructuredProperties from './govern/structuredProperties/StructuredProperties'; /** * Container for all searchable page routes */ export const SearchRoutes = (): JSX.Element => { const entityRegistry = useEntityRegistry(); + const me = useUserContext(); const isNestedDomainsEnabled = useIsNestedDomainsEnabled(); const entities = isNestedDomainsEnabled ? entityRegistry.getEntitiesForSearchRoutes() : entityRegistry.getNonGlossaryEntities(); + const { config } = useAppConfig(); const businessAttributesFlag = useBusinessAttributesFlag(); const appConfigContextLoaded = useIsAppConfigContextLoaded(); + const showStructuredProperties = + config?.featureFlags?.showManageStructuredProperties && + (me.platformPrivileges?.manageStructuredProperties || me.platformPrivileges?.viewStructuredPropertiesPage); + return ( @@ -53,6 +66,9 @@ export const SearchRoutes = (): JSX.Element => { } /> } /> } /> + {showStructuredProperties && ( + } /> + )} { diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts index c3a57830b8c50..9152f2fb8eedb 100644 --- a/datahub-web-react/src/app/analytics/event.ts +++ b/datahub-web-react/src/app/analytics/event.ts @@ -1,4 +1,12 @@ -import { DataHubViewType, EntityType, RecommendationRenderType, ScenarioType } from '../../types.generated'; +import { + AllowedValue, + DataHubViewType, + EntityType, + PropertyCardinality, + PropertyValueInput, + RecommendationRenderType, + ScenarioType, +} from '../../types.generated'; import { EmbedLookupNotFoundReason } from '../embed/lookup/constants'; import { Direction } from '../lineage/types'; import { FilterMode } from '../search/utils/constants'; @@ -82,6 +90,14 @@ export enum EventType { EmbedProfileViewInDataHubEvent, EmbedLookupNotFoundEvent, CreateBusinessAttributeEvent, + CreateStructuredPropertyClickEvent, + CreateStructuredPropertyEvent, + EditStructuredPropertyEvent, + DeleteStructuredPropertyEvent, + ViewStructuredPropertyEvent, + ApplyStructuredPropertyEvent, + UpdateStructuredPropertyOnAssetEvent, + RemoveStructuredPropertyEvent, } /** @@ -640,6 +656,64 @@ export interface CreateBusinessAttributeEvent extends BaseEvent { name: string; } +export interface CreateStructuredPropertyClickEvent extends BaseEvent { + type: EventType.CreateStructuredPropertyClickEvent; +} + +interface StructuredPropertyEvent extends BaseEvent { + propertyType: string; + appliesTo: string[]; + qualifiedName?: string; + allowedAssetTypes?: string[]; + allowedValues?: AllowedValue[]; + cardinality?: PropertyCardinality; + showInFilters?: boolean; + isHidden: boolean; + showInSearchFilters: boolean; + showAsAssetBadge: boolean; + showInAssetSummary: boolean; + showInColumnsTable: boolean; +} + +export interface CreateStructuredPropertyEvent extends StructuredPropertyEvent { + type: EventType.CreateStructuredPropertyEvent; +} + +export interface EditStructuredPropertyEvent extends StructuredPropertyEvent { + type: EventType.EditStructuredPropertyEvent; + propertyUrn: string; +} + +export interface DeleteStructuredPropertyEvent extends StructuredPropertyEvent { + type: EventType.DeleteStructuredPropertyEvent; + propertyUrn: string; +} + +export interface ViewStructuredPropertyEvent extends BaseEvent { + type: EventType.ViewStructuredPropertyEvent; + propertyUrn: string; +} + +interface StructuredPropertyOnAssetEvent extends BaseEvent { + propertyUrn: string; + propertyType: string; + assetUrn: string; + assetType: EntityType; +} +export interface ApplyStructuredPropertyEvent extends StructuredPropertyOnAssetEvent { + type: EventType.ApplyStructuredPropertyEvent; + values: PropertyValueInput[]; +} + +export interface UpdateStructuredPropertyOnAssetEvent extends StructuredPropertyOnAssetEvent { + type: EventType.UpdateStructuredPropertyOnAssetEvent; + values: PropertyValueInput[]; +} + +export interface RemoveStructuredPropertyEvent extends StructuredPropertyOnAssetEvent { + type: EventType.RemoveStructuredPropertyEvent; +} + /** * Event consisting of a union of specific event types. */ @@ -718,4 +792,12 @@ export type Event = | EmbedProfileViewEvent | EmbedProfileViewInDataHubEvent | EmbedLookupNotFoundEvent - | CreateBusinessAttributeEvent; + | CreateBusinessAttributeEvent + | CreateStructuredPropertyClickEvent + | CreateStructuredPropertyEvent + | EditStructuredPropertyEvent + | DeleteStructuredPropertyEvent + | ViewStructuredPropertyEvent + | ApplyStructuredPropertyEvent + | UpdateStructuredPropertyOnAssetEvent + | RemoveStructuredPropertyEvent; diff --git a/datahub-web-react/src/app/buildEntityRegistry.ts b/datahub-web-react/src/app/buildEntityRegistry.ts index 0b70986672be5..181ec7d328a58 100644 --- a/datahub-web-react/src/app/buildEntityRegistry.ts +++ b/datahub-web-react/src/app/buildEntityRegistry.ts @@ -24,6 +24,7 @@ import { RoleEntity } from './entity/Access/RoleEntity'; import { RestrictedEntity } from './entity/restricted/RestrictedEntity'; import { BusinessAttributeEntity } from './entity/businessAttribute/BusinessAttributeEntity'; import { SchemaFieldPropertiesEntity } from './entity/schemaField/SchemaFieldPropertiesEntity'; +import { StructuredPropertyEntity } from './entity/structuredProperty/StructuredPropertyEntity'; export default function buildEntityRegistry() { const registry = new EntityRegistry(); @@ -52,5 +53,6 @@ export default function buildEntityRegistry() { registry.register(new RestrictedEntity()); registry.register(new BusinessAttributeEntity()); registry.register(new SchemaFieldPropertiesEntity()); + registry.register(new StructuredPropertyEntity()); return registry; } diff --git a/datahub-web-react/src/app/entity/Access/RoleEntity.tsx b/datahub-web-react/src/app/entity/Access/RoleEntity.tsx index ab609b04f104a..58a1ba8dd793b 100644 --- a/datahub-web-react/src/app/entity/Access/RoleEntity.tsx +++ b/datahub-web-react/src/app/entity/Access/RoleEntity.tsx @@ -88,4 +88,8 @@ export class RoleEntity implements Entity { supportedCapabilities = () => { return new Set([EntityCapabilityType.OWNERS]); }; + + getGraphName = () => { + return 'roleEntity'; + }; } diff --git a/datahub-web-react/src/app/entity/Entity.tsx b/datahub-web-react/src/app/entity/Entity.tsx index 490f23330c594..c56c97454a1d5 100644 --- a/datahub-web-react/src/app/entity/Entity.tsx +++ b/datahub-web-react/src/app/entity/Entity.tsx @@ -172,6 +172,11 @@ export interface Entity { */ getGenericEntityProperties: (data: T) => GenericEntityProperties | null; + /** + * Returns the graph name of the entity, as it appears in the GMS entity registry + */ + getGraphName: () => string; + /** * Returns the supported features for the entity */ diff --git a/datahub-web-react/src/app/entity/EntityRegistry.tsx b/datahub-web-react/src/app/entity/EntityRegistry.tsx index 00e7385ff5784..0f65390f959df 100644 --- a/datahub-web-react/src/app/entity/EntityRegistry.tsx +++ b/datahub-web-react/src/app/entity/EntityRegistry.tsx @@ -241,4 +241,8 @@ export default class EntityRegistry { const entity = validatedGet(type, this.entityTypeToEntity); return entity.getCustomCardUrlPath?.(); } + + getGraphNameFromType(type: EntityType): string { + return validatedGet(type, this.entityTypeToEntity).getGraphName(); + } } diff --git a/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx b/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx index b827a3c37d6a5..442aaf735575a 100644 --- a/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx +++ b/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx @@ -59,6 +59,8 @@ export class BusinessAttributeEntity implements Entity { getCollectionName = () => 'Business Attributes'; + getGraphName = () => 'businessAttribute'; + getCustomCardUrlPath = () => PageRoutes.BUSINESS_ATTRIBUTE; isBrowseEnabled = () => false; diff --git a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx index 913d502972fe1..8a62a9018661e 100644 --- a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx +++ b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx @@ -69,6 +69,8 @@ export class ChartEntity implements Entity { getAutoCompleteFieldName = () => 'title'; + getGraphName = () => 'chart'; + getPathName = () => 'chart'; getEntityName = () => 'Chart'; diff --git a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx index 89f9122c6287f..941e7fc3f552d 100644 --- a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx +++ b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx @@ -59,6 +59,8 @@ export class ContainerEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'container'; + getPathName = () => 'container'; getEntityName = () => 'Container'; diff --git a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx index 9564cbc18198e..95d4431d59179 100644 --- a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx +++ b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx @@ -291,6 +291,8 @@ export class DashboardEntity implements Entity { ]); }; + getGraphName = () => this.getPathName(); + renderEmbeddedProfile = (urn: string) => ( { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'dataFlow'; + getPathName = () => 'pipelines'; getEntityName = () => 'Pipeline'; diff --git a/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx b/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx index fe1a906371e9d..6bf9548226919 100644 --- a/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx +++ b/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx @@ -64,6 +64,8 @@ export class DataJobEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'dataJob'; + getPathName = () => 'tasks'; getEntityName = () => 'Task'; diff --git a/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx b/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx index 6687ec9f914c1..89cbaf3cbeaa1 100644 --- a/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx +++ b/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx @@ -71,4 +71,8 @@ export class DataPlatformEntity implements Entity { supportedCapabilities = () => { return new Set([]); }; + + getGraphName = () => { + return 'dataPlatform'; + }; } diff --git a/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx b/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx index a542e1b52f510..d0db687ffed92 100644 --- a/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx +++ b/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx @@ -58,4 +58,8 @@ export class DataPlatformInstanceEntity implements Entity supportedCapabilities = () => { return new Set([]); }; + + getGraphName = () => { + return 'dataPlatformInstance'; + }; } diff --git a/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx b/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx index 6b31de84f85bb..90c1127d9a5fc 100644 --- a/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx +++ b/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx @@ -191,4 +191,8 @@ export class DataProductEntity implements Entity { EntityCapabilityType.DOMAINS, ]); }; + + getGraphName = () => { + return 'dataProduct'; + }; } diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx index 21ae085832cb3..07ab27a38f889 100644 --- a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx +++ b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx @@ -85,6 +85,8 @@ export class DatasetEntity implements Entity { getPathName = () => 'dataset'; + getGraphName = () => 'dataset'; + getEntityName = () => 'Dataset'; getCollectionName = () => 'Datasets'; diff --git a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx index 2b67c88a6ff23..81d245c230843 100644 --- a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx +++ b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx @@ -60,6 +60,8 @@ export class DomainEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'domain'; + getPathName = () => 'domain'; getEntityName = () => 'Domain'; diff --git a/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx b/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx index aece3db1312af..3eb950cb0e7ac 100644 --- a/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx +++ b/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx @@ -58,6 +58,8 @@ export class ERModelRelationshipEntity implements Entity { getEntityName = () => 'ER-Model-Relationship'; + getGraphName = () => 'erModelRelationship'; + renderProfile = (urn: string) => ( { EntityCapabilityType.SOFT_DELETE, ]); }; + + getGraphName = () => this.getPathName(); } export default GlossaryNodeEntity; diff --git a/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx b/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx index 8bbc0a693b223..73c5a8e12122d 100644 --- a/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx +++ b/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx @@ -179,4 +179,6 @@ export class GlossaryTermEntity implements Entity { EntityCapabilityType.SOFT_DELETE, ]); }; + + getGraphName = () => this.getPathName(); } diff --git a/datahub-web-react/src/app/entity/group/Group.tsx b/datahub-web-react/src/app/entity/group/Group.tsx index cd9cf1ca6eec4..763db856f33ac 100644 --- a/datahub-web-react/src/app/entity/group/Group.tsx +++ b/datahub-web-react/src/app/entity/group/Group.tsx @@ -40,6 +40,8 @@ export class GroupEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName: () => string = () => 'corpGroup'; + getPathName: () => string = () => 'group'; getEntityName = () => 'Group'; diff --git a/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx b/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx index 2f2786b1c0d96..eecffdb2f3843 100644 --- a/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx +++ b/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx @@ -52,6 +52,8 @@ export class MLFeatureEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'mlFeature'; + getPathName = () => 'features'; getEntityName = () => 'Feature'; diff --git a/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx b/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx index 595c73fbc3cb6..8aa0c056b716f 100644 --- a/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx +++ b/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx @@ -53,6 +53,8 @@ export class MLFeatureTableEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'mlFeatureTable'; + getPathName = () => 'featureTables'; getEntityName = () => 'Feature Table'; diff --git a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx index d4d0b37da9ec9..92f03aaef7a17 100644 --- a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx +++ b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx @@ -52,6 +52,8 @@ export class MLModelEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'mlModel'; + getPathName = () => 'mlModels'; getEntityName = () => 'ML Model'; diff --git a/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx b/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx index 5896c1864cc43..b5d32275f97bf 100644 --- a/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx +++ b/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx @@ -50,6 +50,8 @@ export class MLModelGroupEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'mlModelGroup'; + getPathName = () => 'mlModelGroup'; getEntityName = () => 'ML Group'; diff --git a/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx b/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx index 60c7531a4f57c..119a566b04f13 100644 --- a/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx +++ b/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx @@ -51,6 +51,8 @@ export class MLPrimaryKeyEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'mlPrimaryKey'; + getPathName = () => 'mlPrimaryKeys'; getEntityName = () => 'ML Primary Key'; diff --git a/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx b/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx index 88743012ddbc8..2c59c476195d0 100644 --- a/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx +++ b/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx @@ -44,6 +44,8 @@ export class SchemaFieldPropertiesEntity implements Entity { // Currently unused. renderProfile = (_: string) => <>; + getGraphName = () => 'schemaField'; + renderPreview = (previewType: PreviewType, data: SchemaFieldEntity) => { const parent = data.parent as Dataset; return ( diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx index 86365b8232905..4635486c24d1d 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx @@ -23,6 +23,25 @@ export function mapStructuredPropertyValues(structuredPropertiesEntry: Structure })); } +export function mapStructuredPropertyToPropertyRow(structuredPropertiesEntry: StructuredPropertiesEntry) { + const { displayName, qualifiedName } = structuredPropertiesEntry.structuredProperty.definition; + return { + displayName: displayName || qualifiedName, + qualifiedName, + values: mapStructuredPropertyValues(structuredPropertiesEntry), + dataType: structuredPropertiesEntry.structuredProperty.definition.valueType, + structuredProperty: structuredPropertiesEntry.structuredProperty, + type: + structuredPropertiesEntry.values[0] && structuredPropertiesEntry.values[0].__typename + ? { + type: typeNameToType[structuredPropertiesEntry.values[0].__typename].type, + nativeDataType: typeNameToType[structuredPropertiesEntry.values[0].__typename].nativeDataType, + } + : undefined, + associatedUrn: structuredPropertiesEntry.associatedUrn, + }; +} + // map the properties map into a list of PropertyRow objects to render in a table function getStructuredPropertyRows(entityData?: GenericEntityProperties | null) { const structuredPropertyRows: PropertyRow[] = []; diff --git a/datahub-web-react/src/app/entity/structuredProperty/StructuredPropertyEntity.tsx b/datahub-web-react/src/app/entity/structuredProperty/StructuredPropertyEntity.tsx new file mode 100644 index 0000000000000..2bdfd550652fa --- /dev/null +++ b/datahub-web-react/src/app/entity/structuredProperty/StructuredPropertyEntity.tsx @@ -0,0 +1,86 @@ +import * as React from 'react'; +import styled from 'styled-components'; +import TableIcon from '@src/images/table-icon.svg?react'; +import { TYPE_ICON_CLASS_NAME } from '@src/app/shared/constants'; +import DefaultPreviewCard from '@src/app/preview/DefaultPreviewCard'; +import { EntityType, SearchResult, StructuredPropertyEntity as StructuredProperty } from '../../../types.generated'; +import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { getDataForEntityType } from '../shared/containers/profile/utils'; +import { urlEncodeUrn } from '../shared/utils'; + +const PreviewPropIcon = styled(TableIcon)` + font-size: 20px; +`; + +/** + * Definition of the DataHub Structured Property entity. + */ +export class StructuredPropertyEntity implements Entity { + type: EntityType = EntityType.StructuredProperty; + + icon = (fontSize?: number, styleType?: IconStyleType, color?: string) => { + if (styleType === IconStyleType.TAB_VIEW) { + return ; + } + + if (styleType === IconStyleType.HIGHLIGHT) { + return ; + } + + return ( + + ); + }; + + isSearchEnabled = () => false; + + isBrowseEnabled = () => false; + + isLineageEnabled = () => false; + + getAutoCompleteFieldName = () => 'name'; + + getGraphName = () => 'structuredProperty'; + + getPathName: () => string = () => this.getGraphName(); + + getCollectionName: () => string = () => 'Structured Properties'; + + getEntityName: () => string = () => 'Structured Property'; + + renderProfile: (urn: string) => JSX.Element = (_urn) =>
; // not used right now + + renderPreview = (previewType: PreviewType, data: StructuredProperty) => ( + } + typeIcon={this.icon(14, IconStyleType.ACCENT)} + previewType={previewType} + /> + ); + + renderSearch = (result: SearchResult) => { + return this.renderPreview(PreviewType.SEARCH, result.entity as StructuredProperty); + }; + + displayName = (data: StructuredProperty) => { + return data.definition?.displayName || data.definition?.qualifiedName || data.urn; + }; + + getGenericEntityProperties = (entity: StructuredProperty) => { + return getDataForEntityType({ data: entity, entityType: this.type, getOverrideProperties: (data) => data }); + }; + + supportedCapabilities = () => { + return new Set([]); + }; +} diff --git a/datahub-web-react/src/app/entity/tag/Tag.tsx b/datahub-web-react/src/app/entity/tag/Tag.tsx index 6f0839e5f812b..d3c5b07966099 100644 --- a/datahub-web-react/src/app/entity/tag/Tag.tsx +++ b/datahub-web-react/src/app/entity/tag/Tag.tsx @@ -46,6 +46,8 @@ export class TagEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'tag'; + getPathName: () => string = () => 'tag'; getCollectionName: () => string = () => 'Tags'; diff --git a/datahub-web-react/src/app/entity/user/User.tsx b/datahub-web-react/src/app/entity/user/User.tsx index ec1c5fbdc8698..058349f83eaec 100644 --- a/datahub-web-react/src/app/entity/user/User.tsx +++ b/datahub-web-react/src/app/entity/user/User.tsx @@ -39,6 +39,8 @@ export class UserEntity implements Entity { getAutoCompleteFieldName = () => 'username'; + getGraphName: () => string = () => 'corpuser'; + getPathName: () => string = () => 'user'; getEntityName = () => 'Person'; diff --git a/datahub-web-react/src/app/govern/structuredProperties/AdvancedOptions.tsx b/datahub-web-react/src/app/govern/structuredProperties/AdvancedOptions.tsx new file mode 100644 index 0000000000000..620143258ef5f --- /dev/null +++ b/datahub-web-react/src/app/govern/structuredProperties/AdvancedOptions.tsx @@ -0,0 +1,64 @@ +import { Icon, Input, Text, Tooltip } from '@components'; +import { Collapse, Form } from 'antd'; +import React from 'react'; +import { CollapseHeader, FlexContainer, InputLabel, StyledCollapse } from './styledComponents'; + +interface Props { + isEditMode: boolean; +} + +const AdvancedOptions = ({ isEditMode }: Props) => { + return ( + ( + + )} + expandIconPosition="end" + defaultActiveKey={[]} + > + + + Advanced Options + + + } + forceRender + > + + + Qualified Name + + + + + + + + + + + + + ); +}; + +export default AdvancedOptions; diff --git a/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesDrawer.tsx b/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesDrawer.tsx new file mode 100644 index 0000000000000..f1dccb6db0c22 --- /dev/null +++ b/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesDrawer.tsx @@ -0,0 +1,142 @@ +import { Button, Icon, Input, Text, TextArea } from '@src/alchemy-components'; +import { AllowedValue } from '@src/types.generated'; +import { Form, FormInstance } from 'antd'; +import { Tooltip } from '@components'; +import React, { useEffect, useRef } from 'react'; +import { + AddButtonContainer, + DeleteIconContainer, + FieldGroupContainer, + FormContainer, + InputLabel, + StyledDivider, + ValuesContainer, +} from './styledComponents'; +import { PropValueField } from './utils'; + +interface Props { + showAllowedValuesDrawer: boolean; + propType: PropValueField; + allowedValues: AllowedValue[] | undefined; + isEditMode: boolean; + noOfExistingValues: number; + form: FormInstance; +} + +const AllowedValuesDrawer = ({ + showAllowedValuesDrawer, + propType, + allowedValues, + isEditMode, + noOfExistingValues, + form, +}: Props) => { + useEffect(() => { + form.setFieldsValue({ allowedValues: allowedValues || [{}] }); + }, [form, showAllowedValuesDrawer, allowedValues]); + + const containerRef = useRef(null); + + // Scroll to the bottom to show the newly added fields + const scrollToBottom = () => { + if (containerRef.current) { + containerRef.current.scrollTop = containerRef.current.scrollHeight; + } + }; + + return ( +
+ + {(fields, { add, remove }) => ( + + {fields.length > 0 && ( + + {fields.map((field, index) => { + const isExisting = isEditMode && index < noOfExistingValues; + + return ( + + + Value + + * + + + + + + + + +