fix: pr comments

datahub-project · Dec 27, 2024 · ce1e8c6 · ce1e8c6
1 parent b83b456
commit ce1e8c6
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 13 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py
@@ -123,6 +123,10 @@ class HiveMetastore(BasicSQLAlchemyConfig):
         description="Dataset Subtype name to be 'Table' or 'View' Valid options: ['True', 'False']",
     )
 
+    include_view_lineage: bool = Field(
+        default=False, description="", hidden_from_docs=True
+    )
+
     include_catalog_name_in_ids: bool = Field(
         default=False,
         description="Add the Presto catalog name (e.g. hive) to the generated dataset urns. `urn:li:dataset:(urn:li:dataPlatform:hive,hive.user.logging_events,PROD)` versus `urn:li:dataset:(urn:li:dataPlatform:hive,user.logging_events,PROD)`",
@@ -160,6 +164,9 @@ def get_sql_alchemy_url(
 @capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
 @capability(SourceCapability.DATA_PROFILING, "Not Supported", False)
 @capability(SourceCapability.CLASSIFICATION, "Not Supported", False)
+@capability(
+    SourceCapability.LINEAGE_COARSE, "View lineage is not supported", supported=False
+)
 class HiveMetastoreSource(SQLAlchemySource):
     """
     This plugin extracts the following:
@@ -822,15 +829,6 @@ def loop_views(
                 aspect=view_properties_aspect,
             ).as_workunit()
 
-            if (
-                dataset.view_definition
-                and self.config.mode in [HiveMetastoreConfigMode.hive]
-                and self.config.include_view_lineage
-            ):
-                self.aggregator.add_view_definition(
-                    view_urn=dataset_urn,
-                    view_definition=dataset.view_definition,
-                )
             if self.config.domain:
                 assert self.domain_registry
                 yield from get_domain_wu(

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
@@ -567,7 +567,6 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit
 
         # Generate workunit for aggregated SQL parsing results
         for mcp in self.aggregator.gen_metadata():
-            self.report.report_workunit(mcp.as_workunit())
             yield mcp.as_workunit()
 
     def get_identifier(
@@ -1147,9 +1146,7 @@ def _save_schema_to_resolver(self):
 
     @property
     def include_lineage(self):
-        return self.config.include_view_lineage or (
-            hasattr(self.config, "include_lineage") and self.config.include_lineage
-        )
+        return self.config.include_view_lineage
 
     def get_db_schema(self, dataset_identifier: str) -> Tuple[Optional[str], str]:
         database, schema, _view = dataset_identifier.split(".", 2)