[sc-26028] Set default value for entity system tags (#848)

* [sc-26028] Set default value for entity system tags

* bump version

metaphor/dbt/artifact_parser.py

@@ -647,7 +647,7 @@ def _get_system_tags(
             for name in tag_names
             if name
         ]
-        return SystemTags(tags=tags) if tags else None
+        return SystemTags(tags=tags)
 
     def _parse_virtual_view_node(
         self,

metaphor/looker/lookml_parser.py

@@ -341,7 +341,7 @@ def _build_looker_explore(
         looker_explore=explore,
         structure=_get_model_asset_structure(model, name),
         entity_upstream=EntityUpstream(source_entities=source_entities),
-        system_tags=SystemTags(tags=tags) if tags else None,
+        system_tags=SystemTags(tags=tags),
     )
 
 

metaphor/snowflake/extractor.py

@@ -165,6 +165,11 @@ async def extract(self) -> Collection[ENTITY_TYPES]:
         datasets = list(self._datasets.values())
         tag_datasets(datasets, self._tag_matchers)
 
+        # Dedup system tags
+        for dataset in datasets:
+            assert dataset.system_tags and dataset.system_tags.tags is not None
+            dataset.system_tags.tags = list(set(dataset.system_tags.tags))
+
         entities: List[ENTITY_TYPES] = []
         entities.extend(datasets)
         entities.extend(self._hierarchies.values())
@@ -511,12 +516,12 @@ def _append_tag(
             return
 
         if not column_name:
-            if not dataset.system_tags:
-                dataset.system_tags = SystemTags(tags=[])
-            assert dataset.system_tags.tags is not None
+            assert dataset.system_tags and dataset.system_tags.tags is not None
             dataset.system_tags.tags.append(
                 SystemTag(
-                    key=key, system_tag_source=SystemTagSource.SNOWFLAKE, value=value
+                    key=key,
+                    system_tag_source=SystemTagSource.SNOWFLAKE,
+                    value=value,
                 )
             )
 
@@ -902,6 +907,8 @@ def _init_dataset(
             database=database, schema=schema, table=table
         )
 
+        dataset.system_tags = SystemTags(tags=[])
+
         return dataset
 
     @staticmethod

metaphor/tableau/extractor.py

@@ -405,14 +405,12 @@ def _parse_workbook_query_response(
         source_virtual_views: List[str] = []
         published_datasources: List[str] = []
 
-        system_tags = None
+        system_tags = SystemTags(tags=[])
         if workbook.tags:
-            system_tags = SystemTags(
-                tags=[
-                    SystemTag(value=tag, system_tag_source=SystemTagSource.TABLEAU)
-                    for tag in sorted(tag.name for tag in workbook.tags)
-                ]
-            )
+            system_tags.tags = [
+                SystemTag(value=tag, system_tag_source=SystemTagSource.TABLEAU)
+                for tag in sorted(tag.name for tag in workbook.tags)
+            ]
 
         for published_source in workbook.upstreamDatasources:
             virtual_view_id = str(

metaphor/thought_spot/extractor.py

@@ -753,4 +753,4 @@ def _get_system_tags(tags: List[Tag]) -> Optional[SystemTags]:
             if not (tag.isDeleted or tag.isHidden or tag.isDeprecated)
         ]
 
-        return SystemTags(tags=system_tags) if system_tags else None
+        return SystemTags(tags=system_tags)

metaphor/unity_catalog/extractor.py

@@ -247,6 +247,8 @@ def _init_dataset(self, table_info: TableInfo) -> Dataset:
             ),
         )
 
+        dataset.system_tags = SystemTags(tags=[])
+
         self._datasets[normalized_name] = dataset
 
         return dataset
@@ -432,13 +434,11 @@ def _assign_dataset_system_tags(
                     )
                     dataset = self._datasets.get(normalized_dataset_name)
                     if dataset is not None:
-                        tags = (
+                        assert dataset.system_tags
+                        dataset.system_tags.tags = (
                             catalog_system_tags[catalog][0]
                             + catalog_system_tags[catalog][1][schema.name]
                         )
-                        if tags and not dataset.system_tags:
-                            # We do not need to append to system tags once it's been assigned
-                            dataset.system_tags = SystemTags(tags=tags)
 
     def _extract_table_tags(self, catalog: str) -> None:
         with self._connection.cursor() as cursor:
@@ -468,8 +468,6 @@ def _extract_table_tags(self, catalog: str) -> None:
                     logger.warn(f"Cannot find {normalized_dataset_name} table")
                     continue
 
-                if not dataset.system_tags:
-                    dataset.system_tags = SystemTags(tags=[])
                 assert dataset.system_tags and dataset.system_tags.tags is not None
 
                 if tag_value:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.13.182"
+version = "0.13.183"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]

tests/looker/test_lookml_parser.py

@@ -245,6 +245,7 @@ def test_join(test_root_dir):
                 entity_upstream=EntityUpstream(
                     source_entities=[str(virtual_view_id1), str(virtual_view_id2)]
                 ),
+                system_tags=SystemTags(tags=[]),
             ),
         ],
     )
@@ -317,6 +318,7 @@ def test_explore_in_view(test_root_dir):
                     name="explore1",
                 ),
                 entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id)]),
+                system_tags=SystemTags(tags=[]),
             ),
         ],
     )
@@ -414,6 +416,7 @@ def test_derived_table(test_root_dir):
                 name="explore1",
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id1)]),
+            system_tags=SystemTags(tags=[]),
         ),
         VirtualView(
             logical_id=VirtualViewLogicalID(
@@ -428,6 +431,7 @@ def test_derived_table(test_root_dir):
                 name="explore2",
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id2)]),
+            system_tags=SystemTags(tags=[]),
         ),
         VirtualView(
             logical_id=VirtualViewLogicalID(
@@ -442,6 +446,7 @@ def test_derived_table(test_root_dir):
                 name="explore3",
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id3)]),
+            system_tags=SystemTags(tags=[]),
         ),
     ]
 
@@ -501,6 +506,7 @@ def test_sql_table_name(test_root_dir):
                 name="explore1",
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id1)]),
+            system_tags=SystemTags(tags=[]),
         ),
     ]
 
@@ -674,6 +680,7 @@ def test_complex_includes(test_root_dir):
                 name="explore1",
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id1)]),
+            system_tags=SystemTags(tags=[]),
         ),
     ]
 
@@ -824,6 +831,7 @@ def test_view_extension(test_root_dir):
                 base_view=str(virtual_view_id1),
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id1)]),
+            system_tags=SystemTags(tags=[]),
         ),
     ]
 
@@ -939,6 +947,7 @@ def test_explore_extension(test_root_dir):
                 name="explore1",
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view1)]),
+            system_tags=SystemTags(tags=[]),
         ),
         VirtualView(
             logical_id=VirtualViewLogicalID(
@@ -953,6 +962,7 @@ def test_explore_extension(test_root_dir):
                 name="explore2",
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view2)]),
+            system_tags=SystemTags(tags=[]),
         ),
         VirtualView(
             logical_id=VirtualViewLogicalID(
@@ -967,6 +977,7 @@ def test_explore_extension(test_root_dir):
                 name="explore3",
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view1)]),
+            system_tags=SystemTags(tags=[]),
         ),
         VirtualView(
             logical_id=VirtualViewLogicalID(
@@ -981,6 +992,7 @@ def test_explore_extension(test_root_dir):
                 name="explore4",
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view3)]),
+            system_tags=SystemTags(tags=[]),
         ),
         VirtualView(
             logical_id=VirtualViewLogicalID(
@@ -995,6 +1007,7 @@ def test_explore_extension(test_root_dir):
                 name="base_explore2",
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view2)]),
+            system_tags=SystemTags(tags=[]),
         ),
         VirtualView(
             logical_id=VirtualViewLogicalID(
@@ -1009,5 +1022,6 @@ def test_explore_extension(test_root_dir):
                 name="view3",
             ),
             entity_upstream=EntityUpstream(source_entities=[str(virtual_view3)]),
+            system_tags=SystemTags(tags=[]),
         ),
     ]

tests/snowflake/test_extractor.py

@@ -433,7 +433,7 @@ def test_fetch_hierarchy_system_tags(mock_connect: MagicMock):
 
     extractor._fetch_tags(mock_cursor)
 
-    assert not dataset.system_tags
+    assert dataset.system_tags and dataset.system_tags.tags is not None
     assert extractor._hierarchies.get(dataset_normalized_name(table_name)) is not None
     db_hierarchy = extractor._hierarchies[dataset_normalized_name(table_name)]
     assert db_hierarchy.logical_id == HierarchyLogicalID(

tests/thought_spot/expected.json

@@ -117,6 +117,9 @@
       ],
       "name": "View 1"
     },
+    "systemTags": {
+      "tags": []
+    },
     "thoughtSpot": {
       "columns": [
         {
@@ -168,6 +171,9 @@
       ],
       "name": "JOIN SQL view"
     },
+    "systemTags": {
+      "tags": []
+    },
     "thoughtSpot": {
       "columns": [
         {
@@ -212,6 +218,9 @@
       ],
       "name": "JOIN SQL view"
     },
+    "systemTags": {
+      "tags": []
+    },
     "thoughtSpot": {
       "columns": [
         {
@@ -353,63 +362,63 @@
     }
   },
   {
+    "hierarchyInfo": {
+      "name": "Answer",
+      "type": "THOUGHT_SPOT_VIRTUAL_HIERARCHY"
+    },
     "logicalId": {
       "path": [
         "THOUGHT_SPOT",
         "ANSWER"
       ]
-    },
-    "hierarchyInfo": {
-      "name": "Answer",
-      "type": "THOUGHT_SPOT_VIRTUAL_HIERARCHY"
     }
   },
   {
+    "hierarchyInfo": {
+      "name": "Liveboard",
+      "type": "THOUGHT_SPOT_VIRTUAL_HIERARCHY"
+    },
     "logicalId": {
       "path": [
         "THOUGHT_SPOT",
         "LIVEBOARD"
       ]
-    },
-    "hierarchyInfo": {
-      "name": "Liveboard",
-      "type": "THOUGHT_SPOT_VIRTUAL_HIERARCHY"
     }
   },
   {
+    "hierarchyInfo": {
+      "name": "Table",
+      "type": "THOUGHT_SPOT_VIRTUAL_HIERARCHY"
+    },
     "logicalId": {
       "path": [
         "THOUGHT_SPOT",
         "TABLE"
       ]
-    },
-    "hierarchyInfo": {
-      "name": "Table",
-      "type": "THOUGHT_SPOT_VIRTUAL_HIERARCHY"
     }
   },
   {
+    "hierarchyInfo": {
+      "name": "View",
+      "type": "THOUGHT_SPOT_VIRTUAL_HIERARCHY"
+    },
     "logicalId": {
       "path": [
         "THOUGHT_SPOT",
         "VIEW"
       ]
-    },
-    "hierarchyInfo": {
-      "name": "View",
-      "type": "THOUGHT_SPOT_VIRTUAL_HIERARCHY"
     }
   },
   {
+    "hierarchyInfo": {
+      "name": "Worksheet",
+      "type": "THOUGHT_SPOT_VIRTUAL_HIERARCHY"
+    },
     "logicalId": {
       "path": [
         "THOUGHT_SPOT",
         "WORKSHEET"
       ]
-    },
-    "hierarchyInfo": {
-      "name": "Worksheet",
-      "type": "THOUGHT_SPOT_VIRTUAL_HIERARCHY"
     }
   }
 ]