Extract owner and description for looker dashboards (#943)

* Extract system contact & description

* Bump version

* Update model version

* Bump version

metaphor/looker/extractor.py

@@ -19,6 +19,7 @@
 from metaphor.looker.folder import FolderMap, FolderMetadata, build_directories
 from metaphor.looker.lookml_parser import ModelMap, fullname, parse_project
 from metaphor.models.metadata_change_event import (
+    AssetPlatform,
     AssetStructure,
     Chart,
     ChartType,
@@ -32,6 +33,9 @@
     HierarchyLogicalID,
     HierarchyType,
     SourceInfo,
+    SystemContact,
+    SystemContacts,
+    SystemDescription,
     VirtualViewType,
 )
 
@@ -106,8 +110,9 @@ async def extract(self) -> Collection[ENTITY_TYPES]:
         )
 
         folder_map = self._fetch_folders()
+        user_map = self._fetch_users()
 
-        dashboards = self._fetch_dashboards(model_map, folder_map)
+        dashboards = self._fetch_dashboards(model_map, folder_map, user_map)
 
         entities: List[ENTITY_TYPES] = []
         entities.extend(dashboards)
@@ -129,8 +134,20 @@ def _fetch_folders(self) -> FolderMap:
 
         return folder_map
 
+    def _fetch_users(self) -> Dict[str, str]:
+        user_map: Dict[str, str] = {}
+        users = self._sdk.all_users()
+        json_dump_to_debug_file(users, "all_users.json")
+
+        for user in users:
+            if user.email is None or user.id is None:
+                continue
+            user_map[user.id] = user.email
+
+        return user_map
+
     def _fetch_dashboards(
-        self, model_map: ModelMap, folder_map: FolderMap
+        self, model_map: ModelMap, folder_map: FolderMap, user_map: Dict[str, str]
     ) -> List[Dashboard]:
         dashboards: List[Dashboard] = []
 
@@ -142,6 +159,9 @@ def _fetch_dashboards(
 
             try:
                 dashboard = self._sdk.dashboard(dashboard_id=basic_dashboard.id)
+                json_dump_to_debug_file(
+                    dashboard, f"{basic_dashboard.id}_dashboard.json"
+                )
             except Exception as error:
                 logger.error(f"Failed to fetch dashboard {basic_dashboard.id}: {error}")
                 continue
@@ -188,6 +208,20 @@ def _fetch_dashboards(
 
             assert dashboard.id is not None
 
+            owner_email = user_map.get(dashboard.user_id or "")
+            system_contacts = (
+                SystemContacts(
+                    contacts=[
+                        SystemContact(
+                            email=owner_email,
+                            system_contact_source=AssetPlatform.LOOKER,
+                        )
+                    ]
+                )
+                if owner_email
+                else None
+            )
+
             dashboards.append(
                 Dashboard(
                     logical_id=DashboardLogicalID(
@@ -206,6 +240,15 @@ def _fetch_dashboards(
                         if dashboard.folder and dashboard.folder.id
                         else None
                     ),
+                    system_contacts=system_contacts,
+                    system_description=(
+                        SystemDescription(
+                            description=dashboard.description,
+                            platform=AssetPlatform.LOOKER,
+                        )
+                        if dashboard.description
+                        else None
+                    ),
                 )
             )
 

metaphor/tableau/extractor.py

@@ -23,6 +23,7 @@
 from metaphor.common.logger import get_logger, json_dump_to_debug_file
 from metaphor.models.crawler_run_metadata import Platform
 from metaphor.models.metadata_change_event import (
+    AssetPlatform,
     AssetStructure,
     Chart,
     Dashboard,
@@ -36,7 +37,6 @@
     SourceInfo,
     SystemContact,
     SystemContacts,
-    SystemContactSource,
     SystemTag,
     SystemTags,
     SystemTagSource,
@@ -229,7 +229,7 @@ def _get_system_contacts(
 
         system_contact = SystemContact(
             email=self._users[user_id].email,
-            system_contact_source=SystemContactSource.TABLEAU,
+            system_contact_source=AssetPlatform.TABLEAU,
         )
         return SystemContacts(contacts=[system_contact])
 

metaphor/unity_catalog/extractor.py

@@ -21,6 +21,7 @@
 from metaphor.common.utils import to_utc_datetime_from_timestamp
 from metaphor.models.crawler_run_metadata import Platform
 from metaphor.models.metadata_change_event import (
+    AssetPlatform,
     DataPlatform,
     Dataset,
     DatasetLogicalID,
@@ -39,7 +40,6 @@
     SQLSchema,
     SystemContact,
     SystemContacts,
-    SystemContactSource,
     SystemTag,
     SystemTags,
     SystemTagSource,
@@ -366,7 +366,7 @@ def _init_dataset(self, table_info: TableInfo) -> Dataset:
                 contacts=[
                     SystemContact(
                         email=owner,
-                        system_contact_source=SystemContactSource.UNITY_CATALOG,
+                        system_contact_source=AssetPlatform.UNITY_CATALOG,
                     )
                 ]
             )
@@ -751,7 +751,7 @@ def _init_volume(self, volume: VolumeInfo):
                 contacts=[
                     SystemContact(
                         email=volume.owner,
-                        system_contact_source=SystemContactSource.UNITY_CATALOG,
+                        system_contact_source=AssetPlatform.UNITY_CATALOG,
                     )
                 ]
             )

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.71"
+version = "0.14.72"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]
@@ -42,7 +42,7 @@ llama-index-readers-confluence = { version = "^0.1.4", optional = true }
 llama-index-readers-notion = { version = "^0.1.6", optional = true }
 looker-sdk = { version = "^24.2.0", optional = true }
 lxml = { version = "~=5.0.0", optional = true }
-metaphor-models = "0.37.2"
+metaphor-models = "0.38.0"
 more-itertools = { version = "^10.1.0", optional = true }
 msal = { version = "^1.28.0", optional = true }
 msgraph-beta-sdk = { version = "~1.4.0", optional = true }

tests/looker/expected.json

@@ -28,6 +28,10 @@
         "2"
       ],
       "name": "first"
+    },
+    "systemDescription": {
+      "description": "first dashboard",
+      "platform": "LOOKER"
     }
   },
   {
@@ -59,6 +63,18 @@
         "2"
       ],
       "name": "old version dashboard"
+    },
+    "systemContacts": {
+      "contacts": [
+        {
+          "email": "[email protected]",
+          "systemContactSource": "LOOKER"
+        }
+      ]
+    },
+    "systemDescription": {
+      "description": "foo",
+      "platform": "LOOKER"
     }
   }
 ]

tests/looker/expected_alternative.json

@@ -28,6 +28,10 @@
         "2"
       ],
       "name": "first"
+    },
+    "systemDescription": {
+      "description": "first dashboard",
+      "platform": "LOOKER"
     }
   },
   {
@@ -59,6 +63,18 @@
         "2"
       ],
       "name": "old version dashboard"
+    },
+    "systemContacts": {
+      "contacts": [
+        {
+          "email": "[email protected]",
+          "systemContactSource": "LOOKER"
+        }
+      ]
+    },
+    "systemDescription": {
+      "description": "foo",
+      "platform": "LOOKER"
     }
   }
 ]

tests/looker/test_extractor.py

@@ -94,6 +94,7 @@ def create_extractor(config: LookerRunConfig):
             ),
             Dashboard(
                 id="4",
+                user_id="1",
                 title="old version dashboard",
                 description="foo",
                 preferred_viewer=None,
@@ -139,12 +140,16 @@ def create_extractor(config: LookerRunConfig):
         "4": FolderMetadata(id="4", name="personal descendant", parent_id="3"),
     }
 
-    dashboards = create_extractor(config)._fetch_dashboards(models, folders)
+    users = {
+        "1": "[email protected]",
+    }
+
+    dashboards = create_extractor(config)._fetch_dashboards(models, folders, users)
 
     events = [EventUtil.trim_event(e) for e in dashboards]
     assert events == load_json(f"{test_root_dir}/looker/expected.json")
 
     config.alternative_base_url = "http://dev.test"
-    dashboards = create_extractor(config)._fetch_dashboards(models, folders)
+    dashboards = create_extractor(config)._fetch_dashboards(models, folders, users)
     events = [EventUtil.trim_event(e) for e in dashboards]
     assert events == load_json(f"{test_root_dir}/looker/expected_alternative.json")