User should specify alternative looker source url (#894)

* Add alternative source URL config

* update readme

* Bump version

metaphor/looker/README.md

@@ -84,12 +84,24 @@ For more information about git repo configuration, please refer to [Git Repo Con
 
 ### Optional Configurations
 
+#### Project Source URL
+
 To add a "View LookML" link to Looker explore & views on Metaphor you need to specify a base URL for the Looker project. This can be either a URL to the GitHub repository (`https://github.com/<account>/<repo>`) or Looker IDE (`https://<account>.cloud.looker.com/projects/<project>/files/`).
 
 ```yaml
 project_source_url: <looker_project_source_url>
 ```
 
+#### Alternative Server URL
+
+If the looker users use a different URL to view content on Looker than the server URL for fetching metadata, please provide the alternative base URL so the crawler can generate direct links to the assets:
+
+```yaml
+alternative_base_url: <looker_base_url> // e.g. https://looker.my_company.com
+```
+
+#### SSL Verification
+
 You can also disable SSL verify and change the request timeout if needed, e.g.
 
 ```yaml

metaphor/looker/config.py

@@ -50,6 +50,9 @@ class LookerRunConfig(BaseConfig):
     # Whether to include dashboards in personal folders
     include_personal_folders: bool = False
 
+    # Alternative base url to build the entity source URL
+    alternative_base_url: Optional[str] = None
+
     @model_validator(mode="after")
     def have_local_or_git_dir_for_lookml(self):
         must_set_exactly_one(self.__dict__, ["lookml_dir", "lookml_git_repo"])

metaphor/looker/extractor.py

@@ -70,6 +70,7 @@ def from_config_file(config_file: str) -> "LookerExtractor":
     def __init__(self, config: LookerRunConfig) -> None:
         super().__init__(config)
         self._base_url = config.base_url
+        self._alternative_base_url = config.alternative_base_url
         self._connections = config.connections
         self._lookml_dir = config.lookml_dir
         self._lookml_git_repo = config.lookml_git_repo
@@ -166,7 +167,7 @@ def _fetch_dashboards(
             )
 
             source_info = SourceInfo(
-                main_url=f"{self._base_url}/{dashboard.preferred_viewer}/{dashboard.id}",
+                main_url=f"{self._alternative_base_url or self._base_url}/{dashboard.preferred_viewer}/{dashboard.id}",
             )
 
             # All numeric fields must be converted to "float" to meet quicktype's expectation

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.28"
+version = "0.14.29"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]

tests/looker/expected_alternative.json

@@ -0,0 +1,33 @@
+[
+  {
+    "dashboardInfo": {
+      "charts": [
+        {
+          "chartType": "MAP"
+        }
+      ],
+      "description": "first dashboard",
+      "title": "first",
+      "viewCount": 123.0
+    },
+    "entityUpstream": {
+      "sourceEntities": [
+        "VIRTUAL_VIEW~BCFF7FBF29617570C10466C0CA541AE4"
+      ]
+    },
+    "logicalId": {
+      "dashboardId": "1",
+      "platform": "LOOKER"
+    },
+    "sourceInfo": {
+      "mainUrl": "http://dev.test/me/1"
+    },
+    "structure": {
+      "directories": [
+        "1",
+        "2"
+      ],
+      "name": "first"
+    }
+  }
+]

tests/looker/test_extractor.py

@@ -33,63 +33,69 @@ def test_fetch_dashboards(test_root_dir) -> None:
             )
         },
     )
-    extractor = LookerExtractor(config)
-    extractor._sdk = MagicMock()
-    extractor._sdk.all_dashboards = MagicMock()
-    extractor._sdk.all_dashboards.return_value = [
-        DashboardBase(id="1"),
-        DashboardBase(id="2"),
-        DashboardBase(id="3"),
-    ]
 
-    mock_dashboards = [
-        # In shared folder
-        Dashboard(
-            id="1",
-            title="first",
-            description="first dashboard",
-            preferred_viewer="me",
-            view_count=123,
-            folder=FolderBase(id="2", name="folder1"),
-            dashboard_elements=[
-                DashboardElement(
-                    type="vis",
-                    result_maker=ResultMakerWithIdVisConfigAndDynamicFields(
-                        vis_config={
-                            "type": "looker_map",
-                        },
-                        filterables=[
-                            ResultMakerFilterables(
-                                model="model1",
-                                view="view1",
-                            )
-                        ],
-                    ),
-                )
-            ],
-        ),
-        # In personal folder
-        Dashboard(
-            id="2",
-            title="second",
-            folder=FolderBase(
-                id="3",
-                name="personal",
-                is_personal=True,
+    def create_extractor(config: LookerRunConfig):
+        extractor = LookerExtractor(config)
+        extractor._sdk = MagicMock()
+        extractor._sdk.all_dashboards = MagicMock()
+        extractor._sdk.all_dashboards.return_value = [
+            DashboardBase(id="1"),
+            DashboardBase(id="2"),
+            DashboardBase(id="3"),
+        ]
+
+        mock_dashboards = [
+            # In shared folder
+            Dashboard(
+                id="1",
+                title="first",
+                description="first dashboard",
+                preferred_viewer="me",
+                view_count=123,
+                folder=FolderBase(id="2", name="folder1"),
+                dashboard_elements=[
+                    DashboardElement(
+                        type="vis",
+                        result_maker=ResultMakerWithIdVisConfigAndDynamicFields(
+                            vis_config={
+                                "type": "looker_map",
+                            },
+                            filterables=[
+                                ResultMakerFilterables(
+                                    model="model1",
+                                    view="view1",
+                                )
+                            ],
+                        ),
+                    )
+                ],
             ),
-        ),
-        # In personal descendant folder
-        Dashboard(
-            id="3",
-            title="third",
-            folder=FolderBase(
-                id="4",
-                name="personal descendant",
-                is_personal=False,
-                is_personal_descendant=True,
+            # In personal folder
+            Dashboard(
+                id="2",
+                title="second",
+                folder=FolderBase(
+                    id="3",
+                    name="personal",
+                    is_personal=True,
+                ),
+            ),
+            # In personal descendant folder
+            Dashboard(
+                id="3",
+                title="third",
+                folder=FolderBase(
+                    id="4",
+                    name="personal descendant",
+                    is_personal=False,
+                    is_personal_descendant=True,
+                ),
             ),
-        ),
-    ]
+        ]
+
+        extractor._sdk.dashboard.side_effect = mock_dashboards
+
+        return extractor
 
     models = {
         "model1": Model(
@@ -108,8 +114,12 @@ def test_fetch_dashboards(test_root_dir) -> None:
         "4": FolderMetadata(id="4", name="personal descendant", parent_id="3"),
     }
 
-    extractor._sdk.dashboard.side_effect = mock_dashboards
+    dashboards = create_extractor(config)._fetch_dashboards(models, folders)
 
-    dashboards = extractor._fetch_dashboards(models, folders)
     events = [EventUtil.trim_event(e) for e in dashboards]
     assert events == load_json(f"{test_root_dir}/looker/expected.json")
+
+    config.alternative_base_url = "http://dev.test"
+    dashboards = create_extractor(config)._fetch_dashboards(models, folders)
+    events = [EventUtil.trim_event(e) for e in dashboards]
+    assert events == load_json(f"{test_root_dir}/looker/expected_alternative.json")