diff --git a/metaphor/looker/README.md b/metaphor/looker/README.md index 2a6d9c10..fb179af6 100644 --- a/metaphor/looker/README.md +++ b/metaphor/looker/README.md @@ -84,12 +84,24 @@ For more information about git repo configuration, please refer to [Git Repo Con ### Optional Configurations +#### Project Source URL + To add a "View LookML" link to Looker explore & views on Metaphor you need to specify a base URL for the Looker project. This can be either a URL to the GitHub repository (`https://github.com//`) or Looker IDE (`https://.cloud.looker.com/projects//files/`). ```yaml project_source_url: ``` +#### Alternative Server URL + +If the looker users use a different URL to view content on Looker than the server URL for fetching metadata, please provide the alternative base URL so the crawler can generate direct links to the assets: + +```yaml +alternative_base_url: // e.g. https://looker.my_company.com +``` + +#### SSL Verification + You can also disable SSL verify and change the request timeout if needed, e.g. ```yaml diff --git a/metaphor/looker/config.py b/metaphor/looker/config.py index 89c0a7fc..7d1d6085 100644 --- a/metaphor/looker/config.py +++ b/metaphor/looker/config.py @@ -50,6 +50,9 @@ class LookerRunConfig(BaseConfig): # Whether to include dashboards in personal folders include_personal_folders: bool = False + # Alternative base url to build the entity source URL + alternative_base_url: Optional[str] = None + @model_validator(mode="after") def have_local_or_git_dir_for_lookml(self): must_set_exactly_one(self.__dict__, ["lookml_dir", "lookml_git_repo"]) diff --git a/metaphor/looker/extractor.py b/metaphor/looker/extractor.py index 54f855ac..21d85bc2 100644 --- a/metaphor/looker/extractor.py +++ b/metaphor/looker/extractor.py @@ -70,6 +70,7 @@ def from_config_file(config_file: str) -> "LookerExtractor": def __init__(self, config: LookerRunConfig) -> None: super().__init__(config) self._base_url = config.base_url + self._alternative_base_url = config.alternative_base_url self._connections = config.connections self._lookml_dir = config.lookml_dir self._lookml_git_repo = config.lookml_git_repo @@ -166,7 +167,7 @@ def _fetch_dashboards( ) source_info = SourceInfo( - main_url=f"{self._base_url}/{dashboard.preferred_viewer}/{dashboard.id}", + main_url=f"{self._alternative_base_url or self._base_url}/{dashboard.preferred_viewer}/{dashboard.id}", ) # All numeric fields must be converted to "float" to meet quicktype's expectation diff --git a/pyproject.toml b/pyproject.toml index 8571d6d3..e289d918 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "metaphor-connectors" -version = "0.14.28" +version = "0.14.29" license = "Apache-2.0" description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app." authors = ["Metaphor "] diff --git a/tests/looker/expected_alternative.json b/tests/looker/expected_alternative.json new file mode 100644 index 00000000..c83653d5 --- /dev/null +++ b/tests/looker/expected_alternative.json @@ -0,0 +1,33 @@ +[ + { + "dashboardInfo": { + "charts": [ + { + "chartType": "MAP" + } + ], + "description": "first dashboard", + "title": "first", + "viewCount": 123.0 + }, + "entityUpstream": { + "sourceEntities": [ + "VIRTUAL_VIEW~BCFF7FBF29617570C10466C0CA541AE4" + ] + }, + "logicalId": { + "dashboardId": "1", + "platform": "LOOKER" + }, + "sourceInfo": { + "mainUrl": "http://dev.test/me/1" + }, + "structure": { + "directories": [ + "1", + "2" + ], + "name": "first" + } + } +] diff --git a/tests/looker/test_extractor.py b/tests/looker/test_extractor.py index 7978040a..f3238709 100644 --- a/tests/looker/test_extractor.py +++ b/tests/looker/test_extractor.py @@ -33,63 +33,69 @@ def test_fetch_dashboards(test_root_dir) -> None: ) }, ) - extractor = LookerExtractor(config) - extractor._sdk = MagicMock() - extractor._sdk.all_dashboards = MagicMock() - extractor._sdk.all_dashboards.return_value = [ - DashboardBase(id="1"), - DashboardBase(id="2"), - DashboardBase(id="3"), - ] - mock_dashboards = [ - # In shared folder - Dashboard( - id="1", - title="first", - description="first dashboard", - preferred_viewer="me", - view_count=123, - folder=FolderBase(id="2", name="folder1"), - dashboard_elements=[ - DashboardElement( - type="vis", - result_maker=ResultMakerWithIdVisConfigAndDynamicFields( - vis_config={ - "type": "looker_map", - }, - filterables=[ - ResultMakerFilterables( - model="model1", - view="view1", - ) - ], - ), - ) - ], - ), - # In personal folder - Dashboard( - id="2", - title="second", - folder=FolderBase( - id="3", - name="personal", - is_personal=True, + def create_extractor(config: LookerRunConfig): + extractor = LookerExtractor(config) + extractor._sdk = MagicMock() + extractor._sdk.all_dashboards = MagicMock() + extractor._sdk.all_dashboards.return_value = [ + DashboardBase(id="1"), + DashboardBase(id="2"), + DashboardBase(id="3"), + ] + + mock_dashboards = [ + # In shared folder + Dashboard( + id="1", + title="first", + description="first dashboard", + preferred_viewer="me", + view_count=123, + folder=FolderBase(id="2", name="folder1"), + dashboard_elements=[ + DashboardElement( + type="vis", + result_maker=ResultMakerWithIdVisConfigAndDynamicFields( + vis_config={ + "type": "looker_map", + }, + filterables=[ + ResultMakerFilterables( + model="model1", + view="view1", + ) + ], + ), + ) + ], ), - ), - # In personal descendant folder - Dashboard( - id="3", - title="third", - folder=FolderBase( - id="4", - name="personal descendant", - is_personal=False, - is_personal_descendant=True, + # In personal folder + Dashboard( + id="2", + title="second", + folder=FolderBase( + id="3", + name="personal", + is_personal=True, + ), + ), + # In personal descendant folder + Dashboard( + id="3", + title="third", + folder=FolderBase( + id="4", + name="personal descendant", + is_personal=False, + is_personal_descendant=True, + ), ), - ), - ] + ] + + extractor._sdk.dashboard.side_effect = mock_dashboards + + return extractor models = { "model1": Model( @@ -108,8 +114,12 @@ def test_fetch_dashboards(test_root_dir) -> None: "4": FolderMetadata(id="4", name="personal descendant", parent_id="3"), } - extractor._sdk.dashboard.side_effect = mock_dashboards + dashboards = create_extractor(config)._fetch_dashboards(models, folders) - dashboards = extractor._fetch_dashboards(models, folders) events = [EventUtil.trim_event(e) for e in dashboards] assert events == load_json(f"{test_root_dir}/looker/expected.json") + + config.alternative_base_url = "http://dev.test" + dashboards = create_extractor(config)._fetch_dashboards(models, folders) + events = [EventUtil.trim_event(e) for e in dashboards] + assert events == load_json(f"{test_root_dir}/looker/expected_alternative.json")