Dedicated folder for looker explores views (#944)

* Add a folder for views and explores * Add a folder for views and explores * Bump version * Make it as a config option * Update metaphor/looker/README.md Co-authored-by: Mars Lan <[email protected]> * Update metaphor/looker/config.py Co-authored-by: Mars Lan <[email protected]> * Update metaphor/looker/README.md Co-authored-by: Mars Lan <[email protected]> * Address comments * Address comments --------- Co-authored-by: Mars Lan <[email protected]>
MetaphorData · Aug 8, 2024 · 3a8b233 · 3a8b233
1 parent afe1642
commit 3a8b233
Show file tree

Hide file tree

Showing 6 changed files with 87 additions and 55 deletions.
diff --git a/metaphor/looker/README.md b/metaphor/looker/README.md
@@ -109,6 +109,14 @@ verify_ssl: false
 timeout: 30  # default 120 seconds
 ```
 
+#### Looker Explore/View folder name
+
+Looker explores and Looker views will be put in a folder named "Looker Models" by default. You can customize the name by using this config:
+
+```yaml
+explore_view_folder_name: "Looker Views & Explores"  # default to "LookML Models"
+```
+
 #### Output Destination
 
 See [Output Config](../common/docs/output.md) for more information on the optional `output` config.

diff --git a/metaphor/looker/config.py b/metaphor/looker/config.py
@@ -53,6 +53,9 @@ class LookerRunConfig(BaseConfig):
     # Alternative base url to build the entity source URL
     alternative_base_url: Optional[str] = None
 
+    # LookML explores & views folder name
+    explore_view_folder_name: str = "LookML Models"
+
     @model_validator(mode="after")
     def have_local_or_git_dir_for_lookml(self):
         must_set_exactly_one(self.__dict__, ["lookml_dir", "lookml_git_repo"])

diff --git a/metaphor/looker/extractor.py b/metaphor/looker/extractor.py
@@ -80,6 +80,7 @@ def __init__(self, config: LookerRunConfig) -> None:
         self._lookml_git_repo = config.lookml_git_repo
         self._project_source_url = config.project_source_url
         self._include_personal_folders = config.include_personal_folders
+        self._explore_view_folder_name = config.explore_view_folder_name
 
         # Load config using environment variables instead from looker.ini file
         # See https://github.com/looker-open-source/sdk-codegen#environment-variable-configuration
@@ -106,7 +107,10 @@ async def extract(self) -> Collection[ENTITY_TYPES]:
         logger.info(f"Parsing LookML project at {lookml_dir}")
 
         model_map, virtual_views = parse_project(
-            lookml_dir, connections, self._project_source_url
+            lookml_dir,
+            connections,
+            self._explore_view_folder_name,
+            self._project_source_url,
         )
 
         folder_map = self._fetch_folders()

diff --git a/metaphor/looker/lookml_parser.py b/metaphor/looker/lookml_parser.py
@@ -198,9 +198,11 @@ def fullname(model: str, name: str) -> str:
     return f"{model}.{name}"
 
 
-def _get_model_asset_structure(model: str, name: str) -> AssetStructure:
+def _get_model_asset_structure(
+    model: str, name: str, explore_view_folder_name: str
+) -> AssetStructure:
     return AssetStructure(
-        directories=[model],
+        directories=[explore_view_folder_name, model],
         name=name,
     )
 
@@ -210,6 +212,7 @@ def _build_looker_view(
     raw_view: Dict,
     raw_model: RawModel,
     connection: LookerConnectionConfig,
+    explore_view_folder_name: str,
     url: Optional[str],
 ) -> VirtualView:
     name = raw_view["name"]
@@ -258,7 +261,7 @@ def _build_looker_view(
             name=fullname(model, name), type=VirtualViewType.LOOKER_VIEW
         ),
         looker_view=view,
-        structure=_get_model_asset_structure(model, name),
+        structure=_get_model_asset_structure(model, name, explore_view_folder_name),
         entity_upstream=(
             EntityUpstream(source_entities=source_entities) if source_entities else None
         ),
@@ -279,7 +282,11 @@ def _get_base_view_name(raw_explore: Dict, raw_model: RawModel) -> str:
 
 
 def _build_looker_explore(
-    model: str, raw_explore: Dict, raw_model: RawModel, url: Optional[str]
+    model: str,
+    raw_explore: Dict,
+    raw_model: RawModel,
+    explore_view_folder_name: str,
+    url: Optional[str],
 ) -> VirtualView:
     name = raw_explore["name"]
     base_view_name = _get_base_view_name(raw_explore, raw_model)
@@ -342,7 +349,7 @@ def _build_looker_explore(
             name=fullname(model, name), type=VirtualViewType.LOOKER_EXPLORE
         ),
         looker_explore=explore,
-        structure=_get_model_asset_structure(model, name),
+        structure=_get_model_asset_structure(model, name, explore_view_folder_name),
         entity_upstream=EntityUpstream(source_entities=source_entities),
         system_tags=SystemTags(tags=tags),
     )
@@ -604,6 +611,7 @@ def _load_model(
 def parse_project(
     base_dir: str,
     connections: Dict[str, LookerConnectionConfig],
+    explore_view_folder_name,
     projectSourceUrl: Optional[str] = None,
 ) -> Tuple[ModelMap, List[VirtualView]]:
     """
@@ -629,6 +637,7 @@ def parse_project(
                     view,
                     resolved_model,
                     connection,
+                    explore_view_folder_name,
                     entity_urls.get(view["name"]),
                 )
                 for view in resolved_model.raw_views.values()
@@ -644,6 +653,7 @@ def parse_project(
                     model_name,
                     explore,
                     resolved_model,
+                    explore_view_folder_name,
                     entity_urls.get(explore["name"]),
                 )
                 for explore in resolved_model.raw_explores.values()

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.72"
+version = "0.14.73"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]