Make it as a config option

MetaphorData · Aug 8, 2024 · 5dd38c5 · 5dd38c5
1 parent 3bcabc0
commit 5dd38c5
Show file tree

Hide file tree

Showing 5 changed files with 49 additions and 24 deletions.
diff --git a/metaphor/looker/README.md b/metaphor/looker/README.md
@@ -109,6 +109,14 @@ verify_ssl: false
 timeout: 30  # default 120 seconds
 ```
 
+#### Looker Explore/View folder name
+
+We will put Looker explores and Looker views in a folder in Metaphor, and you can customize the folder name.
+
+```yaml
+explore_view_folder_name: "LookML models"  # default is "LookML models"
+```
+
 #### Output Destination
 
 See [Output Config](../common/docs/output.md) for more information on the optional `output` config.

diff --git a/metaphor/looker/config.py b/metaphor/looker/config.py
@@ -53,6 +53,9 @@ class LookerRunConfig(BaseConfig):
     # Alternative base url to build the entity source URL
     alternative_base_url: Optional[str] = None
 
+    # LookML explores & views folder name
+    explore_view_folder_name: str = "LookML models"
+
     @model_validator(mode="after")
     def have_local_or_git_dir_for_lookml(self):
         must_set_exactly_one(self.__dict__, ["lookml_dir", "lookml_git_repo"])

diff --git a/metaphor/looker/extractor.py b/metaphor/looker/extractor.py
@@ -80,6 +80,7 @@ def __init__(self, config: LookerRunConfig) -> None:
         self._lookml_git_repo = config.lookml_git_repo
         self._project_source_url = config.project_source_url
         self._include_personal_folders = config.include_personal_folders
+        self._explore_view_folder_name = config.explore_view_folder_name
 
         # Load config using environment variables instead from looker.ini file
         # See https://github.com/looker-open-source/sdk-codegen#environment-variable-configuration
@@ -106,7 +107,10 @@ async def extract(self) -> Collection[ENTITY_TYPES]:
         logger.info(f"Parsing LookML project at {lookml_dir}")
 
         model_map, virtual_views = parse_project(
-            lookml_dir, connections, self._project_source_url
+            lookml_dir,
+            connections,
+            self._explore_view_folder_name,
+            self._project_source_url,
         )
 
         folder_map = self._fetch_folders()

diff --git a/metaphor/looker/lookml_parser.py b/metaphor/looker/lookml_parser.py
@@ -43,8 +43,6 @@
 # lkml parser's debug log can be very noisy
 logging.getLogger("lkml.parser").setLevel(logging.WARNING)
 
-VIEW_EXPLORE_FOLDER = "LookML models"
-
 
 @dataclass
 class Explore:
@@ -200,9 +198,11 @@ def fullname(model: str, name: str) -> str:
     return f"{model}.{name}"
 
 
-def _get_model_asset_structure(model: str, name: str) -> AssetStructure:
+def _get_model_asset_structure(
+    model: str, name: str, explore_view_folder_name: str
+) -> AssetStructure:
     return AssetStructure(
-        directories=[VIEW_EXPLORE_FOLDER, model],
+        directories=[explore_view_folder_name, model],
         name=name,
     )
 
@@ -212,6 +212,7 @@ def _build_looker_view(
     raw_view: Dict,
     raw_model: RawModel,
     connection: LookerConnectionConfig,
+    explore_view_folder_name: str,
     url: Optional[str],
 ) -> VirtualView:
     name = raw_view["name"]
@@ -260,7 +261,7 @@ def _build_looker_view(
             name=fullname(model, name), type=VirtualViewType.LOOKER_VIEW
         ),
         looker_view=view,
-        structure=_get_model_asset_structure(model, name),
+        structure=_get_model_asset_structure(model, name, explore_view_folder_name),
         entity_upstream=(
             EntityUpstream(source_entities=source_entities) if source_entities else None
         ),
@@ -281,7 +282,11 @@ def _get_base_view_name(raw_explore: Dict, raw_model: RawModel) -> str:
 
 
 def _build_looker_explore(
-    model: str, raw_explore: Dict, raw_model: RawModel, url: Optional[str]
+    model: str,
+    raw_explore: Dict,
+    raw_model: RawModel,
+    explore_view_folder_name: str,
+    url: Optional[str],
 ) -> VirtualView:
     name = raw_explore["name"]
     base_view_name = _get_base_view_name(raw_explore, raw_model)
@@ -344,7 +349,7 @@ def _build_looker_explore(
             name=fullname(model, name), type=VirtualViewType.LOOKER_EXPLORE
         ),
         looker_explore=explore,
-        structure=_get_model_asset_structure(model, name),
+        structure=_get_model_asset_structure(model, name, explore_view_folder_name),
         entity_upstream=EntityUpstream(source_entities=source_entities),
         system_tags=SystemTags(tags=tags),
     )
@@ -606,6 +611,7 @@ def _load_model(
 def parse_project(
     base_dir: str,
     connections: Dict[str, LookerConnectionConfig],
+    explore_view_folder_name,
     projectSourceUrl: Optional[str] = None,
 ) -> Tuple[ModelMap, List[VirtualView]]:
     """
@@ -631,6 +637,7 @@ def parse_project(
                     view,
                     resolved_model,
                     connection,
+                    explore_view_folder_name,
                     entity_urls.get(view["name"]),
                 )
                 for view in resolved_model.raw_views.values()
@@ -646,6 +653,7 @@ def parse_project(
                     model_name,
                     explore,
                     resolved_model,
+                    explore_view_folder_name,
                     entity_urls.get(explore["name"]),
                 )
                 for explore in resolved_model.raw_explores.values()

diff --git a/tests/looker/test_lookml_parser.py b/tests/looker/test_lookml_parser.py
@@ -1,11 +1,6 @@
 from metaphor.common.entity_id import EntityId
 from metaphor.looker.config import LookerConnectionConfig
-from metaphor.looker.lookml_parser import (
-    VIEW_EXPLORE_FOLDER,
-    Explore,
-    Model,
-    parse_project,
-)
+from metaphor.looker.lookml_parser import Explore, Model, parse_project
 from metaphor.models.metadata_change_event import (
     AssetStructure,
     DataPlatform,
@@ -27,6 +22,8 @@
 )
 from tests.test_utils import compare_list_ignore_order
 
+VIEW_EXPLORE_FOLDER = "LookML models"
+
 connection_map = {
     "snowflake": LookerConnectionConfig(
         database="db",
@@ -44,7 +41,7 @@
 
 def test_empty_model(test_root_dir):
     models_map, virtual_views = parse_project(
-        f"{test_root_dir}/looker/empty_model", connection_map
+        f"{test_root_dir}/looker/empty_model", connection_map, VIEW_EXPLORE_FOLDER
     )
 
     expected = {"model1": Model(explores={})}
@@ -54,7 +51,10 @@ def test_empty_model(test_root_dir):
 
 def test_basic(test_root_dir):
     models_map, virtual_views = parse_project(
-        f"{test_root_dir}/looker/basic", connection_map, "http://foo/files"
+        f"{test_root_dir}/looker/basic",
+        connection_map,
+        VIEW_EXPLORE_FOLDER,
+        "http://foo/files",
     )
 
     dataset_id = EntityId(
@@ -136,7 +136,7 @@ def test_basic(test_root_dir):
 
 def test_join(test_root_dir):
     models_map, virtual_views = parse_project(
-        f"{test_root_dir}/looker/join", connection_map, ""
+        f"{test_root_dir}/looker/join", connection_map, VIEW_EXPLORE_FOLDER, ""
     )
 
     dataset_id1 = EntityId(
@@ -258,7 +258,7 @@ def test_join(test_root_dir):
 
 def test_explore_in_view(test_root_dir):
     models_map, virtual_views = parse_project(
-        f"{test_root_dir}/looker/explore_in_view", connection_map
+        f"{test_root_dir}/looker/explore_in_view", connection_map, VIEW_EXPLORE_FOLDER
     )
 
     dataset_id = EntityId(
@@ -331,7 +331,7 @@ def test_explore_in_view(test_root_dir):
 
 def test_derived_table(test_root_dir):
     models_map, virtual_views = parse_project(
-        f"{test_root_dir}/looker/derived_table", connection_map
+        f"{test_root_dir}/looker/derived_table", connection_map, VIEW_EXPLORE_FOLDER
     )
 
     virtual_view_id1 = EntityId(
@@ -458,7 +458,7 @@ def test_derived_table(test_root_dir):
 
 def test_sql_table_name(test_root_dir):
     models_map, virtual_views = parse_project(
-        f"{test_root_dir}/looker/sql_table_name", connection_map
+        f"{test_root_dir}/looker/sql_table_name", connection_map, VIEW_EXPLORE_FOLDER
     )
 
     virtual_view_id1 = EntityId(
@@ -518,7 +518,9 @@ def test_sql_table_name(test_root_dir):
 
 def test_include_relative_to_model(test_root_dir):
     _, virtual_views = parse_project(
-        f"{test_root_dir}/looker/include_relative_to_model", connection_map
+        f"{test_root_dir}/looker/include_relative_to_model",
+        connection_map,
+        VIEW_EXPLORE_FOLDER,
     )
 
     dataset_id1 = EntityId(
@@ -569,7 +571,7 @@ def test_include_relative_to_model(test_root_dir):
 
 def test_complex_includes(test_root_dir):
     models_map, virtual_views = parse_project(
-        f"{test_root_dir}/looker/complex_includes", connection_map
+        f"{test_root_dir}/looker/complex_includes", connection_map, VIEW_EXPLORE_FOLDER
     )
 
     virtual_view_id1 = EntityId(
@@ -692,7 +694,7 @@ def test_complex_includes(test_root_dir):
 
 def test_view_extension(test_root_dir):
     models_map, virtual_views = parse_project(
-        f"{test_root_dir}/looker/view_extension", connection_map
+        f"{test_root_dir}/looker/view_extension", connection_map, VIEW_EXPLORE_FOLDER
     )
 
     virtual_view_id1 = EntityId(
@@ -843,7 +845,7 @@ def test_view_extension(test_root_dir):
 
 def test_explore_extension(test_root_dir):
     models_map, virtual_views = parse_project(
-        f"{test_root_dir}/looker/explore_extension", connection_map
+        f"{test_root_dir}/looker/explore_extension", connection_map, VIEW_EXPLORE_FOLDER
     )
 
     virtual_view1 = EntityId(