diff --git a/metaphor/looker/README.md b/metaphor/looker/README.md index fb179af6..71290cf1 100644 --- a/metaphor/looker/README.md +++ b/metaphor/looker/README.md @@ -109,6 +109,14 @@ verify_ssl: false timeout: 30 # default 120 seconds ``` +#### Looker Explore/View folder name + +Looker explores and Looker views will be put in a folder named "Looker Models" by default. You can customize the name by using this config: + +```yaml +explore_view_folder_name: "Looker Views & Explores" # default to "LookML Models" +``` + #### Output Destination See [Output Config](../common/docs/output.md) for more information on the optional `output` config. diff --git a/metaphor/looker/config.py b/metaphor/looker/config.py index 7d1d6085..d165e764 100644 --- a/metaphor/looker/config.py +++ b/metaphor/looker/config.py @@ -53,6 +53,9 @@ class LookerRunConfig(BaseConfig): # Alternative base url to build the entity source URL alternative_base_url: Optional[str] = None + # LookML explores & views folder name + explore_view_folder_name: str = "LookML Models" + @model_validator(mode="after") def have_local_or_git_dir_for_lookml(self): must_set_exactly_one(self.__dict__, ["lookml_dir", "lookml_git_repo"]) diff --git a/metaphor/looker/extractor.py b/metaphor/looker/extractor.py index a22b2ca5..ca2d2604 100644 --- a/metaphor/looker/extractor.py +++ b/metaphor/looker/extractor.py @@ -80,6 +80,7 @@ def __init__(self, config: LookerRunConfig) -> None: self._lookml_git_repo = config.lookml_git_repo self._project_source_url = config.project_source_url self._include_personal_folders = config.include_personal_folders + self._explore_view_folder_name = config.explore_view_folder_name # Load config using environment variables instead from looker.ini file # See https://github.com/looker-open-source/sdk-codegen#environment-variable-configuration @@ -106,7 +107,10 @@ async def extract(self) -> Collection[ENTITY_TYPES]: logger.info(f"Parsing LookML project at {lookml_dir}") model_map, virtual_views = parse_project( - lookml_dir, connections, self._project_source_url + lookml_dir, + connections, + self._explore_view_folder_name, + self._project_source_url, ) folder_map = self._fetch_folders() diff --git a/metaphor/looker/lookml_parser.py b/metaphor/looker/lookml_parser.py index 409913d2..090d9798 100644 --- a/metaphor/looker/lookml_parser.py +++ b/metaphor/looker/lookml_parser.py @@ -198,9 +198,11 @@ def fullname(model: str, name: str) -> str: return f"{model}.{name}" -def _get_model_asset_structure(model: str, name: str) -> AssetStructure: +def _get_model_asset_structure( + model: str, name: str, explore_view_folder_name: str +) -> AssetStructure: return AssetStructure( - directories=[model], + directories=[explore_view_folder_name, model], name=name, ) @@ -210,6 +212,7 @@ def _build_looker_view( raw_view: Dict, raw_model: RawModel, connection: LookerConnectionConfig, + explore_view_folder_name: str, url: Optional[str], ) -> VirtualView: name = raw_view["name"] @@ -258,7 +261,7 @@ def _build_looker_view( name=fullname(model, name), type=VirtualViewType.LOOKER_VIEW ), looker_view=view, - structure=_get_model_asset_structure(model, name), + structure=_get_model_asset_structure(model, name, explore_view_folder_name), entity_upstream=( EntityUpstream(source_entities=source_entities) if source_entities else None ), @@ -279,7 +282,11 @@ def _get_base_view_name(raw_explore: Dict, raw_model: RawModel) -> str: def _build_looker_explore( - model: str, raw_explore: Dict, raw_model: RawModel, url: Optional[str] + model: str, + raw_explore: Dict, + raw_model: RawModel, + explore_view_folder_name: str, + url: Optional[str], ) -> VirtualView: name = raw_explore["name"] base_view_name = _get_base_view_name(raw_explore, raw_model) @@ -342,7 +349,7 @@ def _build_looker_explore( name=fullname(model, name), type=VirtualViewType.LOOKER_EXPLORE ), looker_explore=explore, - structure=_get_model_asset_structure(model, name), + structure=_get_model_asset_structure(model, name, explore_view_folder_name), entity_upstream=EntityUpstream(source_entities=source_entities), system_tags=SystemTags(tags=tags), ) @@ -604,6 +611,7 @@ def _load_model( def parse_project( base_dir: str, connections: Dict[str, LookerConnectionConfig], + explore_view_folder_name, projectSourceUrl: Optional[str] = None, ) -> Tuple[ModelMap, List[VirtualView]]: """ @@ -629,6 +637,7 @@ def parse_project( view, resolved_model, connection, + explore_view_folder_name, entity_urls.get(view["name"]), ) for view in resolved_model.raw_views.values() @@ -644,6 +653,7 @@ def parse_project( model_name, explore, resolved_model, + explore_view_folder_name, entity_urls.get(explore["name"]), ) for explore in resolved_model.raw_explores.values() diff --git a/pyproject.toml b/pyproject.toml index 279d0da0..c5f407d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "metaphor-connectors" -version = "0.14.72" +version = "0.14.73" license = "Apache-2.0" description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app." authors = ["Metaphor "] diff --git a/tests/looker/test_lookml_parser.py b/tests/looker/test_lookml_parser.py index 41b6dccb..fd4b460e 100644 --- a/tests/looker/test_lookml_parser.py +++ b/tests/looker/test_lookml_parser.py @@ -22,6 +22,8 @@ ) from tests.test_utils import compare_list_ignore_order +VIEW_EXPLORE_FOLDER = "LookML models" + connection_map = { "snowflake": LookerConnectionConfig( database="db", @@ -39,7 +41,7 @@ def test_empty_model(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/empty_model", connection_map + f"{test_root_dir}/looker/empty_model", connection_map, VIEW_EXPLORE_FOLDER ) expected = {"model1": Model(explores={})} @@ -49,7 +51,10 @@ def test_empty_model(test_root_dir): def test_basic(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/basic", connection_map, "http://foo/files" + f"{test_root_dir}/looker/basic", + connection_map, + VIEW_EXPLORE_FOLDER, + "http://foo/files", ) dataset_id = EntityId( @@ -91,7 +96,7 @@ def test_basic(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_id)]), structure=AssetStructure( - directories=["model1"], + directories=[VIEW_EXPLORE_FOLDER, "model1"], name="view1", ), ), @@ -107,7 +112,7 @@ def test_basic(test_root_dir): url="http://foo/files/model1.model.lkml", ), structure=AssetStructure( - directories=["model1"], + directories=[VIEW_EXPLORE_FOLDER, "model1"], name="explore1", ), entity_upstream=EntityUpstream( @@ -131,7 +136,7 @@ def test_basic(test_root_dir): def test_join(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/join", connection_map, "" + f"{test_root_dir}/looker/join", connection_map, VIEW_EXPLORE_FOLDER, "" ) dataset_id1 = EntityId( @@ -191,7 +196,7 @@ def test_join(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_id1)]), structure=AssetStructure( - directories=["model1"], + directories=[VIEW_EXPLORE_FOLDER, "model1"], name="view1", ), ), @@ -210,7 +215,7 @@ def test_join(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_id2)]), structure=AssetStructure( - directories=["model1"], + directories=[VIEW_EXPLORE_FOLDER, "model1"], name="view2", ), ), @@ -239,7 +244,7 @@ def test_join(test_root_dir): label="label", ), structure=AssetStructure( - directories=["model1"], + directories=[VIEW_EXPLORE_FOLDER, "model1"], name="explore1", ), entity_upstream=EntityUpstream( @@ -253,7 +258,7 @@ def test_join(test_root_dir): def test_explore_in_view(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/explore_in_view", connection_map + f"{test_root_dir}/looker/explore_in_view", connection_map, VIEW_EXPLORE_FOLDER ) dataset_id = EntityId( @@ -299,7 +304,7 @@ def test_explore_in_view(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_id)]), structure=AssetStructure( - directories=["model1"], + directories=[VIEW_EXPLORE_FOLDER, "model1"], name="view1", ), ), @@ -314,7 +319,7 @@ def test_explore_in_view(test_root_dir): label="label", ), structure=AssetStructure( - directories=["model1"], + directories=[VIEW_EXPLORE_FOLDER, "model1"], name="explore1", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id)]), @@ -326,7 +331,7 @@ def test_explore_in_view(test_root_dir): def test_derived_table(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/derived_table", connection_map + f"{test_root_dir}/looker/derived_table", connection_map, VIEW_EXPLORE_FOLDER ) virtual_view_id1 = EntityId( @@ -361,7 +366,7 @@ def test_derived_table(test_root_dir): name="model.view1", type=VirtualViewType.LOOKER_VIEW ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view1", ), looker_view=LookerView( @@ -379,7 +384,7 @@ def test_derived_table(test_root_dir): name="model.view2", type=VirtualViewType.LOOKER_VIEW ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view2", ), looker_view=LookerView( @@ -397,7 +402,7 @@ def test_derived_table(test_root_dir): name="model.view3", type=VirtualViewType.LOOKER_VIEW ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view3", ), looker_view=LookerView(), @@ -412,7 +417,7 @@ def test_derived_table(test_root_dir): base_view=str(virtual_view_id1), ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="explore1", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id1)]), @@ -427,7 +432,7 @@ def test_derived_table(test_root_dir): base_view=str(virtual_view_id2), ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="explore2", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id2)]), @@ -442,7 +447,7 @@ def test_derived_table(test_root_dir): base_view=str(virtual_view_id3), ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="explore3", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id3)]), @@ -453,7 +458,7 @@ def test_derived_table(test_root_dir): def test_sql_table_name(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/sql_table_name", connection_map + f"{test_root_dir}/looker/sql_table_name", connection_map, VIEW_EXPLORE_FOLDER ) virtual_view_id1 = EntityId( @@ -489,7 +494,7 @@ def test_sql_table_name(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_id1)]), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view1", ), ), @@ -502,7 +507,7 @@ def test_sql_table_name(test_root_dir): base_view=str(virtual_view_id1), ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="explore1", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id1)]), @@ -513,7 +518,9 @@ def test_sql_table_name(test_root_dir): def test_include_relative_to_model(test_root_dir): _, virtual_views = parse_project( - f"{test_root_dir}/looker/include_relative_to_model", connection_map + f"{test_root_dir}/looker/include_relative_to_model", + connection_map, + VIEW_EXPLORE_FOLDER, ) dataset_id1 = EntityId( @@ -542,7 +549,7 @@ def test_include_relative_to_model(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_id1)]), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view1", ), ), @@ -555,7 +562,7 @@ def test_include_relative_to_model(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_id2)]), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view2", ), ), @@ -564,7 +571,7 @@ def test_include_relative_to_model(test_root_dir): def test_complex_includes(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/complex_includes", connection_map + f"{test_root_dir}/looker/complex_includes", connection_map, VIEW_EXPLORE_FOLDER ) virtual_view_id1 = EntityId( @@ -624,7 +631,7 @@ def test_complex_includes(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_id1)]), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view1", ), ), @@ -637,7 +644,7 @@ def test_complex_includes(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_id2)]), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view2", ), ), @@ -650,7 +657,7 @@ def test_complex_includes(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_id3)]), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view3", ), ), @@ -663,7 +670,7 @@ def test_complex_includes(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_id4)]), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view4", ), ), @@ -676,7 +683,7 @@ def test_complex_includes(test_root_dir): base_view=str(virtual_view_id1), ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="explore1", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view_id1)]), @@ -687,7 +694,7 @@ def test_complex_includes(test_root_dir): def test_view_extension(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/view_extension", connection_map + f"{test_root_dir}/looker/view_extension", connection_map, VIEW_EXPLORE_FOLDER ) virtual_view_id1 = EntityId( @@ -733,7 +740,7 @@ def test_view_extension(test_root_dir): name="model.view1", type=VirtualViewType.LOOKER_VIEW ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view1", ), looker_view=LookerView( @@ -746,7 +753,7 @@ def test_view_extension(test_root_dir): name="model.view2", type=VirtualViewType.LOOKER_VIEW ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view2", ), looker_view=LookerView( @@ -763,7 +770,7 @@ def test_view_extension(test_root_dir): name="model.view3", type=VirtualViewType.LOOKER_VIEW ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view3", ), looker_view=LookerView( @@ -784,7 +791,7 @@ def test_view_extension(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_table3)]), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view4", ), ), @@ -793,7 +800,7 @@ def test_view_extension(test_root_dir): name="model.base_view2", type=VirtualViewType.LOOKER_VIEW ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="base_view2", ), looker_view=LookerView( @@ -810,7 +817,7 @@ def test_view_extension(test_root_dir): name="model.base_view3", type=VirtualViewType.LOOKER_VIEW ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="base_view3", ), looker_view=LookerView( @@ -823,7 +830,7 @@ def test_view_extension(test_root_dir): name="model.explore1", type=VirtualViewType.LOOKER_EXPLORE ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="explore1", ), looker_explore=LookerExplore( @@ -838,7 +845,7 @@ def test_view_extension(test_root_dir): def test_explore_extension(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/explore_extension", connection_map + f"{test_root_dir}/looker/explore_extension", connection_map, VIEW_EXPLORE_FOLDER ) virtual_view1 = EntityId( @@ -904,7 +911,7 @@ def test_explore_extension(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_view1)]), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view1", ), ), @@ -917,7 +924,7 @@ def test_explore_extension(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_view2)]), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view2", ), ), @@ -930,7 +937,7 @@ def test_explore_extension(test_root_dir): ), entity_upstream=EntityUpstream(source_entities=[str(dataset_view3)]), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view3", ), ), @@ -943,7 +950,7 @@ def test_explore_extension(test_root_dir): base_view=str(virtual_view1), ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="explore1", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view1)]), @@ -958,7 +965,7 @@ def test_explore_extension(test_root_dir): base_view=str(virtual_view2), ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="explore2", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view2)]), @@ -973,7 +980,7 @@ def test_explore_extension(test_root_dir): base_view=str(virtual_view1), ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="explore3", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view1)]), @@ -988,7 +995,7 @@ def test_explore_extension(test_root_dir): base_view=str(virtual_view3), ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="explore4", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view3)]), @@ -1003,7 +1010,7 @@ def test_explore_extension(test_root_dir): base_view=str(virtual_view2), ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="base_explore2", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view2)]), @@ -1018,7 +1025,7 @@ def test_explore_extension(test_root_dir): base_view=str(virtual_view3), ), structure=AssetStructure( - directories=["model"], + directories=[VIEW_EXPLORE_FOLDER, "model"], name="view3", ), entity_upstream=EntityUpstream(source_entities=[str(virtual_view3)]),