diff --git a/metaphor/looker/README.md b/metaphor/looker/README.md index fb179af6..cd6a7327 100644 --- a/metaphor/looker/README.md +++ b/metaphor/looker/README.md @@ -109,6 +109,14 @@ verify_ssl: false timeout: 30 # default 120 seconds ``` +#### Looker Explore/View folder name + +We will put Looker explores and Looker views in a folder in Metaphor, and you can customize the folder name. + +```yaml +explore_view_folder_name: "LookML models" # default is "LookML models" +``` + #### Output Destination See [Output Config](../common/docs/output.md) for more information on the optional `output` config. diff --git a/metaphor/looker/config.py b/metaphor/looker/config.py index 7d1d6085..45ca504f 100644 --- a/metaphor/looker/config.py +++ b/metaphor/looker/config.py @@ -53,6 +53,9 @@ class LookerRunConfig(BaseConfig): # Alternative base url to build the entity source URL alternative_base_url: Optional[str] = None + # LookML explores & views folder name + explore_view_folder_name: str = "LookML models" + @model_validator(mode="after") def have_local_or_git_dir_for_lookml(self): must_set_exactly_one(self.__dict__, ["lookml_dir", "lookml_git_repo"]) diff --git a/metaphor/looker/extractor.py b/metaphor/looker/extractor.py index a22b2ca5..ca2d2604 100644 --- a/metaphor/looker/extractor.py +++ b/metaphor/looker/extractor.py @@ -80,6 +80,7 @@ def __init__(self, config: LookerRunConfig) -> None: self._lookml_git_repo = config.lookml_git_repo self._project_source_url = config.project_source_url self._include_personal_folders = config.include_personal_folders + self._explore_view_folder_name = config.explore_view_folder_name # Load config using environment variables instead from looker.ini file # See https://github.com/looker-open-source/sdk-codegen#environment-variable-configuration @@ -106,7 +107,10 @@ async def extract(self) -> Collection[ENTITY_TYPES]: logger.info(f"Parsing LookML project at {lookml_dir}") model_map, virtual_views = parse_project( - lookml_dir, connections, self._project_source_url + lookml_dir, + connections, + self._explore_view_folder_name, + self._project_source_url, ) folder_map = self._fetch_folders() diff --git a/metaphor/looker/lookml_parser.py b/metaphor/looker/lookml_parser.py index ec6fd9d4..090d9798 100644 --- a/metaphor/looker/lookml_parser.py +++ b/metaphor/looker/lookml_parser.py @@ -43,8 +43,6 @@ # lkml parser's debug log can be very noisy logging.getLogger("lkml.parser").setLevel(logging.WARNING) -VIEW_EXPLORE_FOLDER = "LookML models" - @dataclass class Explore: @@ -200,9 +198,11 @@ def fullname(model: str, name: str) -> str: return f"{model}.{name}" -def _get_model_asset_structure(model: str, name: str) -> AssetStructure: +def _get_model_asset_structure( + model: str, name: str, explore_view_folder_name: str +) -> AssetStructure: return AssetStructure( - directories=[VIEW_EXPLORE_FOLDER, model], + directories=[explore_view_folder_name, model], name=name, ) @@ -212,6 +212,7 @@ def _build_looker_view( raw_view: Dict, raw_model: RawModel, connection: LookerConnectionConfig, + explore_view_folder_name: str, url: Optional[str], ) -> VirtualView: name = raw_view["name"] @@ -260,7 +261,7 @@ def _build_looker_view( name=fullname(model, name), type=VirtualViewType.LOOKER_VIEW ), looker_view=view, - structure=_get_model_asset_structure(model, name), + structure=_get_model_asset_structure(model, name, explore_view_folder_name), entity_upstream=( EntityUpstream(source_entities=source_entities) if source_entities else None ), @@ -281,7 +282,11 @@ def _get_base_view_name(raw_explore: Dict, raw_model: RawModel) -> str: def _build_looker_explore( - model: str, raw_explore: Dict, raw_model: RawModel, url: Optional[str] + model: str, + raw_explore: Dict, + raw_model: RawModel, + explore_view_folder_name: str, + url: Optional[str], ) -> VirtualView: name = raw_explore["name"] base_view_name = _get_base_view_name(raw_explore, raw_model) @@ -344,7 +349,7 @@ def _build_looker_explore( name=fullname(model, name), type=VirtualViewType.LOOKER_EXPLORE ), looker_explore=explore, - structure=_get_model_asset_structure(model, name), + structure=_get_model_asset_structure(model, name, explore_view_folder_name), entity_upstream=EntityUpstream(source_entities=source_entities), system_tags=SystemTags(tags=tags), ) @@ -606,6 +611,7 @@ def _load_model( def parse_project( base_dir: str, connections: Dict[str, LookerConnectionConfig], + explore_view_folder_name, projectSourceUrl: Optional[str] = None, ) -> Tuple[ModelMap, List[VirtualView]]: """ @@ -631,6 +637,7 @@ def parse_project( view, resolved_model, connection, + explore_view_folder_name, entity_urls.get(view["name"]), ) for view in resolved_model.raw_views.values() @@ -646,6 +653,7 @@ def parse_project( model_name, explore, resolved_model, + explore_view_folder_name, entity_urls.get(explore["name"]), ) for explore in resolved_model.raw_explores.values() diff --git a/tests/looker/test_lookml_parser.py b/tests/looker/test_lookml_parser.py index 9b703445..fd4b460e 100644 --- a/tests/looker/test_lookml_parser.py +++ b/tests/looker/test_lookml_parser.py @@ -1,11 +1,6 @@ from metaphor.common.entity_id import EntityId from metaphor.looker.config import LookerConnectionConfig -from metaphor.looker.lookml_parser import ( - VIEW_EXPLORE_FOLDER, - Explore, - Model, - parse_project, -) +from metaphor.looker.lookml_parser import Explore, Model, parse_project from metaphor.models.metadata_change_event import ( AssetStructure, DataPlatform, @@ -27,6 +22,8 @@ ) from tests.test_utils import compare_list_ignore_order +VIEW_EXPLORE_FOLDER = "LookML models" + connection_map = { "snowflake": LookerConnectionConfig( database="db", @@ -44,7 +41,7 @@ def test_empty_model(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/empty_model", connection_map + f"{test_root_dir}/looker/empty_model", connection_map, VIEW_EXPLORE_FOLDER ) expected = {"model1": Model(explores={})} @@ -54,7 +51,10 @@ def test_empty_model(test_root_dir): def test_basic(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/basic", connection_map, "http://foo/files" + f"{test_root_dir}/looker/basic", + connection_map, + VIEW_EXPLORE_FOLDER, + "http://foo/files", ) dataset_id = EntityId( @@ -136,7 +136,7 @@ def test_basic(test_root_dir): def test_join(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/join", connection_map, "" + f"{test_root_dir}/looker/join", connection_map, VIEW_EXPLORE_FOLDER, "" ) dataset_id1 = EntityId( @@ -258,7 +258,7 @@ def test_join(test_root_dir): def test_explore_in_view(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/explore_in_view", connection_map + f"{test_root_dir}/looker/explore_in_view", connection_map, VIEW_EXPLORE_FOLDER ) dataset_id = EntityId( @@ -331,7 +331,7 @@ def test_explore_in_view(test_root_dir): def test_derived_table(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/derived_table", connection_map + f"{test_root_dir}/looker/derived_table", connection_map, VIEW_EXPLORE_FOLDER ) virtual_view_id1 = EntityId( @@ -458,7 +458,7 @@ def test_derived_table(test_root_dir): def test_sql_table_name(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/sql_table_name", connection_map + f"{test_root_dir}/looker/sql_table_name", connection_map, VIEW_EXPLORE_FOLDER ) virtual_view_id1 = EntityId( @@ -518,7 +518,9 @@ def test_sql_table_name(test_root_dir): def test_include_relative_to_model(test_root_dir): _, virtual_views = parse_project( - f"{test_root_dir}/looker/include_relative_to_model", connection_map + f"{test_root_dir}/looker/include_relative_to_model", + connection_map, + VIEW_EXPLORE_FOLDER, ) dataset_id1 = EntityId( @@ -569,7 +571,7 @@ def test_include_relative_to_model(test_root_dir): def test_complex_includes(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/complex_includes", connection_map + f"{test_root_dir}/looker/complex_includes", connection_map, VIEW_EXPLORE_FOLDER ) virtual_view_id1 = EntityId( @@ -692,7 +694,7 @@ def test_complex_includes(test_root_dir): def test_view_extension(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/view_extension", connection_map + f"{test_root_dir}/looker/view_extension", connection_map, VIEW_EXPLORE_FOLDER ) virtual_view_id1 = EntityId( @@ -843,7 +845,7 @@ def test_view_extension(test_root_dir): def test_explore_extension(test_root_dir): models_map, virtual_views = parse_project( - f"{test_root_dir}/looker/explore_extension", connection_map + f"{test_root_dir}/looker/explore_extension", connection_map, VIEW_EXPLORE_FOLDER ) virtual_view1 = EntityId(