From cc6e8b1dddc6a04d7c8e6306530cd939fcebc494 Mon Sep 17 00:00:00 2001 From: Mike Gouline <1960272+gouline@users.noreply.github.com> Date: Thu, 25 Jul 2024 11:51:13 +1000 Subject: [PATCH] Allow tags on exposures (#271) * Allow tags on exposures * Fix help and tests * Naming and docs cleanup --- Makefile | 1 + dbtmetabase/__main__.py | 10 ++++ dbtmetabase/_exposures.py | 21 +++++++- tests/fixtures/exposure/default/exposures.yml | 14 +++++ tests/test_exposures.py | 52 ++++++++++--------- 5 files changed, 72 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 1fb21a26..a7126aa0 100644 --- a/Makefile +++ b/Makefile @@ -94,6 +94,7 @@ sandbox-exposures: --metabase-password $$MB_PASSWORD \ --output-path sandbox/models/exposures \ --output-grouping collection \ + --tag metabase \ --verbose ) ( . sandbox/.env && cd sandbox && \ diff --git a/dbtmetabase/__main__.py b/dbtmetabase/__main__.py index d8771b68..a28a8441 100644 --- a/dbtmetabase/__main__.py +++ b/dbtmetabase/__main__.py @@ -155,6 +155,7 @@ def _add_setup(func: Callable) -> Callable: @click.option( "--http-header", "http_headers", + metavar="KEY VALUE", type=(str, str), multiple=True, help="Additional HTTP request headers.", @@ -377,6 +378,13 @@ def models( is_flag=True, help="Exclude items that have not been verified. Only applies to entity types that support verification.", ) +@click.option( + "--tag", + "tags", + metavar="TAG", + multiple=True, + help="Optional tags for exported dbt exposures.", +) def exposures( output_path: str, output_grouping: Optional[str], @@ -384,6 +392,7 @@ def exposures( exclude_collections: Optional[Sequence[str]], allow_personal_collections: bool, exclude_unverified: bool, + tags: Sequence[str], core: DbtMetabase, ): core.extract_exposures( @@ -395,6 +404,7 @@ def exposures( ), allow_personal_collections=allow_personal_collections, exclude_unverified=exclude_unverified, + tags=tags, ) diff --git a/dbtmetabase/_exposures.py b/dbtmetabase/_exposures.py index 4e95927e..f27ec08a 100644 --- a/dbtmetabase/_exposures.py +++ b/dbtmetabase/_exposures.py @@ -6,7 +6,15 @@ from abc import ABCMeta, abstractmethod from operator import itemgetter from pathlib import Path -from typing import Iterable, Mapping, MutableMapping, MutableSequence, Optional, Tuple +from typing import ( + Iterable, + Mapping, + MutableMapping, + MutableSequence, + Optional, + Sequence, + Tuple, +) from dbtmetabase.metabase import Metabase @@ -47,6 +55,7 @@ def extract_exposures( collection_filter: Optional[Filter] = None, allow_personal_collections: bool = False, exclude_unverified: bool = False, + tags: Optional[Sequence[str]] = None, ) -> Iterable[Mapping]: """Extract dbt exposures from Metabase. @@ -56,6 +65,7 @@ def extract_exposures( collection_filter (Optional[Filter], optional): Filter Metabase collections. Defaults to None. allow_personal_collections (bool, optional): Allow personal Metabase collections. Defaults to False. exclude_unverified (bool, optional): Exclude items that have not been verified. Only applies to entity types that support verification. Defaults to False. + tags (Sequence[str], optional): Optional tags for exported dbt exposures. Defaults to None. Returns: Iterable[Mapping]: List of parsed exposures. @@ -188,6 +198,7 @@ def extract_exposures( if depend.lower() in ctx.model_refs ] ), + tags=tags, ), } ) @@ -301,6 +312,7 @@ def __format_exposure( creator_email: str, native_query: Optional[str], depends_on: Iterable[str], + tags: Optional[Sequence[str]], ) -> Mapping: """Builds dbt exposure representation (see https://docs.getdbt.com/reference/exposure-properties).""" @@ -334,7 +346,7 @@ def __format_exposure( + f"Created On: __{created_at}__" ) - return { + exposure = { "name": name, "label": label, "description": safe_description( @@ -350,6 +362,11 @@ def __format_exposure( "depends_on": list(depends_on), } + if tags: + exposure["tags"] = list(tags) + + return exposure + @staticmethod def __write_exposures( exposures: Iterable[Mapping], diff --git a/tests/fixtures/exposure/default/exposures.yml b/tests/fixtures/exposure/default/exposures.yml index f312f851..468c4967 100644 --- a/tests/fixtures/exposure/default/exposures.yml +++ b/tests/fixtures/exposure/default/exposures.yml @@ -14,6 +14,8 @@ exposures: email: dbtmetabase@example.com depends_on: - ref('orders') + tags: + - metabase - name: dummy label: Dummy description: '### Visualization: Table @@ -46,6 +48,8 @@ exposures: name: dbtmetabase email: dbtmetabase@example.com depends_on: [] + tags: + - metabase - name: dummy_1 label: Dummy description: '### Visualization: Table @@ -78,6 +82,8 @@ exposures: name: dbtmetabase email: dbtmetabase@example.com depends_on: [] + tags: + - metabase - name: orders___customers label: Orders + Customers description: '### Visualization: Table @@ -102,6 +108,8 @@ exposures: depends_on: - ref('customers') - ref('orders') + tags: + - metabase - name: orders___customers__filtered_by_status_is_completed label: Orders + Customers, Filtered by Status is completed description: '### Visualization: Table @@ -126,6 +134,8 @@ exposures: depends_on: - ref('customers') - ref('orders') + tags: + - metabase - name: returned_order_count_sql label: Returned Order Count SQL description: "### Visualization: Scalar\n\nNo description provided in Metabase\n\ @@ -141,6 +151,8 @@ exposures: depends_on: - ref('stg_orders') - ref('stg_payments') + tags: + - metabase - name: the_dashboard label: The Dashboard description: '### Dashboard Cards: 3 @@ -167,3 +179,5 @@ exposures: - ref('orders') - ref('stg_orders') - ref('stg_payments') + tags: + - metabase diff --git a/tests/test_exposures.py b/tests/test_exposures.py index cdf87902..d365408b 100644 --- a/tests/test_exposures.py +++ b/tests/test_exposures.py @@ -6,7 +6,9 @@ from tests._mocks import FIXTURES_PATH, TMP_PATH, MockDbtMetabase -TMP_PATH.mkdir(exist_ok=True) + +def setup_module(): + TMP_PATH.mkdir(exist_ok=True) @pytest.fixture(name="core") @@ -23,12 +25,36 @@ def _assert_exposures(expected_path: Path, actual_path: Path): assert actual["exposures"] == sorted(expected["exposures"], key=itemgetter("name")) -def test_exposures(core: MockDbtMetabase): +def test_exposures_default(core: MockDbtMetabase): fixtures_path = FIXTURES_PATH / "exposure" / "default" output_path = TMP_PATH / "exposure" / "default" core.extract_exposures( output_path=str(output_path), output_grouping=None, + tags=["metabase"], + ) + + _assert_exposures( + fixtures_path / "exposures.yml", + output_path / "exposures.yml", + ) + + +def test_exposures_default_aliased(core: MockDbtMetabase): + for model in core.manifest.read_models(): + if not model.name.startswith("stg_"): + model.alias = f"{model.name}_alias" + + aliases = [m.alias for m in core.manifest.read_models()] + assert "orders_alias" in aliases + assert "customers_alias" in aliases + + fixtures_path = FIXTURES_PATH / "exposure" / "default" + output_path = TMP_PATH / "exposure" / "aliased" + core.extract_exposures( + output_path=str(output_path), + output_grouping=None, + tags=["metabase"], ) _assert_exposures( @@ -62,25 +88,3 @@ def test_exposures_grouping_type(core: MockDbtMetabase): for file in (fixtures_path / "dashboard").iterdir(): _assert_exposures(file, output_path / "dashboard" / file.name) - - -def test_exposures_aliased_ref(core: MockDbtMetabase): - for model in core.manifest.read_models(): - if not model.name.startswith("stg_"): - model.alias = f"{model.name}_alias" - - aliases = [m.alias for m in core.manifest.read_models()] - assert "orders_alias" in aliases - assert "customers_alias" in aliases - - fixtures_path = FIXTURES_PATH / "exposure" / "default" - output_path = TMP_PATH / "exposure" / "aliased" - core.extract_exposures( - output_path=str(output_path), - output_grouping=None, - ) - - _assert_exposures( - fixtures_path / "exposures.yml", - output_path / "exposures.yml", - )