fix: Consistent model selection in parsing (#141)

dbtmetabase/__init__.py

@@ -724,8 +724,7 @@ def exposures(
     )
 
     # Load models
-    dbt_models, aliases = dbt.read_models()
-    del aliases  # Unused in this particular function
+    dbt_models, _ = dbt.read_models()
 
     # Instantiate Metabase interface
     metabase = MetabaseInterface(

dbtmetabase/metabase.py

@@ -1,25 +1,30 @@
-import re
 import json
-import requests
-import time
-import yaml
 import os
-
+import re
+import time
 from typing import (
-    Sequence,
-    Optional,
-    Tuple,
     Iterable,
-    MutableMapping,
-    Union,
     List,
     Mapping,
+    MutableMapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
 )
 
-from dbtmetabase.models import exceptions
+import requests
+import yaml
 
 from .logger.logging import logger
-from .models.metabase import MetabaseModel, MetabaseColumn, ModelType, NullValue
+from .models import exceptions
+from .models.metabase import (
+    METABASE_MODEL_DEFAULT_SCHEMA,
+    MetabaseColumn,
+    MetabaseModel,
+    ModelType,
+    NullValue,
+)
 
 
 class MetabaseClient:
@@ -458,7 +463,9 @@ def build_metadata_lookups(
             table_schema = table.get("schema")
             # table["schema"] is null for bigquery datasets
             bigquery_schema = metadata.get("details", {}).get("dataset-id")
-            table_schema = (table_schema or bigquery_schema or "public").upper()
+            table_schema = (
+                table_schema or bigquery_schema or METABASE_MODEL_DEFAULT_SCHEMA
+            ).upper()
             table_name = table["name"].upper()
 
             if schemas_to_exclude:

dbtmetabase/models/metabase.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass, field
 from enum import Enum
-
-from typing import Sequence, Optional, MutableMapping
+from typing import MutableMapping, Optional, Sequence
 
 # Allowed metabase.* fields
 # Must be covered by MetabaseModel attributes
@@ -16,6 +15,9 @@
     "semantic_type",
 ]
 
+# Default model schema (only schema in BigQuery)
+METABASE_MODEL_DEFAULT_SCHEMA = "PUBLIC"
+
 
 class ModelType(str, Enum):
     nodes = "nodes"

dbtmetabase/parsers/dbt.py

@@ -1,9 +1,9 @@
 from abc import ABCMeta, abstractmethod
 from os.path import expanduser
-from typing import Optional, Mapping, MutableMapping, Iterable, Tuple, List
+from typing import Iterable, List, Mapping, MutableMapping, Optional, Tuple
 
 from ..logger.logging import logger
-from ..models.metabase import MetabaseModel, MetabaseColumn, NullValue
+from ..models.metabase import MetabaseColumn, MetabaseModel, NullValue
 
 
 class DbtReader(metaclass=ABCMeta):
@@ -31,11 +31,11 @@ def __init__(
         """
 
         self.path = expanduser(path)
-        self.database = database
-        self.schema = schema
-        self.schema_excludes = schema_excludes
-        self.includes = includes
-        self.excludes = excludes
+        self.database = database.upper() if database else None
+        self.schema = schema.upper() if schema else None
+        self.schema_excludes = [x.upper() for x in schema_excludes or []]
+        self.includes = [x.upper() for x in includes or []]
+        self.excludes = [x.upper() for x in excludes or []]
         self.alias_mapping: MutableMapping = {}
 
     @abstractmethod
@@ -46,6 +46,18 @@ def read_models(
     ) -> Tuple[List[MetabaseModel], MutableMapping]:
         pass
 
+    def model_selected(self, name: str) -> bool:
+        """Checks whether model passes inclusion/exclusion criteria.
+
+        Args:
+            name (str): Model name.
+
+        Returns:
+            bool: True if included, false otherwise.
+        """
+        n = name.upper()
+        return n not in self.excludes and (not self.includes or n in self.includes)
+
     def set_column_foreign_key(
         self,
         column: Mapping,

dbtmetabase/parsers/dbt_folder.py

@@ -1,15 +1,17 @@
 import re
-import yaml
 from pathlib import Path
 from typing import List, Mapping, MutableMapping, Optional, Tuple
 
+import yaml
+
 from ..logger.logging import logger
 from ..models.metabase import (
-    MetabaseModel,
+    METABASE_COLUMN_META_FIELDS,
+    METABASE_MODEL_DEFAULT_SCHEMA,
+    METABASE_MODEL_META_FIELDS,
     MetabaseColumn,
+    MetabaseModel,
     ModelType,
-    METABASE_MODEL_META_FIELDS,
-    METABASE_COLUMN_META_FIELDS,
 )
 from .dbt import DbtReader
 
@@ -34,78 +36,86 @@ def read_models(
             list -- List of dbt models in Metabase-friendly format.
         """
 
-        database = self.database
-        schema = self.schema
-        schema_excludes = self.schema_excludes
-        includes = self.includes
-        excludes = self.excludes
-
-        if schema_excludes is None:
-            schema_excludes = []
-        if includes is None:
-            includes = []
-        if excludes is None:
-            excludes = []
-        if schema is None:
-            schema = "public"
-
-        # Args that allow API interface for both readers to be interchangeable while passing CI
-        del database, docs_url
-
         mb_models: List[MetabaseModel] = []
 
+        schema = self.schema or METABASE_MODEL_DEFAULT_SCHEMA
+
         for path in (Path(self.path) / "models").rglob("*.yml"):
             with open(path, "r", encoding="utf-8") as stream:
                 schema_file = yaml.safe_load(stream)
-                if schema_file is None:
+                if not schema_file:
                     logger().warning("Skipping empty or invalid YAML: %s", path)
                     continue
+
                 for model in schema_file.get("models", []):
-                    name = model.get("alias", model["name"])
+                    model_name = model.get("alias", model["name"]).upper()
+
                     # Refs will still use file name -- this alias mapping is good for getting the right name in the database
                     if "alias" in model:
-                        self.alias_mapping[name] = model["name"]
+                        self.alias_mapping[model_name] = model["name"].upper()
+
                     logger().info("\nProcessing model: %s", path)
-                    if (not includes or name in includes) and (name not in excludes):
-                        mb_models.append(
-                            self._read_model(
-                                model=model,
-                                schema=schema.upper(),
-                                model_type=ModelType.nodes,
-                                include_tags=include_tags,
-                            )
+
+                    if not self.model_selected(model_name):
+                        logger().debug(
+                            "Skipping %s not included in includes or excluded by excludes",
+                            model_name,
                         )
-                        logger().debug(mb_models[-1].ref)
+                        continue
+
+                    mb_models.append(
+                        self._read_model(
+                            model=model,
+                            schema=schema,
+                            model_type=ModelType.nodes,
+                            include_tags=include_tags,
+                        )
+                    )
+
                 for source in schema_file.get("sources", []):
-                    source_schema_name = source.get("schema", source["name"])
+                    source_schema_name = source.get("schema", source["name"]).upper()
+
                     if "{{" in source_schema_name and "}}" in source_schema_name:
                         logger().warning(
-                            "dbt Folder Reader cannot resolve jinja expressions- use the Manifest Reader instead."
+                            "dbt folder reader cannot resolve Jinja expressions, defaulting to current schema"
                         )
                         source_schema_name = schema
-                    if source_schema_name.upper() != schema.upper():
+
+                    elif source_schema_name != schema:
+                        logger().debug(
+                            "Skipping schema %s not in target schema %s",
+                            source_schema_name,
+                            schema,
+                        )
                         continue
+
                     for model in source.get("tables", []):
-                        name = model.get("identifier", model["name"])
+                        model_name = model.get("identifier", model["name"]).upper()
+
                         # These will be used to resolve our regex parsed source() references
                         if "identifier" in model:
-                            self.alias_mapping[name] = model["name"]
+                            self.alias_mapping[model_name] = model["name"].upper()
+
                         logger().info(
-                            "\nProcessing source: %s -- table: %s", path, name
+                            "\nProcessing source: %s -- table: %s", path, model_name
                         )
-                        if (not includes or name in includes) and (
-                            name not in excludes
-                        ):
-                            mb_models.append(
-                                self._read_model(
-                                    model=model,
-                                    source=source["name"],
-                                    model_type=ModelType.sources,
-                                    schema=source_schema_name.upper(),
-                                    include_tags=include_tags,
-                                )
+
+                        if not self.model_selected(model_name):
+                            logger().debug(
+                                "Skipping %s not included in includes or excluded by excludes",
+                                model_name,
+                            )
+                            continue
+
+                        mb_models.append(
+                            self._read_model(
+                                model=model,
+                                source=source["name"],
+                                model_type=ModelType.sources,
+                                schema=source_schema_name,
+                                include_tags=include_tags,
                             )
-                            logger().debug(mb_models[-1].ref)
+                        )
 
         return mb_models, self.alias_mapping
 
@@ -168,7 +178,7 @@ def _read_column(self, column: Mapping, schema: str) -> MetabaseColumn:
 
         Arguments:
             column {dict} -- One dbt column to read.
-            schema {str} -- Schema as passed doen from CLI args or parsed from `source`
+            schema {str} -- Schema as passed down from CLI args or parsed from `source`
 
         Returns:
             dict -- One dbt column in Metabase-friendly format.