Fix dbt connector by removing undocumented fields (#615)

* Fix dbt connector by removing undocumented fields

* Add test for safe_del

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
         exclude: ^(metaphor/dbt/generated/.+)$
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.3.0
+    rev: v1.5.1
     hooks:
       - id: mypy
         additional_dependencies:

metaphor/dbt/gen_models.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -e
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+cd ${SCRIPT_DIR}
+
 # Generate various data models for dbt manifest & catalog using official JSON schemas
 
 if [ $# -ne 2 ]; then
@@ -11,25 +14,26 @@ fi
 SCHEMA="$1"
 VERSION="$2"
 
-URL=https://schemas.getdbt.com/dbt/$SCHEMA/$VERSION.json
+URL=https://schemas.getdbt.com/dbt/${SCHEMA}/${VERSION}.json
 
 OUTPUT=generated/dbt_${SCHEMA}_${VERSION}.py
 
 CLASS_NAME=""
-if [[ "$SCHEMA" == "manifest" ]]; then
+if [[ "${SCHEMA}" == "manifest" ]]; then
   CLASS_NAME="DbtManifest"
-elif [[ "$SCHEMA" == "catalog" ]]; then
+elif [[ "${SCHEMA}" == "catalog" ]]; then
   CLASS_NAME="DbtCatalog"
 else
   echo -e "Choose either 'manifest' or 'catalog'"
   exit  1
 fi
 
 poetry run datamodel-codegen \
-  --url $URL \
-  --class-name $CLASS_NAME \
+  --url ${URL} \
+  --class-name ${CLASS_NAME} \
   --enum-field-as-literal all \
-  --output $OUTPUT
+  --input-file-type jsonschema \
+  --output ${OUTPUT}
 
 # Disable mypy type-checking for generated files
-sed -i '' '1s;^;# mypy: ignore-errors\n\n;' $OUTPUT
+sed -i '' '1s;^;# mypy: ignore-errors\n\n;' ${OUTPUT}

metaphor/dbt/manifest_parser.py

@@ -244,6 +244,20 @@ def sanitize_manifest(self, manifest_json: Dict, schema_version: str) -> Dict:
         # avoid hitting any validation issues.
         manifest_json["docs"] = {}
 
+        # Temporarily strip off all the extra "labels" in "semantic_models" until
+        # https://github.com/dbt-labs/dbt-core/issues/8763 is fixed
+        for _, semantic_model in manifest_json.get("semantic_models", {}).items():
+            semantic_model.pop("label", None)
+
+            for entity in semantic_model.get("entities", []):
+                entity.pop("label", None)
+
+            for dimension in semantic_model.get("dimensions", []):
+                dimension.pop("label", None)
+
+            for measure in semantic_model.get("measures", []):
+                measure.pop("label", None)
+
         return manifest_json
 
     def parse(self, manifest_json: Dict) -> None:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.12.58"
+version = "0.12.59"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]
@@ -92,11 +92,11 @@ apache-airflow = "^2.6.3"
 bandit = "^1.7.2"
 black = "^23.3.0"
 coverage = "^7.1.0"
-datamodel-code-generator = { extras = ["http"], version = "^0.21.0" }
+datamodel-code-generator = { extras = ["http"], version = "^0.22.0" }
 flake8 = "^6.0.0"
 freezegun = "^1.2.2"
 isort = "^5.11.4"
-mypy = "^1.3"
+mypy = "^1.5.1"
 pytest = "^7.2.1"
 pytest-asyncio = "^0.21.0"
 pytest-cov = "^4.0.0"