Merge pull request #12 from finn-auto/DP-2431/check-target-schema-in-…

…snapshots [DP-2431] check target schema in snapshots
finn-auto · Dec 4, 2023 · 0ceb859 · 0ceb859
2 parents 2eec800 + c0799c3
commit 0ceb859
Show file tree

Hide file tree

Showing 4 changed files with 120 additions and 2 deletions.
diff --git a/pre_commit_dbt/check_snapshot_has_target_schema.py b/pre_commit_dbt/check_snapshot_has_target_schema.py
@@ -9,7 +9,7 @@
 from pre_commit_dbt.utils import get_json
 from pre_commit_dbt.utils import get_missing_file_paths
 from pre_commit_dbt.utils import get_model_sqls
-from pre_commit_dbt.utils import get_models
+from pre_commit_dbt.utils import get_snapshots
 from pre_commit_dbt.utils import JsonOpenError
 
 
@@ -23,7 +23,8 @@ def validate_tags(
     filenames = set(sqls.keys())
 
     # get manifest nodes that pre-commit found as changed
-    models = get_models(manifest, filenames)
+    models = get_snapshots(manifest, filenames)
+
     for model in models:
         # tags can be specified only from manifest
         raw_code = model.node.get("raw_code", [])

diff --git a/pre_commit_dbt/utils.py b/pre_commit_dbt/utils.py
@@ -144,6 +144,18 @@ def get_models(
             yield Model(key, node.get("name"), filename, node)  # pragma: no mutate
 
 
+def get_snapshots(
+    manifest: Dict[str, Any],
+    filenames: Set[str]
+) -> Generator[Model, None, None]:
+    nodes = manifest.get("nodes", {})
+    for key, node in nodes.items():
+        split_key = key.split(".")
+        filename = split_key[-1]
+        if filename in filenames and split_key[0] == "snapshot":
+            yield Model(key, node.get("name"), filename, node)  # pragma: no mutate
+
+
 def get_ephemeral(
     manifest: Dict[str, Any],
 ) -> List[str]:

diff --git a/tests/unit/test-manifest.json b/tests/unit/test-manifest.json
@@ -0,0 +1,77 @@
+{"metadata":
+{"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v10.json",
+  "dbt_version": "1.6.0",
+  "generated_at": "2023-11-28T16:20:41.115139Z",
+  "invocation_id": "7b7b0778-6eab-40ba-b520-3004a18639f1",
+  "env": {},
+  "project_name": "finnauto",
+  "project_id": "7d1e44fc8134dfc5d0dffea3bd5bdf39",
+  "user_id": "946185e7-9473-4754-ab18-8d8995b68c96",
+  "send_anonymous_usage_stats": true,
+  "adapter_type": "bigquery"},
+  "nodes": {"snapshot.project.in_snapshot_target_schema": {
+    "database": "datawarehouse-304513",
+    "schema": "in_snapshot_target_schema",
+    "name": "in_snapshot_target_schema",
+    "resource_type": "snapshot",
+    "package_name": "finnauto",
+    "path": "npu_calc_snapshots/in_snapshot_target_schema.sql",
+    "original_file_path": "sources/snapshots/in_snapshot_target_schema.sql",
+    "unique_id": "snapshot.finnauto.in_snapshot_target_schema",
+    "fqn": ["finnauto", "snapshots", "in_snapshot_target_schema", "in_snapshot_target_schema"],
+    "alias": "in_snapshot_target_schema",
+    "checksum": {"name": "sha256", "checksum": "6b60716708112b1f25b335e121410f02041d158a74954abcdef7c51235562a0f"},
+    "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "finn_car_id_subscription_id_row_num", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "strategy": "timestamp", "target_schema": "dwh_snapshots_prod", "target_database": null, "updated_at": "snapshot_run_timestamp", "check_cols": null, "invalidate_hard_deletes": true, "post-hook": [], "pre-hook": []},
+    "tags": [],
+    "description": "",
+    "columns": {},
+    "meta": {},
+    "group": null,
+    "docs": {"show": true, "node_color": null},
+    "patch_path": null,
+    "build_path": null,
+    "deferred": false,
+    "unrendered_config": {"target_schema": "dwh_snapshots_prod", "unique_key": "finn_car_id_subscription_id_row_num", "strategy": "timestamp", "updated_at": "snapshot_run_timestamp", "invalidate_hard_deletes": true},
+    "created_at": 1701188441.6212869,
+    "relation_name": "`datawarehouse-304513`.`dwh_snapshots_prod`.`in_snapshot_target_schema`",
+    "raw_code": "\n\n{{\n    config(\n      target_schema=generate_schema_name('dwh_snapshots_prod'),\n       unique_key='finn_car_id_subscription_id_row_num',\n      strategy='timestamp',\n      updated_at='snapshot_run_timestamp',\n      invalidate_hard_deletes=True\n    )\n}}\n\nWITH npu_calc AS (\n    SELECT\n        {{ star_strings(from=ref('full_joined_npu_calc')) }},\n        CAST(all_npus_de.npu_expected AS STRING) AS npu_expected,\n        CURRENT_TIMESTAMP() AS snapshot_run_timestampasdf,\n        CONCAT(finn_car_id, '-', IFNULL(subscription_id,\"-\")) AS finn_car_id_subscription_id,\n        ROW_NUMBER() OVER(PARTITION BY CONCAT(finn_car_id, '-', IFNULL(subscription_id,\"-\")) ORDER BY finn_car_id) AS row_number\n    FROM {{ ref('full_joined_npu_calc') }}\n    LEFT JOIN {{  ref('il_fleet_all_cars_npus_blue_dragon_de')  }} AS all_npus_de\n    USING (finn_car_id)\n)\n\nSELECT\n    * EXCEPT(row_number),\n    CONCAT(finn_car_id_subscription_id, '_', CAST(row_number AS STRING)) AS finn_car_id_subscription_id_row_num\nFROM npu_calc\n\n",
+    "language": "sql",
+    "refs": [{"name": "full_joined_npu_calc", "package": null, "version": null}, {"name": "full_joined_npu_calc", "package": null, "version": null}, {"name": "il_fleet_all_cars_npus_blue_dragon_de", "package": null, "version": null}],
+    "sources": [],
+    "metrics": [],
+    "depends_on": {"macros": ["macro.finnauto.star_strings"], "nodes": ["model.finnauto.full_joined_npu_calc", "model.finnauto.il_fleet_all_cars_npus_blue_dragon_de"]}
+    },
+    "snapshot.project.in_snapshot_target_schema_fail": {
+    "database": "datawarehouse-304513",
+    "schema": "in_snapshot_target_schema_fail",
+    "name": "in_snapshot_target_schema_fail",
+    "resource_type": "snapshot",
+    "package_name": "finnauto",
+    "path": "npu_calc_snapshots/in_snapshot_target_schema_fail.sql",
+    "original_file_path": "sources/snapshots/in_snapshot_target_schema_fail.sql",
+    "unique_id": "snapshot.finnauto.in_snapshot_target_schema_fail",
+    "fqn": ["finnauto", "snapshots", "in_snapshot_target_schema_fail", "in_snapshot_target_schemain_snapshot_target_schema_fail"],
+    "alias": "in_snapshot_target_schema_fail",
+    "checksum": {"name": "sha256", "checksum": "6b60716708112b1f25b335e121410f02041d158a74954abcdef7c51235562a0f"},
+    "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "finn_car_id_subscription_id_row_num", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "strategy": "timestamp", "target_schema": "dwh_snapshots_prod", "target_database": null, "updated_at": "snapshot_run_timestamp", "check_cols": null, "invalidate_hard_deletes": true, "post-hook": [], "pre-hook": []},
+    "tags": [],
+    "description": "",
+    "columns": {},
+    "meta": {},
+    "group": null,
+    "docs": {"show": true, "node_color": null},
+    "patch_path": null,
+    "build_path": null,
+    "deferred": false,
+    "unrendered_config": {"target_schema": "dwh_snapshots_prod", "unique_key": "finn_car_id_subscription_id_row_num", "strategy": "timestamp", "updated_at": "snapshot_run_timestamp", "invalidate_hard_deletes": true},
+    "created_at": 1701188441.6212869,
+    "relation_name": "`datawarehouse-304513`.`dwh_snapshots_prod`.`in_snapshot_target_schema_fail`",
+    "raw_code": "\n\n{{\n    config(\n      unique_key='finn_car_id_subscription_id_row_num',\n      strategy='timestamp',\n      updated_at='snapshot_run_timestamp',\n      invalidate_hard_deletes=True\n    )\n}}\n\nWITH npu_calc AS (\n    SELECT\n        {{ star_strings(from=ref('full_joined_npu_calc')) }},\n        CAST(all_npus_de.npu_expected AS STRING) AS npu_expected,\n        CURRENT_TIMESTAMP() AS snapshot_run_timestampasdf,\n        CONCAT(finn_car_id, '-', IFNULL(subscription_id,\"-\")) AS finn_car_id_subscription_id,\n        ROW_NUMBER() OVER(PARTITION BY CONCAT(finn_car_id, '-', IFNULL(subscription_id,\"-\")) ORDER BY finn_car_id) AS row_number\n    FROM {{ ref('full_joined_npu_calc') }}\n    LEFT JOIN {{  ref('il_fleet_all_cars_npus_blue_dragon_de')  }} AS all_npus_de\n    USING (finn_car_id)\n)\n\nSELECT\n    * EXCEPT(row_number),\n    CONCAT(finn_car_id_subscription_id, '_', CAST(row_number AS STRING)) AS finn_car_id_subscription_id_row_num\nFROM npu_calc\n\n",
+    "language": "sql",
+    "refs": [{"name": "full_joined_npu_calc", "package": null, "version": null}, {"name": "full_joined_npu_calc", "package": null, "version": null}, {"name": "il_fleet_all_cars_npus_blue_dragon_de", "package": null, "version": null}],
+    "sources": [],
+    "metrics": [],
+    "depends_on": {"macros": ["macro.finnauto.star_strings"], "nodes": ["model.finnauto.full_joined_npu_calc", "model.finnauto.il_fleet_all_cars_npus_blue_dragon_de"]}
+    }
+  }
+}
diff --git a/tests/unit/test_check_snapshot_target_schema.py b/tests/unit/test_check_snapshot_target_schema.py
@@ -0,0 +1,28 @@
+import pytest
+
+from pre_commit_dbt.check_snapshot_has_target_schema import main
+
+
+def test_check_snapshot_target_schema():
+
+    result = main(
+        argv=[
+            "in_snapshot_target_schema.sql",
+            "--manifest",
+            "test-manifest.json",
+        ],
+    )
+    assert result == 0
+
+
+def test_check_snapshot_target_schema_fail():
+
+    result = main(
+        argv=[
+            "in_snapshot_target_schema_fail.sql",
+            "--manifest",
+            "test-manifest.json",
+        ],
+    )
+    assert result == 1
+