diff --git a/pre_commit_dbt/check_snapshot_has_target_schema.py b/pre_commit_dbt/check_snapshot_has_target_schema.py index 9721bb3e..adb5ee1b 100644 --- a/pre_commit_dbt/check_snapshot_has_target_schema.py +++ b/pre_commit_dbt/check_snapshot_has_target_schema.py @@ -9,7 +9,7 @@ from pre_commit_dbt.utils import get_json from pre_commit_dbt.utils import get_missing_file_paths from pre_commit_dbt.utils import get_model_sqls -from pre_commit_dbt.utils import get_models +from pre_commit_dbt.utils import get_snapshots from pre_commit_dbt.utils import JsonOpenError @@ -23,7 +23,8 @@ def validate_tags( filenames = set(sqls.keys()) # get manifest nodes that pre-commit found as changed - models = get_models(manifest, filenames) + models = get_snapshots(manifest, filenames) + for model in models: # tags can be specified only from manifest raw_code = model.node.get("raw_code", []) diff --git a/pre_commit_dbt/utils.py b/pre_commit_dbt/utils.py index 799cfc61..0bc6c38f 100644 --- a/pre_commit_dbt/utils.py +++ b/pre_commit_dbt/utils.py @@ -144,6 +144,18 @@ def get_models( yield Model(key, node.get("name"), filename, node) # pragma: no mutate +def get_snapshots( + manifest: Dict[str, Any], + filenames: Set[str] +) -> Generator[Model, None, None]: + nodes = manifest.get("nodes", {}) + for key, node in nodes.items(): + split_key = key.split(".") + filename = split_key[-1] + if filename in filenames and split_key[0] == "snapshot": + yield Model(key, node.get("name"), filename, node) # pragma: no mutate + + def get_ephemeral( manifest: Dict[str, Any], ) -> List[str]: diff --git a/tests/unit/test-manifest.json b/tests/unit/test-manifest.json new file mode 100644 index 00000000..717b0888 --- /dev/null +++ b/tests/unit/test-manifest.json @@ -0,0 +1,77 @@ +{"metadata": +{"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v10.json", + "dbt_version": "1.6.0", + "generated_at": "2023-11-28T16:20:41.115139Z", + "invocation_id": "7b7b0778-6eab-40ba-b520-3004a18639f1", + "env": {}, + "project_name": "finnauto", + "project_id": "7d1e44fc8134dfc5d0dffea3bd5bdf39", + "user_id": "946185e7-9473-4754-ab18-8d8995b68c96", + "send_anonymous_usage_stats": true, + "adapter_type": "bigquery"}, + "nodes": {"snapshot.project.in_snapshot_target_schema": { + "database": "datawarehouse-304513", + "schema": "in_snapshot_target_schema", + "name": "in_snapshot_target_schema", + "resource_type": "snapshot", + "package_name": "finnauto", + "path": "npu_calc_snapshots/in_snapshot_target_schema.sql", + "original_file_path": "sources/snapshots/in_snapshot_target_schema.sql", + "unique_id": "snapshot.finnauto.in_snapshot_target_schema", + "fqn": ["finnauto", "snapshots", "in_snapshot_target_schema", "in_snapshot_target_schema"], + "alias": "in_snapshot_target_schema", + "checksum": {"name": "sha256", "checksum": "6b60716708112b1f25b335e121410f02041d158a74954abcdef7c51235562a0f"}, + "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "finn_car_id_subscription_id_row_num", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "strategy": "timestamp", "target_schema": "dwh_snapshots_prod", "target_database": null, "updated_at": "snapshot_run_timestamp", "check_cols": null, "invalidate_hard_deletes": true, "post-hook": [], "pre-hook": []}, + "tags": [], + "description": "", + "columns": {}, + "meta": {}, + "group": null, + "docs": {"show": true, "node_color": null}, + "patch_path": null, + "build_path": null, + "deferred": false, + "unrendered_config": {"target_schema": "dwh_snapshots_prod", "unique_key": "finn_car_id_subscription_id_row_num", "strategy": "timestamp", "updated_at": "snapshot_run_timestamp", "invalidate_hard_deletes": true}, + "created_at": 1701188441.6212869, + "relation_name": "`datawarehouse-304513`.`dwh_snapshots_prod`.`in_snapshot_target_schema`", + "raw_code": "\n\n{{\n config(\n target_schema=generate_schema_name('dwh_snapshots_prod'),\n unique_key='finn_car_id_subscription_id_row_num',\n strategy='timestamp',\n updated_at='snapshot_run_timestamp',\n invalidate_hard_deletes=True\n )\n}}\n\nWITH npu_calc AS (\n SELECT\n {{ star_strings(from=ref('full_joined_npu_calc')) }},\n CAST(all_npus_de.npu_expected AS STRING) AS npu_expected,\n CURRENT_TIMESTAMP() AS snapshot_run_timestampasdf,\n CONCAT(finn_car_id, '-', IFNULL(subscription_id,\"-\")) AS finn_car_id_subscription_id,\n ROW_NUMBER() OVER(PARTITION BY CONCAT(finn_car_id, '-', IFNULL(subscription_id,\"-\")) ORDER BY finn_car_id) AS row_number\n FROM {{ ref('full_joined_npu_calc') }}\n LEFT JOIN {{ ref('il_fleet_all_cars_npus_blue_dragon_de') }} AS all_npus_de\n USING (finn_car_id)\n)\n\nSELECT\n * EXCEPT(row_number),\n CONCAT(finn_car_id_subscription_id, '_', CAST(row_number AS STRING)) AS finn_car_id_subscription_id_row_num\nFROM npu_calc\n\n", + "language": "sql", + "refs": [{"name": "full_joined_npu_calc", "package": null, "version": null}, {"name": "full_joined_npu_calc", "package": null, "version": null}, {"name": "il_fleet_all_cars_npus_blue_dragon_de", "package": null, "version": null}], + "sources": [], + "metrics": [], + "depends_on": {"macros": ["macro.finnauto.star_strings"], "nodes": ["model.finnauto.full_joined_npu_calc", "model.finnauto.il_fleet_all_cars_npus_blue_dragon_de"]} + }, + "snapshot.project.in_snapshot_target_schema_fail": { + "database": "datawarehouse-304513", + "schema": "in_snapshot_target_schema_fail", + "name": "in_snapshot_target_schema_fail", + "resource_type": "snapshot", + "package_name": "finnauto", + "path": "npu_calc_snapshots/in_snapshot_target_schema_fail.sql", + "original_file_path": "sources/snapshots/in_snapshot_target_schema_fail.sql", + "unique_id": "snapshot.finnauto.in_snapshot_target_schema_fail", + "fqn": ["finnauto", "snapshots", "in_snapshot_target_schema_fail", "in_snapshot_target_schemain_snapshot_target_schema_fail"], + "alias": "in_snapshot_target_schema_fail", + "checksum": {"name": "sha256", "checksum": "6b60716708112b1f25b335e121410f02041d158a74954abcdef7c51235562a0f"}, + "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "finn_car_id_subscription_id_row_num", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "strategy": "timestamp", "target_schema": "dwh_snapshots_prod", "target_database": null, "updated_at": "snapshot_run_timestamp", "check_cols": null, "invalidate_hard_deletes": true, "post-hook": [], "pre-hook": []}, + "tags": [], + "description": "", + "columns": {}, + "meta": {}, + "group": null, + "docs": {"show": true, "node_color": null}, + "patch_path": null, + "build_path": null, + "deferred": false, + "unrendered_config": {"target_schema": "dwh_snapshots_prod", "unique_key": "finn_car_id_subscription_id_row_num", "strategy": "timestamp", "updated_at": "snapshot_run_timestamp", "invalidate_hard_deletes": true}, + "created_at": 1701188441.6212869, + "relation_name": "`datawarehouse-304513`.`dwh_snapshots_prod`.`in_snapshot_target_schema_fail`", + "raw_code": "\n\n{{\n config(\n unique_key='finn_car_id_subscription_id_row_num',\n strategy='timestamp',\n updated_at='snapshot_run_timestamp',\n invalidate_hard_deletes=True\n )\n}}\n\nWITH npu_calc AS (\n SELECT\n {{ star_strings(from=ref('full_joined_npu_calc')) }},\n CAST(all_npus_de.npu_expected AS STRING) AS npu_expected,\n CURRENT_TIMESTAMP() AS snapshot_run_timestampasdf,\n CONCAT(finn_car_id, '-', IFNULL(subscription_id,\"-\")) AS finn_car_id_subscription_id,\n ROW_NUMBER() OVER(PARTITION BY CONCAT(finn_car_id, '-', IFNULL(subscription_id,\"-\")) ORDER BY finn_car_id) AS row_number\n FROM {{ ref('full_joined_npu_calc') }}\n LEFT JOIN {{ ref('il_fleet_all_cars_npus_blue_dragon_de') }} AS all_npus_de\n USING (finn_car_id)\n)\n\nSELECT\n * EXCEPT(row_number),\n CONCAT(finn_car_id_subscription_id, '_', CAST(row_number AS STRING)) AS finn_car_id_subscription_id_row_num\nFROM npu_calc\n\n", + "language": "sql", + "refs": [{"name": "full_joined_npu_calc", "package": null, "version": null}, {"name": "full_joined_npu_calc", "package": null, "version": null}, {"name": "il_fleet_all_cars_npus_blue_dragon_de", "package": null, "version": null}], + "sources": [], + "metrics": [], + "depends_on": {"macros": ["macro.finnauto.star_strings"], "nodes": ["model.finnauto.full_joined_npu_calc", "model.finnauto.il_fleet_all_cars_npus_blue_dragon_de"]} + } + } +} \ No newline at end of file diff --git a/tests/unit/test_check_snapshot_target_schema.py b/tests/unit/test_check_snapshot_target_schema.py new file mode 100644 index 00000000..e19a9c9a --- /dev/null +++ b/tests/unit/test_check_snapshot_target_schema.py @@ -0,0 +1,28 @@ +import pytest + +from pre_commit_dbt.check_snapshot_has_target_schema import main + + +def test_check_snapshot_target_schema(): + + result = main( + argv=[ + "in_snapshot_target_schema.sql", + "--manifest", + "test-manifest.json", + ], + ) + assert result == 0 + + +def test_check_snapshot_target_schema_fail(): + + result = main( + argv=[ + "in_snapshot_target_schema_fail.sql", + "--manifest", + "test-manifest.json", + ], + ) + assert result == 1 +