diff --git a/models/contacts/contacts.yml b/models/contacts/contacts.yml index 14414305..5de80d54 100644 --- a/models/contacts/contacts.yml +++ b/models/contacts/contacts.yml @@ -12,7 +12,7 @@ models: - type: foreign_key expression: "{{ env_var('POSTGRES_SCHEMA') }}.document_metadata (uuid) ON DELETE CASCADE" - type: unique - tests: + data_tests: - not_null - relationships: to: ref('document_metadata') @@ -51,7 +51,7 @@ models: constraints: - type: foreign_key expression: "{{ env_var('POSTGRES_SCHEMA') }}.contact (uuid) ON DELETE CASCADE" - tests: + data_tests: - not_null - relationships: to: ref('contact') @@ -72,7 +72,7 @@ models: constraints: - type: foreign_key expression: "{{ env_var('POSTGRES_SCHEMA') }}.contact (uuid) ON DELETE CASCADE" - tests: + data_tests: - not_null - relationships: to: ref('contact') @@ -91,7 +91,7 @@ models: constraints: - type: foreign_key expression: "{{ env_var('POSTGRES_SCHEMA') }}.contact (uuid) ON DELETE CASCADE" - tests: + data_tests: - not_null - relationships: to: ref('contact') diff --git a/models/forms/forms.yml b/models/forms/forms.yml index 94c7d427..cbf125df 100644 --- a/models/forms/forms.yml +++ b/models/forms/forms.yml @@ -12,7 +12,7 @@ models: - type: foreign_key expression: "{{ env_var('POSTGRES_SCHEMA') }}.document_metadata (uuid) ON DELETE CASCADE" - type: unique - tests: + data_tests: - not_null - relationships: to: ref('document_metadata') diff --git a/models/root/document_metadata.sql b/models/root/document_metadata.sql index 231e11c1..4343bd67 100644 --- a/models/root/document_metadata.sql +++ b/models/root/document_metadata.sql @@ -1,3 +1,5 @@ +{% set import_couchdb_data = select_table("{{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }}", ref('test_source_table')) %} + {{ config( materialized = 'incremental', @@ -18,7 +20,7 @@ SELECT _deleted, saved_timestamp, doc->>'type' as doc_type -FROM {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} source_table +FROM {{ import_couchdb_data }} {% if is_incremental() %} WHERE source_table.saved_timestamp >= {{ max_existing_timestamp('saved_timestamp') }} {% endif %} diff --git a/models/root/root.yml b/models/root/root.yml index 9f10a33b..470b5766 100644 --- a/models/root/root.yml +++ b/models/root/root.yml @@ -16,8 +16,6 @@ models: data_type: string constraints: - type: unique - tests: - - not_null - name: saved_timestamp data_type: timestamp - name: _deleted diff --git a/models/root/tests/document_metadata.yml b/models/root/tests/document_metadata.yml new file mode 100644 index 00000000..bc0d5924 --- /dev/null +++ b/models/root/tests/document_metadata.yml @@ -0,0 +1,16 @@ +unit_tests: + - name: test_document_metadata + description: | + This unit test validates the transformation logic in the `document_metadata` model + for the first test scenario, ensuring that the input data from `source_table` is correctly transformed. + model: document_metadata + overrides: + macros: + is_incremental: false + given: + - input: ref('test_source_table') + format: csv + fixture: contacts_test_source_table_input + expect: + format: csv + fixture: contacts_document_metadata_input diff --git a/models/root/tests/test_source_table.sql b/models/root/tests/test_source_table.sql new file mode 100644 index 00000000..1d3f0140 --- /dev/null +++ b/models/root/tests/test_source_table.sql @@ -0,0 +1,7 @@ +{{ config(materialized='table') }} + +select + null::timestamp as saved_timestamp, + null::text as _id, + null::boolean as _deleted, + null::jsonb as doc diff --git a/models/users/user.yml b/models/users/user.yml index dee78de8..78f3bd5f 100644 --- a/models/users/user.yml +++ b/models/users/user.yml @@ -12,7 +12,7 @@ models: - type: unique - type: foreign_key expression: "{{ env_var('POSTGRES_SCHEMA') }}.document_metadata (uuid) ON DELETE CASCADE" - tests: + data_tests: - not_null - relationships: to: ref('contact') diff --git a/test/dbt/Dockerfile b/test/dbt/Dockerfile index f856573e..e0549922 100644 --- a/test/dbt/Dockerfile +++ b/test/dbt/Dockerfile @@ -7,17 +7,19 @@ RUN flake8 dbt-run.py FROM python:3 AS release RUN pip install --upgrade cffi \ - && pip install cryptography~=3.4 \ - && pip install dbt-core dbt-postgres + && pip install cryptography~=3.4 \ + && pip install dbt-core dbt-postgres RUN adduser dbt USER dbt WORKDIR /dbt/ -COPY test/profiles.yml profiles.yml +COPY test/profiles.yml profiles.yml COPY dbt_project.yml dbt_project.yml +COPY packages.yml packages.yml COPY models models COPY macros macros +COPY test/fixtures tests/fixtures COPY test/sqltest tests COPY test/run_dbt_tests_docker.sh run_dbt_tests_docker.sh diff --git a/test/fixtures/contacts_document_metadata_input.csv b/test/fixtures/contacts_document_metadata_input.csv new file mode 100644 index 00000000..34346840 --- /dev/null +++ b/test/fixtures/contacts_document_metadata_input.csv @@ -0,0 +1,5 @@ +uuid,_deleted,saved_timestamp,doc_type +"1",false,"2024-08-01 00:00:00.000","person" +"2",false,"2024-08-01 00:00:00.000","person" +"3",false,"2024-08-02 00:00:00.000","clinic" +"4",true,"2024-08-02 00:00:00.000","person" diff --git a/test/fixtures/contacts_expect.csv b/test/fixtures/contacts_expect.csv new file mode 100644 index 00000000..7c8b346a --- /dev/null +++ b/test/fixtures/contacts_expect.csv @@ -0,0 +1,3 @@ +uuid,name,phone,phone2,active,notes,contact_id,muted,parent_uuid,reported +"1","John Doe","1234567890","0987654321",true,"Test note","C123",false,"parent1","2021-08-01 00:00:00" +"2","Jane Doe","1234567890","0987654321",false,"Another note","C124",false,"parent2","2021-08-01 00:00:00" diff --git a/test/fixtures/contacts_test_source_table_input.csv b/test/fixtures/contacts_test_source_table_input.csv new file mode 100644 index 00000000..5b4dc464 --- /dev/null +++ b/test/fixtures/contacts_test_source_table_input.csv @@ -0,0 +1,5 @@ +saved_timestamp,_id,_deleted,doc +"2024-08-01 00:00:00.000","1",false,"{""type"": ""person"", ""name"": ""John Doe"", ""phone"": ""1234567890"", ""alternative_phone"": ""0987654321"", ""is_active"": ""true"", ""notes"": ""Test note"", ""contact_id"": ""C123"", ""muted"": ""false"", ""parent"": {""_id"": ""parent1""}, ""reported_date"": 1627804800000}" +"2024-08-01 00:00:00.000","2",false,"{""type"": ""person"", ""name"": ""Jane Doe"", ""phone"": ""1234567890"", ""alternative_phone"": ""0987654321"", ""is_active"": ""false"", ""notes"": ""Another note"", ""contact_id"": ""C124"", ""muted"": """", ""parent"": {""_id"": ""parent2""}, ""reported_date"": 1627804800000}" +"2024-08-02 00:00:00.000","3",false,"{""type"": ""clinic"", ""name"": ""Test Clinic"", ""phone"": ""1122334455"", ""alternative_phone"": ""5544332211"", ""is_active"": ""true"", ""notes"": ""Clinic note"", ""contact_id"": ""C125"", ""muted"": ""true"", ""parent"": {""_id"": ""parent3""}, ""reported_date"": 1627804800000}" +"2024-08-02 00:00:00.000","4",true,"{""type"": ""person"", ""name"": ""Deleted Contact"", ""phone"": ""0000000000"", ""alternative_phone"": ""1111111111"", ""is_active"": ""false"", ""notes"": ""This record should be deleted"", ""contact_id"": ""C126"", ""muted"": ""false"", ""parent"": {""_id"": ""parent4""}, ""reported_date"": 1627804800000}"