From ad88744a4827824e290d474f4af8e82d64288e5f Mon Sep 17 00:00:00 2001 From: Lore Date: Wed, 4 Sep 2024 09:55:03 -0400 Subject: [PATCH] fix(#156): unit test fixture not found when integrating with cht-sync (#158) * move fixture outside folder * add test-paths * add dbtignore file * fix patient data test * rename test folder to tests * fix sonar failed conditions --------- Co-authored-by: Maria Lorena Rodriguez Viruel --- .dbtignore | 2 ++ .github/workflows/test.yml | 2 +- dbt_project.yml | 2 +- {test => tests}/.gitkeep | 0 {test => tests}/.sonarcloud.properties | 0 {test => tests}/dbt/Dockerfile | 18 ++++++++---------- {test => tests}/docker-compose.yml | 2 +- .../contact_document_metadata_initial.csv | 0 .../contact/contact_initial_expected.csv | 0 .../contact/contact_source_table_initial.csv | 0 .../data_record_document_metadata_initial.csv | 0 .../data_record_initial_expected.csv | 0 .../data_record_source_table_initial.csv | 0 .../document_metadata_initial.csv | 0 .../document_metadata_initial_expected.csv | 0 .../patient/patient_initial_expected.csv | 0 .../patient/patient_person_initial.csv | 0 .../patient/patient_source_table_initial.csv | 0 .../fixtures/person/person_contact_initial.csv | 0 .../person/person_initial_expected.csv | 0 .../person/person_source_table_initial.csv | 0 .../fixtures/place/place_contact_initial.csv | 0 .../fixtures/place/place_initial_expected.csv | 0 .../place/place_source_table_initial.csv | 0 .../user/user_document_metadata_initial.csv | 0 .../fixtures/user/user_initial_expected.csv | 0 .../user/user_source_table_initial.csv | 0 {test => tests}/init.sql | 0 {test => tests}/profiles.yml | 0 {test => tests}/run_dbt_tests.sh | 0 {test => tests}/run_dbt_tests_docker.sh | 0 {test => tests}/sqltest/contact.sql | 0 {test => tests}/sqltest/data_record.sql | 0 {test => tests}/sqltest/patient.sql | 2 +- {test => tests}/sqltest/person.sql | 0 {test => tests}/sqltest/place.sql | 0 {test => tests}/sqltest/user.sql | 0 37 files changed, 14 insertions(+), 14 deletions(-) create mode 100644 .dbtignore rename {test => tests}/.gitkeep (100%) rename {test => tests}/.sonarcloud.properties (100%) rename {test => tests}/dbt/Dockerfile (53%) rename {test => tests}/docker-compose.yml (94%) rename {test => tests}/fixtures/contact/contact_document_metadata_initial.csv (100%) rename {test => tests}/fixtures/contact/contact_initial_expected.csv (100%) rename {test => tests}/fixtures/contact/contact_source_table_initial.csv (100%) rename {test => tests}/fixtures/data_record/data_record_document_metadata_initial.csv (100%) rename {test => tests}/fixtures/data_record/data_record_initial_expected.csv (100%) rename {test => tests}/fixtures/data_record/data_record_source_table_initial.csv (100%) rename {test => tests}/fixtures/document_metadata/document_metadata_initial.csv (100%) rename {test => tests}/fixtures/document_metadata/document_metadata_initial_expected.csv (100%) rename {test => tests}/fixtures/patient/patient_initial_expected.csv (100%) rename {test => tests}/fixtures/patient/patient_person_initial.csv (100%) rename {test => tests}/fixtures/patient/patient_source_table_initial.csv (100%) rename {test => tests}/fixtures/person/person_contact_initial.csv (100%) rename {test => tests}/fixtures/person/person_initial_expected.csv (100%) rename {test => tests}/fixtures/person/person_source_table_initial.csv (100%) rename {test => tests}/fixtures/place/place_contact_initial.csv (100%) rename {test => tests}/fixtures/place/place_initial_expected.csv (100%) rename {test => tests}/fixtures/place/place_source_table_initial.csv (100%) rename {test => tests}/fixtures/user/user_document_metadata_initial.csv (100%) rename {test => tests}/fixtures/user/user_initial_expected.csv (100%) rename {test => tests}/fixtures/user/user_source_table_initial.csv (100%) rename {test => tests}/init.sql (100%) rename {test => tests}/profiles.yml (100%) rename {test => tests}/run_dbt_tests.sh (100%) rename {test => tests}/run_dbt_tests_docker.sh (100%) rename {test => tests}/sqltest/contact.sql (100%) rename {test => tests}/sqltest/data_record.sql (100%) rename {test => tests}/sqltest/patient.sql (85%) rename {test => tests}/sqltest/person.sql (100%) rename {test => tests}/sqltest/place.sql (100%) rename {test => tests}/sqltest/user.sql (100%) diff --git a/.dbtignore b/.dbtignore new file mode 100644 index 00000000..212cd2c0 --- /dev/null +++ b/.dbtignore @@ -0,0 +1,2 @@ +# .dbtignore +init.sql diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 92337ddf..82dcd46c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,4 +19,4 @@ jobs: if: steps.get-docker-hub-username.outputs.dockerhub_username - name: Run dbt and tests run: ./run_dbt_tests.sh - working-directory: ./test + working-directory: ./tests diff --git a/dbt_project.yml b/dbt_project.yml index 249bf171..b2ecd57d 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -5,5 +5,5 @@ config-version: 2 # This setting configures which "profile" dbt uses for this project. profile: 'default' -on-run-end: +on-run-end: - "{{ log_dbt_results(results) }}" diff --git a/test/.gitkeep b/tests/.gitkeep similarity index 100% rename from test/.gitkeep rename to tests/.gitkeep diff --git a/test/.sonarcloud.properties b/tests/.sonarcloud.properties similarity index 100% rename from test/.sonarcloud.properties rename to tests/.sonarcloud.properties diff --git a/test/dbt/Dockerfile b/tests/dbt/Dockerfile similarity index 53% rename from test/dbt/Dockerfile rename to tests/dbt/Dockerfile index e0549922..17c3f6cb 100644 --- a/test/dbt/Dockerfile +++ b/tests/dbt/Dockerfile @@ -1,26 +1,24 @@ FROM python:3 AS testing -RUN pip install flake8 - -RUN flake8 dbt-run.py +RUN pip install flake8 \ + && flake8 dbt-run.py FROM python:3 AS release RUN pip install --upgrade cffi \ && pip install cryptography~=3.4 \ - && pip install dbt-core dbt-postgres - -RUN adduser dbt + && pip install dbt-core dbt-postgres \ + && adduser dbt USER dbt WORKDIR /dbt/ -COPY test/profiles.yml profiles.yml +COPY tests/profiles.yml profiles.yml COPY dbt_project.yml dbt_project.yml COPY packages.yml packages.yml COPY models models COPY macros macros -COPY test/fixtures tests/fixtures -COPY test/sqltest tests -COPY test/run_dbt_tests_docker.sh run_dbt_tests_docker.sh +COPY tests/fixtures tests/fixtures +COPY tests/sqltest tests +COPY tests/run_dbt_tests_docker.sh run_dbt_tests_docker.sh CMD ["bash", "./run_dbt_tests_docker.sh"] diff --git a/test/docker-compose.yml b/tests/docker-compose.yml similarity index 94% rename from test/docker-compose.yml rename to tests/docker-compose.yml index dd4b3644..077c2137 100644 --- a/test/docker-compose.yml +++ b/tests/docker-compose.yml @@ -15,7 +15,7 @@ services: dbt: build: context: ../ - dockerfile: ./test/dbt/Dockerfile + dockerfile: ./tests/dbt/Dockerfile environment: - POSTGRES_HOST=postgres - POSTGRES_USER=pipelinetest diff --git a/test/fixtures/contact/contact_document_metadata_initial.csv b/tests/fixtures/contact/contact_document_metadata_initial.csv similarity index 100% rename from test/fixtures/contact/contact_document_metadata_initial.csv rename to tests/fixtures/contact/contact_document_metadata_initial.csv diff --git a/test/fixtures/contact/contact_initial_expected.csv b/tests/fixtures/contact/contact_initial_expected.csv similarity index 100% rename from test/fixtures/contact/contact_initial_expected.csv rename to tests/fixtures/contact/contact_initial_expected.csv diff --git a/test/fixtures/contact/contact_source_table_initial.csv b/tests/fixtures/contact/contact_source_table_initial.csv similarity index 100% rename from test/fixtures/contact/contact_source_table_initial.csv rename to tests/fixtures/contact/contact_source_table_initial.csv diff --git a/test/fixtures/data_record/data_record_document_metadata_initial.csv b/tests/fixtures/data_record/data_record_document_metadata_initial.csv similarity index 100% rename from test/fixtures/data_record/data_record_document_metadata_initial.csv rename to tests/fixtures/data_record/data_record_document_metadata_initial.csv diff --git a/test/fixtures/data_record/data_record_initial_expected.csv b/tests/fixtures/data_record/data_record_initial_expected.csv similarity index 100% rename from test/fixtures/data_record/data_record_initial_expected.csv rename to tests/fixtures/data_record/data_record_initial_expected.csv diff --git a/test/fixtures/data_record/data_record_source_table_initial.csv b/tests/fixtures/data_record/data_record_source_table_initial.csv similarity index 100% rename from test/fixtures/data_record/data_record_source_table_initial.csv rename to tests/fixtures/data_record/data_record_source_table_initial.csv diff --git a/test/fixtures/document_metadata/document_metadata_initial.csv b/tests/fixtures/document_metadata/document_metadata_initial.csv similarity index 100% rename from test/fixtures/document_metadata/document_metadata_initial.csv rename to tests/fixtures/document_metadata/document_metadata_initial.csv diff --git a/test/fixtures/document_metadata/document_metadata_initial_expected.csv b/tests/fixtures/document_metadata/document_metadata_initial_expected.csv similarity index 100% rename from test/fixtures/document_metadata/document_metadata_initial_expected.csv rename to tests/fixtures/document_metadata/document_metadata_initial_expected.csv diff --git a/test/fixtures/patient/patient_initial_expected.csv b/tests/fixtures/patient/patient_initial_expected.csv similarity index 100% rename from test/fixtures/patient/patient_initial_expected.csv rename to tests/fixtures/patient/patient_initial_expected.csv diff --git a/test/fixtures/patient/patient_person_initial.csv b/tests/fixtures/patient/patient_person_initial.csv similarity index 100% rename from test/fixtures/patient/patient_person_initial.csv rename to tests/fixtures/patient/patient_person_initial.csv diff --git a/test/fixtures/patient/patient_source_table_initial.csv b/tests/fixtures/patient/patient_source_table_initial.csv similarity index 100% rename from test/fixtures/patient/patient_source_table_initial.csv rename to tests/fixtures/patient/patient_source_table_initial.csv diff --git a/test/fixtures/person/person_contact_initial.csv b/tests/fixtures/person/person_contact_initial.csv similarity index 100% rename from test/fixtures/person/person_contact_initial.csv rename to tests/fixtures/person/person_contact_initial.csv diff --git a/test/fixtures/person/person_initial_expected.csv b/tests/fixtures/person/person_initial_expected.csv similarity index 100% rename from test/fixtures/person/person_initial_expected.csv rename to tests/fixtures/person/person_initial_expected.csv diff --git a/test/fixtures/person/person_source_table_initial.csv b/tests/fixtures/person/person_source_table_initial.csv similarity index 100% rename from test/fixtures/person/person_source_table_initial.csv rename to tests/fixtures/person/person_source_table_initial.csv diff --git a/test/fixtures/place/place_contact_initial.csv b/tests/fixtures/place/place_contact_initial.csv similarity index 100% rename from test/fixtures/place/place_contact_initial.csv rename to tests/fixtures/place/place_contact_initial.csv diff --git a/test/fixtures/place/place_initial_expected.csv b/tests/fixtures/place/place_initial_expected.csv similarity index 100% rename from test/fixtures/place/place_initial_expected.csv rename to tests/fixtures/place/place_initial_expected.csv diff --git a/test/fixtures/place/place_source_table_initial.csv b/tests/fixtures/place/place_source_table_initial.csv similarity index 100% rename from test/fixtures/place/place_source_table_initial.csv rename to tests/fixtures/place/place_source_table_initial.csv diff --git a/test/fixtures/user/user_document_metadata_initial.csv b/tests/fixtures/user/user_document_metadata_initial.csv similarity index 100% rename from test/fixtures/user/user_document_metadata_initial.csv rename to tests/fixtures/user/user_document_metadata_initial.csv diff --git a/test/fixtures/user/user_initial_expected.csv b/tests/fixtures/user/user_initial_expected.csv similarity index 100% rename from test/fixtures/user/user_initial_expected.csv rename to tests/fixtures/user/user_initial_expected.csv diff --git a/test/fixtures/user/user_source_table_initial.csv b/tests/fixtures/user/user_source_table_initial.csv similarity index 100% rename from test/fixtures/user/user_source_table_initial.csv rename to tests/fixtures/user/user_source_table_initial.csv diff --git a/test/init.sql b/tests/init.sql similarity index 100% rename from test/init.sql rename to tests/init.sql diff --git a/test/profiles.yml b/tests/profiles.yml similarity index 100% rename from test/profiles.yml rename to tests/profiles.yml diff --git a/test/run_dbt_tests.sh b/tests/run_dbt_tests.sh similarity index 100% rename from test/run_dbt_tests.sh rename to tests/run_dbt_tests.sh diff --git a/test/run_dbt_tests_docker.sh b/tests/run_dbt_tests_docker.sh similarity index 100% rename from test/run_dbt_tests_docker.sh rename to tests/run_dbt_tests_docker.sh diff --git a/test/sqltest/contact.sql b/tests/sqltest/contact.sql similarity index 100% rename from test/sqltest/contact.sql rename to tests/sqltest/contact.sql diff --git a/test/sqltest/data_record.sql b/tests/sqltest/data_record.sql similarity index 100% rename from test/sqltest/data_record.sql rename to tests/sqltest/data_record.sql diff --git a/test/sqltest/patient.sql b/tests/sqltest/patient.sql similarity index 85% rename from test/sqltest/patient.sql rename to tests/sqltest/patient.sql index d83ac460..b803905c 100644 --- a/test/sqltest/patient.sql +++ b/tests/sqltest/patient.sql @@ -1,5 +1,5 @@ SELECT -FROM v1.couchdb couchdb +FROM {{ source('couchdb', env_var('POSTGRES_TABLE')) }} couchdb LEFT JOIN {{ ref('patient') }} patient ON couchdb._id = patient.uuid WHERE couchdb._deleted = false diff --git a/test/sqltest/person.sql b/tests/sqltest/person.sql similarity index 100% rename from test/sqltest/person.sql rename to tests/sqltest/person.sql diff --git a/test/sqltest/place.sql b/tests/sqltest/place.sql similarity index 100% rename from test/sqltest/place.sql rename to tests/sqltest/place.sql diff --git a/test/sqltest/user.sql b/tests/sqltest/user.sql similarity index 100% rename from test/sqltest/user.sql rename to tests/sqltest/user.sql