From 194b3d5eae806782b561329fbeedc29eb9d532fd Mon Sep 17 00:00:00 2001 From: Lore Date: Wed, 1 Nov 2023 09:25:33 -0300 Subject: [PATCH] chore(#27): Add dbt unit test brac (#28) * Add tests to contact views working models * Add tests to use views working models * feat: make model chnages and temporarily remove dashboard models * feat: fix failing models * Merge conflicts * feat: fix failing models * Add users.yml to gitignore * Add tests for contact_views models * Add tests to useviews models * Fix some test * Add tests for main models * Add relationships and accepted_values tests * Add relationships tests on contact views models * Add relationships tests to main models * Fix Did not find matching node for patch error * Fix accepted values errors * Fix tests * Fixing failing tests * Delete some failing tests * First views model test * Add useviews model tests * Change the way columns are referenced in couchdb * Trying to see if the problem are the seeds * Add test to pncview_actual_enrrollments model * Add test for form_metadata model * Add test for impactview_month * Add test to formview_inmunization_follow_up model * Add test for formview_assessment_follow_up model * Add test for formview_muac_follow_up * Add test on couples of views models * Fix views.yml duplicate * Add more test on views models * Add more test to views models * Delete failed tests * Add first test for impact anc models * Delete exclude command from run dbt * Revert deletion of failing tests * Reverse change * reverse deletions * Fix brac name * Finish test for views models * Move and rename impact test file. Implement impact tests * Delete commented lines --------- Co-authored-by: Maria Lorena Rodriguez Viruel Co-authored-by: Njuguna Ndung'u --- .gitignore | 4 + dbt_project.yml | 35 +-- models/brac/brac.yml | 74 +++++ models/brac/contact_views/contact_views.yml | 143 ++++++++++ models/brac/contact_views/contactview_chp.sql | 3 +- models/brac/contact_views/contactview_chw.sql | 3 +- .../contactview_person_fields.sql | 3 +- .../impact/ANC/impact_ancview_danger_sign.sql | 2 +- .../impact/ANC/impact_ancview_delivery.sql | 2 +- .../impact/ANC/impact_ancview_pregnancy.sql | 33 --- .../ANC/impact_ancview_pregnancy_visit.sql | 2 +- .../PNC/get_dashboard_data_pnc_impact.sql | 68 ----- .../PNC/impact_pncview_actual_enrollments.sql | 2 +- .../impact/PNC/impact_pncview_danger_sign.sql | 2 +- .../impact_pncview_expected_enrollments.sql | 2 +- .../brac/impact/PNC/impact_pncview_visit.sql | 2 +- .../hhvisit/get_dashboard_data_hh_impact.sql | 148 ---------- models/brac/impact/impact.yml | 128 +++++++++ models/brac/indexes.sql | 9 - .../useviews/useview_assessment_follow_up.sql | 4 +- .../brac/useviews/useview_postnatal_care.sql | 2 +- models/brac/useviews/useviews.yml | 143 ++++++++++ models/brac/views/views.yml | 269 ++++++++++++++++++ models/contact_views/contact_views.yml | 1 + models/user_tables/chws.sql | 21 -- models/user_tables/user_tables.yml | 9 - 26 files changed, 793 insertions(+), 321 deletions(-) create mode 100644 models/brac/brac.yml create mode 100644 models/brac/contact_views/contact_views.yml delete mode 100644 models/brac/impact/ANC/impact_ancview_pregnancy.sql delete mode 100644 models/brac/impact/PNC/get_dashboard_data_pnc_impact.sql delete mode 100644 models/brac/impact/hhvisit/get_dashboard_data_hh_impact.sql create mode 100644 models/brac/impact/impact.yml delete mode 100644 models/brac/indexes.sql create mode 100644 models/brac/useviews/useviews.yml create mode 100644 models/brac/views/views.yml delete mode 100644 models/user_tables/chws.sql delete mode 100644 models/user_tables/user_tables.yml diff --git a/.gitignore b/.gitignore index 6567ff4a..25c43f9f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ dbt_packages/ logs/ */.env */dbt_packages/ +tests/.user.yml +dbt-env/* +.user.yml +dbt-env/* diff --git a/dbt_project.yml b/dbt_project.yml index a3c72e0c..af47e092 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,34 +1,35 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: "pipeline" -version: "0.0.1" +name: 'pipeline' +version: '0.0.1' config-version: 2 +vars: + # The `start_date` variable will be accessible in all resources + end_date: '2023-06-01' + # This setting configures which "profile" dbt uses for this project. -profile: "default" +profile: 'default' # These configurations specify where dbt should look for different types of files. # The `source-paths` config, for example, states that models in this project can be # found in the "models/" directory. You probably won't need to change these! -model-paths: ["models"] -analysis-paths: ["analysis"] -test-paths: ["tests"] -seed-paths: ["seeds"] -macro-paths: ["macros"] -snapshot-paths: ["snapshots"] +model-paths: ['models'] +analysis-paths: ['analysis'] +test-paths: ['tests'] +seed-paths: ['seeds'] +macro-paths: ['macros'] +snapshot-paths: ['snapshots'] -target-path: "target" # directory which will store compiled SQL files +target-path: 'target' # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - - "target" - - "dbt_modules" + - 'target' + - 'dbt_modules' -# Configuring models -# Full documentation: https://docs.getdbt.com/docs/configuring-models +vars: + end_date: "2020-12-12" -# In this example config, we tell dbt to build all models in the example/ directory -# as tables. These settings can be overridden in the individual model files -# using the `{{ config(...) }}` macro. models: type_partitions: root: diff --git a/models/brac/brac.yml b/models/brac/brac.yml new file mode 100644 index 00000000..d36cb3ba --- /dev/null +++ b/models/brac/brac.yml @@ -0,0 +1,74 @@ +version: 2 + +models: + - name: brac_hmis + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('contactview_chp') + field: uuid + - name: households_registered + tests: + - not_null + - name: num_of_hh_visited + tests: + - not_null + - name: num_of_hh_bed_nets + tests: + - not_null + - name: num_of_hh_latrines + tests: + - not_null + - name: num_of_hh_safe_water + tests: + - not_null + - name: num_of_comm_members + tests: + - not_null + - name: num_of_health_forum + tests: + - not_null + - name: num_pregnant_care + tests: + - not_null + - name: num_pregnant_identified + tests: + - not_null + - name: num_facility_deliveries + tests: + - not_null + - name: num_home_deliveries + tests: + - not_null + - name: brac-kpis + columns: + - name: chp_uuid + tests: + - not_null + - relationships: + to: ref('contactview_chp') + field: uuid + - name: count_all_assessments + tests: + - not_null + - name: u5_malaria_treatment + tests: + - not_null + - name: u5_diarrhea_treatment + tests: + - not_null + - name: u5_pneumonia_treatment + tests: + - not_null + - name: u1_malaria_treatment + tests: + - not_null + - name: u1_diarrhea_treatment + tests: + - not_null + - name: u1_pneumonia_treatment + tests: + - not_null diff --git a/models/brac/contact_views/contact_views.yml b/models/brac/contact_views/contact_views.yml new file mode 100644 index 00000000..e6d76556 --- /dev/null +++ b/models/brac/contact_views/contact_views.yml @@ -0,0 +1,143 @@ +version: 2 + +models: + - name: contactview_person + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: name + tests: + - not_null + - name: patient_id + tests: + - not_null + - name: contactview_person_fields + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('contactview_metadata') + field: uuid + tests: + - dbt_utils.fewer_rows_than: + compare_model: ref('contactview_metadata') + - name: contactview_chw + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('contactview_person_fields') + field: uuid + tests: + - dbt_utils.fewer_rows_than: + compare_model: ref('contactview_metadata') + - dbt_utils.fewer_rows_than: + compare_model: ref('contactview_person_fields') + - name: contactview_chp + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('contactview_chw') + field: uuid + - name: branch_uuid + tests: + - not_null + - name: supervisor_uuid + tests: + - not_null + tests: + - dbt_utils.fewer_rows_than: + compare_model: ref('contactview_metadata') + - dbt_utils.fewer_rows_than: + compare_model: ref('contactview_person_fields') + - name: contactview_muted + columns: + - name: contact_uuid + tests: + - not_null + - unique + - relationships: + to: ref('contactview_metadata') + field: uuid + - name: type + tests: + - not_null + - accepted_values: + values: + [contact, clinic, district_hospital, health_center, person] + - name: mute_status + tests: + - not_null + - name: date + tests: + - not_null + tests: + - dbt_utils.fewer_rows_than: + compare_model: ref('contactview_metadata') + - name: contactview_hospital + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('contactview_metadata') + field: uuid + tests: + - dbt_utils.fewer_rows_than: + compare_model: ref('contactview_metadata') + - name: contactview_hierarchy + columns: + - name: branch_uuid + tests: + - not_null + - relationships: + to: ref('contactview_branch') + field: uuid + - name: chw_uuid + tests: + - not_null + - relationships: + to: ref('contactview_chp') + field: uuid + - name: chw_area_uuid + tests: + - not_null + - name: contactview_branch + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('contactview_hospital') + field: uuid + - name: contactview_clinic + columns: + - name: uuid + tests: + - not_null + - relationships: + to: ref('contactview_metadata') + field: uuid + - name: chw_uuid + tests: + - not_null + - relationships: + to: ref('contactview_chw') + field: uuid + tests: + - dbt_utils.fewer_rows_than: + compare_model: ref('contactview_metadata') diff --git a/models/brac/contact_views/contactview_chp.sql b/models/brac/contact_views/contactview_chp.sql index e635a1d9..5e69e4f3 100644 --- a/models/brac/contact_views/contactview_chp.sql +++ b/models/brac/contact_views/contactview_chp.sql @@ -47,8 +47,7 @@ SELECT chp.doc #>> '{chp_profile, g_language, mother_tongue}'::text[] AS mother_tongue, chp.doc #>> '{chp_profile, g_language, other_languages}'::text[] AS other_languages, chp.doc #>> '{chp_profile, g_other_details, incentives}'::text[] AS incentives, - chp.doc #>> '{chp_profile, g_other_details, chp_services}'::text[] AS chp_services, - contactview_chw."@timestamp" AS "@timestamp" + chp.doc #>> '{chp_profile, g_other_details, chp_services}'::text[] AS chp_services FROM {{ ref("contactview_chw") }} JOIN {{ ref("raw_contacts") }} ON contactview_chw.area_uuid = (raw_contacts.doc ->> '_id'::text) diff --git a/models/brac/contact_views/contactview_chw.sql b/models/brac/contact_views/contactview_chw.sql index eba7af04..437014c1 100644 --- a/models/brac/contact_views/contactview_chw.sql +++ b/models/brac/contact_views/contactview_chw.sql @@ -5,8 +5,7 @@ SELECT chw.name, pplfields.date_of_birth, pplfields.parent_type, chwarea.uuid AS area_uuid, - chwarea.parent_uuid AS branch_uuid, - pplfields."@timestamp" AS "@timestamp" + chwarea.parent_uuid AS branch_uuid FROM {{ ref("contactview_person_fields") }} pplfields JOIN {{ ref("contactview_metadata") }} chw ON chw.uuid = pplfields.uuid JOIN {{ ref("contactview_metadata") }} chwarea ON chw.parent_uuid = chwarea.uuid diff --git a/models/brac/contact_views/contactview_person_fields.sql b/models/brac/contact_views/contactview_person_fields.sql index 6cff06df..13f35005 100644 --- a/models/brac/contact_views/contactview_person_fields.sql +++ b/models/brac/contact_views/contactview_person_fields.sql @@ -2,8 +2,7 @@ SELECT person.uuid, person.phone, person.phone2, person.date_of_birth, - parent.type AS parent_type, - person."@timestamp" AS "@timestamp" + parent.type AS parent_type FROM {{ ref("contactview_metadata") }} person LEFT JOIN {{ ref("contactview_metadata") }} parent ON person.parent_uuid = parent.uuid WHERE person.type = 'person'::text \ No newline at end of file diff --git a/models/brac/impact/ANC/impact_ancview_danger_sign.sql b/models/brac/impact/ANC/impact_ancview_danger_sign.sql index b414b1e3..ae411b31 100644 --- a/models/brac/impact/ANC/impact_ancview_danger_sign.sql +++ b/models/brac/impact/ANC/impact_ancview_danger_sign.sql @@ -43,4 +43,4 @@ SELECT reported_by_parent, reported FROM - danger_sign_CTE; + danger_sign_CTE diff --git a/models/brac/impact/ANC/impact_ancview_delivery.sql b/models/brac/impact/ANC/impact_ancview_delivery.sql index c6e4de1e..ab167afc 100644 --- a/models/brac/impact/ANC/impact_ancview_delivery.sql +++ b/models/brac/impact/ANC/impact_ancview_delivery.sql @@ -36,4 +36,4 @@ LEFT JOIN preg ON (pnc.reported > preg.reported AND pnc.reported < (preg.reporte WHERE pnc.follow_up_count = '1' AND (pnc.pregnancy_outcome IN (VALUES ('healthy'), ('still_birth'))) - AND pnc.patient_id IS NOT NULL AND pnc.patient_id <> ''; + AND pnc.patient_id IS NOT NULL AND pnc.patient_id <> '' diff --git a/models/brac/impact/ANC/impact_ancview_pregnancy.sql b/models/brac/impact/ANC/impact_ancview_pregnancy.sql deleted file mode 100644 index 39946a3d..00000000 --- a/models/brac/impact/ANC/impact_ancview_pregnancy.sql +++ /dev/null @@ -1,33 +0,0 @@ -{{ config( - materialized='table', - description='ANC Pregnancy view for Brac Uganda' -) }} - -WITH config_cte AS ( - SELECT ( value #>> '{lmp_calcs,maximum_days_pregnant}'::TEXT[])::INTEGER AS maximum_days_pregnant - FROM configuration - WHERE key = 'anc'::TEXT AND value ? 'lmp_calcs'::TEXT -) - -SELECT - preg.uuid AS uuid, - preg.lmp::DATE, - preg.edd, - preg.imported, - preg.patient_id, - preg.chw AS reported_by, - contact.parent_uuid AS reported_by_parent, - preg.reported, - 'pregnancy'::TEXT AS form, - preg.lmp <> '' AS has_lmp, - date_trunc('day', preg.reported::TIMESTAMP WITH TIME ZONE) <= (preg.lmp::DATE + '84 days'::INTERVAL) AS early_reg, - (preg.lmp::DATE + '84 days'::INTERVAL)::DATE AS first_tri_end, - (preg.lmp::DATE + '168 days'::INTERVAL)::DATE AS second_tri_end, - (preg.lmp::DATE + ((config.maximum_days_pregnant || ' days')::INTERVAL))::DATE AS mdd, - date_part('days', now() - preg.lmp::DATE::TIMESTAMP WITH TIME ZONE)::INTEGER AS days_since_lmp, - date_part('days', preg.reported - preg.lmp::DATE::TIMESTAMP WITHOUT TIME ZONE)::INTEGER AS days_pregnant_at_reg, - preg.danger_signs <> ''::TEXT AS danger_sign_at_reg, - preg.risk_factors <> ''::TEXT AS has_risk_factor -FROM config_cte AS config, - {{ ref('useview_pregnancy') }} AS preg -INNER JOIN {{ ref('contactview_metadata') }} AS contact ON contact.uuid = preg.chw; diff --git a/models/brac/impact/ANC/impact_ancview_pregnancy_visit.sql b/models/brac/impact/ANC/impact_ancview_pregnancy_visit.sql index b6324411..78d8bb19 100644 --- a/models/brac/impact/ANC/impact_ancview_pregnancy_visit.sql +++ b/models/brac/impact/ANC/impact_ancview_pregnancy_visit.sql @@ -15,4 +15,4 @@ SELECT FROM {{ ref('useview_visit') }} WHERE - visit_type = 'anc'; + visit_type = 'anc' diff --git a/models/brac/impact/PNC/get_dashboard_data_pnc_impact.sql b/models/brac/impact/PNC/get_dashboard_data_pnc_impact.sql deleted file mode 100644 index 97ceddcc..00000000 --- a/models/brac/impact/PNC/get_dashboard_data_pnc_impact.sql +++ /dev/null @@ -1,68 +0,0 @@ -{{ config( - materialized='table', - description='Dashboard data for PNC impact in BRAC Uganda' -) }} - -WITH period_CTE AS ( - SELECT generate_series( - date_trunc(param_interval_unit, now() - (param_num_units || ' ' || param_interval_unit)::interval), - CASE - WHEN param_include_current THEN now() - ELSE now() - ('1 ' || param_interval_unit)::interval - END, - ('1 ' || param_interval_unit)::interval - )::date AS start -) - -SELECT - CASE - WHEN param_facility_group_by = 'clinic' OR param_facility_group_by = 'health_center' OR param_facility_group_by = 'district_hospital' - THEN place_period.district_hospital_uuid - ELSE 'All' - END AS district_hospital_uuid, - CASE - WHEN param_facility_group_by = 'clinic' OR param_facility_group_by = 'health_center' OR param_facility_group_by = 'district_hospital' - THEN place_period.district_hospital_name - ELSE 'All' - END AS district_hospital_name, - CASE - WHEN param_facility_group_by = 'clinic' OR param_facility_group_by = 'health_center' - THEN place_period.health_center_uuid - ELSE 'All' - END AS health_center_uuid, - CASE - WHEN param_facility_group_by = 'clinic' OR param_facility_group_by = 'health_center' - THEN place_period.health_center_name - ELSE 'All' - END AS health_center_name, - 'All' AS clinic_uuid, - 'All' AS clinic_name, - place_period.period_start AS period_start, - date_part('epoch', place_period.period_start)::numeric AS period_start_epoch, - CASE - WHEN param_facility_group_by = 'health_center' - THEN place_period.health_center_uuid - WHEN param_facility_group_by = 'district_hospital' - THEN place_period.district_hospital_uuid - ELSE 'All' - END AS facility_join_field, - /* Other fields... (add the remaining fields here) */ -FROM - ( - -- Subqueries and joins... (add the subqueries and joins here) - ) AS place_period -GROUP BY - district_hospital_uuid, - district_hospital_name, - health_center_uuid, - health_center_name, - clinic_uuid, - clinic_name, - period_start, - period_start_epoch, - facility_join_field -ORDER BY - district_hospital_name, - health_center_name, - clinic_name, - period_start; diff --git a/models/brac/impact/PNC/impact_pncview_actual_enrollments.sql b/models/brac/impact/PNC/impact_pncview_actual_enrollments.sql index a683fd0e..c3edcba3 100644 --- a/models/brac/impact/PNC/impact_pncview_actual_enrollments.sql +++ b/models/brac/impact/PNC/impact_pncview_actual_enrollments.sql @@ -53,4 +53,4 @@ SELECT AND (first_pnc_visit_date::DATE - delivery_date::DATE)::INT <= 3::INT THEN TRUE ELSE FALSE END AS first_visit_on_time -FROM confirmed_deliveries_CTE; +FROM confirmed_deliveries_CTE diff --git a/models/brac/impact/PNC/impact_pncview_danger_sign.sql b/models/brac/impact/PNC/impact_pncview_danger_sign.sql index ce7a5783..66c9fee7 100644 --- a/models/brac/impact/PNC/impact_pncview_danger_sign.sql +++ b/models/brac/impact/PNC/impact_pncview_danger_sign.sql @@ -28,4 +28,4 @@ WITH pnc_danger_signs AS ( -- Model: pncview_danger_sign SELECT * -FROM pnc_danger_signs; +FROM pnc_danger_signs diff --git a/models/brac/impact/PNC/impact_pncview_expected_enrollments.sql b/models/brac/impact/PNC/impact_pncview_expected_enrollments.sql index 9fcd3d2d..d4030973 100644 --- a/models/brac/impact/PNC/impact_pncview_expected_enrollments.sql +++ b/models/brac/impact/PNC/impact_pncview_expected_enrollments.sql @@ -33,4 +33,4 @@ WITH pnc_expected_enrollments AS ( -- Model: pncview_expected_enrollments SELECT * -FROM pnc_expected_enrollments; +FROM pnc_expected_enrollments diff --git a/models/brac/impact/PNC/impact_pncview_visit.sql b/models/brac/impact/PNC/impact_pncview_visit.sql index 7b165721..716d2295 100644 --- a/models/brac/impact/PNC/impact_pncview_visit.sql +++ b/models/brac/impact/PNC/impact_pncview_visit.sql @@ -41,4 +41,4 @@ WITH pnc_visits AS ( -- Model: pncview_visits SELECT * -FROM pnc_visits; +FROM pnc_visits diff --git a/models/brac/impact/hhvisit/get_dashboard_data_hh_impact.sql b/models/brac/impact/hhvisit/get_dashboard_data_hh_impact.sql deleted file mode 100644 index 103f56b5..00000000 --- a/models/brac/impact/hhvisit/get_dashboard_data_hh_impact.sql +++ /dev/null @@ -1,148 +0,0 @@ -WITH hh_data AS ( -WITH main_data AS ( - WITH period_CTE AS ( - SELECT generate_series( - CASE - WHEN param_include_current - THEN date_trunc(param_interval_unit, now() - (param_num_units || ' ' || param_interval_unit)::INTERVAL) - ELSE - date_trunc(param_interval_unit, min(reported))::DATE - END, - CASE - WHEN param_include_current - THEN now() - ELSE now() - ('1 ' || param_interval_unit)::INTERVAL - END, - ('1 ' || param_interval_unit)::INTERVAL - )::DATE AS start - ) - - SELECT - CASE - WHEN param_facility_group_by IN ('clinic', 'health_center', 'district_hospital') - THEN place_period.district_hospital_uuid - ELSE 'All' - END AS _district_hospital_uuid, - CASE - WHEN param_facility_group_by IN ('clinic', 'health_center', 'district_hospital') - THEN place_period.district_hospital_name - ELSE 'All' - END AS _district_hospital_name, - CASE - WHEN param_facility_group_by IN ('clinic', 'health_center') - THEN place_period.health_center_uuid - ELSE 'All' - END AS _health_center_uuid, - CASE - WHEN param_facility_group_by IN ('clinic', 'health_center') - THEN place_period.health_center_name - ELSE 'All' - END AS _health_center_name, - 'All'::TEXT AS _clinic_uuid, - 'All'::TEXT AS _clinic_name, - place_period.period_start AS _period_start, - date_part('epoch', place_period.period_start)::NUMERIC AS _period_start_epoch, - CASE - WHEN param_facility_group_by = 'health_center' - THEN place_period.health_center_uuid - WHEN param_facility_group_by = 'district_hospital' - THEN place_period.district_hospital_uuid - ELSE 'All' - END AS _facility_join_field, - COALESCE(sum(hhcount.hh_registered), 0) AS hh_registered, - COALESCE(sum(hh_visit), 0) AS hh_visit - FROM - ( - SELECT - district_hospital.uuid AS district_hospital_uuid, - district_hospital.name AS district_hospital_name, - health_center.uuid AS health_center_uuid, - health_center.name AS health_center_name, - period_CTE.start AS period_start - FROM - period_CTE, - contactview_metadata AS health_center - INNER JOIN contactview_metadata AS district_hospital ON - (health_center.parent_uuid = district_hospital.uuid) - WHERE - district_hospital.type = 'district_hospital' - AND district_hospital.name NOT IN ('HQ', 'HW OVC') - ) AS place_period - LEFT JOIN - ( - SELECT - reported_by_parent, - date_trunc(param_interval_unit, reported)::DATE AS reported_month, - count(DISTINCT household_id) AS hh_visit - FROM - hhview_visits - GROUP BY - reported_by_parent, - reported_month - ) AS hh_visit ON (place_period.period_start = hh_visit.reported_month AND place_period.health_center_uuid = hh_visit.reported_by_parent) - LEFT JOIN ( - SELECT - parent_uuid, - date_trunc(param_interval_unit, reported) AS reported_month, - count(DISTINCT uuid) AS hh_registered - FROM - contactview_metadata - WHERE - TYPE = 'clinic' - GROUP BY - reported_month, - parent_uuid - ) AS hhcount ON (place_period.period_start = hhcount.reported_month AND place_period.health_center_uuid = hhcount.parent_uuid) - GROUP BY - _district_hospital_uuid, - _district_hospital_name, - _health_center_uuid, - _health_center_name, - _clinic_uuid, - _clinic_name, - _period_start, - _facility_join_field - ORDER BY - _district_hospital_name, - _health_center_name, - _clinic_name, - _period_start -) - -SELECT - _district_hospital_uuid, - _district_hospital_name, - _health_center_uuid, - _health_center_name, - _clinic_uuid, - _clinic_name, - _period_start, - _period_start_epoch, - _facility_join_field, - hh_registered, - -- Cumulative sum over all months - hh_visit, - sum(hh_registered) OVER (PARTITION BY _facility_join_field ORDER BY _period_start) AS total_hh_registered -FROM - main_data -) - -SELECT -_district_hospital_uuid, -_district_hospital_name, -_health_center_uuid, -_health_center_name, -_clinic_uuid, -_clinic_name, -_period_start, -_period_start_epoch, -_facility_join_field, -hh_registered, -total_hh_registered, -hh_visit, -safe_divide(hh_visit, total_hh_registered, 2) AS percent_hh_visit -FROM -hh_data -WHERE --- Filter the required data only. -_period_start >= now() - ((1 + param_num_units::INT) || ' ' || param_interval_unit)::INTERVAL; diff --git a/models/brac/impact/impact.yml b/models/brac/impact/impact.yml new file mode 100644 index 00000000..0447c5bc --- /dev/null +++ b/models/brac/impact/impact.yml @@ -0,0 +1,128 @@ +models: + - name: impact_ancview_danger_sign + columns: + - name: uuid + tests: + - not_null + - unique + - name: reported_by + tests: + - not_null + - name: form + tests: + - not_null + - name: impact_ancview_delivery + columns: + - name: uuid + tests: + - not_null + - name: impact_ancview_pregnancy_visit + columns: + - name: uuid + tests: + - not_null + - name: iccmview_assessment_follow_up + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('useview_assessment_follow_up') + field: uuid + - name: form + tests: + - not_null + - accepted_values: + values: ['assessment_follow_up'] + - name: iccmview_assessment + columns: + - name: uuid + tests: + - not_null + - relationships: + to: ref('useview_assessment') + field: uuid + - name: form + tests: + - not_null + - name: u1_iccmview_assessment_follow_up + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('useview_assessment_follow_up') + field: uuid + - name: form + tests: + - not_null + - accepted_values: + values: ['assessment_follow_up'] + - name: u1_iccmview_assessment + columns: + - name: uuid + tests: + - not_null + - relationships: + to: ref('useview_assessment') + field: uuid + - name: form + tests: + - not_null + - name: impact_pncview_actual_enrollments + columns: + - name: delivery_id + tests: + - not_null + - relationships: + to: ref('useview_postnatal_care') + field: uuid + - name: patient_id + tests: + - not_null + - name: impact_pncview_danger_sign + columns: + - name: uuid + tests: + - not_null + - relationships: + to: ref('useview_postnatal_care') + field: uuid + - name: date_of_event + tests: + - not_null + - name: patient_id + tests: + - not_null + - name: impact_pncview_expected_enrollments + columns: + - name: pregnancy_id + tests: + - not_null + - unique + - relationships: + to: ref('ancview_pregnancy') + field: uuid + - name: patient_id + tests: + - not_null + - name: reported_by + tests: + - not_null + - name: impact_pncview_visit + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('useview_postnatal_care') + field: uuid + - name: form + tests: + - not_null + - name: patient_id + tests: + - not_null diff --git a/models/brac/indexes.sql b/models/brac/indexes.sql deleted file mode 100644 index b0753f67..00000000 --- a/models/brac/indexes.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ config(schema='v1', materialized = 'raw_sql') }} - -CREATE INDEX IF NOT EXISTS couchdb_doc_id ON {{ env_var('ROOT_POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} ((doc ->> '_id'::text) text_ops); -CREATE INDEX IF NOT EXISTS couchdb_doc_type ON {{ env_var('ROOT_POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }}((doc ->> 'type'::text) text_ops); -CREATE INDEX IF NOT EXISTS couchdb_doc_form ON {{ env_var('ROOT_POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }}((doc ->> 'form'::text) text_ops) WHERE (doc ->> 'type'::text) = 'data_record'::text; -CREATE INDEX IF NOT EXISTS couchdb_doc_form_patient_id ON {{ env_var('ROOT_POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }}((doc #>> '{fields,patient_id}'::text[]) text_ops) WHERE (doc ->> 'type'::text) = 'data_record'::text; -CREATE INDEX IF NOT EXISTS couchdb_doc_form_place_id ON {{ env_var('ROOT_POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }}((doc #>> '{fields,place_id}'::text[]) text_ops) WHERE (doc ->> 'type'::text) = 'data_record'::text; -CREATE INDEX IF NOT EXISTS couchdb_doc_form_source_id ON {{ env_var('ROOT_POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }}((doc #>> '{fields,inputs,source_id}'::text[]) text_ops) WHERE (doc ->> 'type'::text) = 'data_record'::text; -CREATE INDEX IF NOT EXISTS couchdb_doc_assessment_patient_age ON {{ env_var('ROOT_POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }}((nullif(doc #>> '{fields,patient_age_in_years}', '')::int)) WHERE (doc ->> 'form'::text) = 'assessment'::text AND (doc #>> '{fields,patient_age_in_years}') != ''; \ No newline at end of file diff --git a/models/brac/useviews/useview_assessment_follow_up.sql b/models/brac/useviews/useview_assessment_follow_up.sql index 7defd3ff..2df844f4 100644 --- a/models/brac/useviews/useview_assessment_follow_up.sql +++ b/models/brac/useviews/useview_assessment_follow_up.sql @@ -14,7 +14,7 @@ }} SELECT -{{ dbt_utils.surrogate_key(['form_source_id', 'reported', 'uuid']) }} AS useview_assessment_follow_up_source_date_uuid, +{{ dbt_utils.generate_surrogate_key(['form_source_id', 'reported', 'uuid']) }} AS useview_assessment_follow_up_source_date_uuid, * FROM( @@ -62,4 +62,4 @@ FROM( {% if is_incremental() %} AND "@timestamp" > {{ max_existing_timestamp('"@timestamp"') }} {% endif %} -) x \ No newline at end of file +) x diff --git a/models/brac/useviews/useview_postnatal_care.sql b/models/brac/useviews/useview_postnatal_care.sql index b862df07..80964e24 100644 --- a/models/brac/useviews/useview_postnatal_care.sql +++ b/models/brac/useviews/useview_postnatal_care.sql @@ -68,4 +68,4 @@ doc ->> 'form' = 'postnatal_care'::text {% if is_incremental() %} AND "@timestamp" > {{ max_existing_timestamp('"@timestamp"') }} - {% endif %} \ No newline at end of file + {% endif %} diff --git a/models/brac/useviews/useviews.yml b/models/brac/useviews/useviews.yml new file mode 100644 index 00000000..b08bce9f --- /dev/null +++ b/models/brac/useviews/useviews.yml @@ -0,0 +1,143 @@ +version: 2 + +models: + - name: useview_visit + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: form + tests: + - not_null + - name: source_id + tests: + - not_null + - name: visit_type + tests: + - not_null + - accepted_values: + values: [anc, pnc, imm, iccm] + - name: danger_signs + tests: + - not_null + - name: useview_pregnancy + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: chw + tests: + - not_null + - name: useview_pregnancy_visit + columns: + - name: xmlforms_uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: useview_postnatal_care + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: form + tests: + - not_null + - name: useview_place_record + columns: + - name: place_id + tests: + - not_null + - relationships: + to: ref('couchdb') + field: doc#>>'{fields,place_id}' + - name: useview_patient_record + columns: + - name: patient_id + tests: + - not_null + - relationships: + to: ref('couchdb') + field: doc#>>'{fields,patient_id}' + - name: useview_household_survey + columns: + - name: uuid + tests: + - not_null + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: useview_health_forum + columns: + - name: xmlforms_uuid + tests: + - not_null + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: useview_family_survey + columns: + - name: uuid + tests: + - not_null + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: useview_assessment + columns: + - name: uuid + tests: + - not_null + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: form + tests: + - not_null + - name: formview_fp_patient_record + columns: + - name: uuid + tests: + - not_null + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: form + tests: + - not_null + - accepted_values: + values: + [ + 'family_planning', + 'fp_follow_up_long_term', + 'fp_follow_up_refill', + 'fp_follow_up_prospective', + 'fp_follow_up_short_term', + ] + - name: useview_assessment_follow_up + columns: + - name: uuid + tests: + - not_null + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: form + tests: + - not_null + - accepted_values: + values: ['assessment_follow_up'] diff --git a/models/brac/views/views.yml b/models/brac/views/views.yml new file mode 100644 index 00000000..dd1b8c85 --- /dev/null +++ b/models/brac/views/views.yml @@ -0,0 +1,269 @@ +version: 2 + +models: + - name: pncview_actual_enrollments + columns: + - name: delivery_id + tests: + - not_null + - unique + - relationships: + to: ref('useview_postnatal_care') + field: uuid + - name: patient_id + tests: + - not_null + - name: ancview_pregnancy + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('useview_pregnancy') + field: uuid + - name: reported_by + tests: + - not_null + - name: form + tests: + - not_null + - accepted_values: + values: [pregnancy] + - name: form_metadata + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: reported_by + tests: + - not_null + - name: chw + tests: + - not_null + - name: form + tests: + - not_null + - name: formname + tests: + - not_null + - name: impactview_month + columns: + - name: month + tests: + - not_null + - name: epoch + tests: + - not_null + - name: impactview_month_facility + columns: + - name: month + tests: + - not_null + - name: epoch + tests: + - not_null + - name: formview_immunization_follow_up + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: formview_assessment_follow_up + columns: + - name: xmlforms_uuid + tests: + - not_null + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: formview_muac_follow_up + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: form + tests: + - not_null + - accepted_values: + values: [muac_follow_up] + - name: formview_health_forum + columns: + - name: xmlforms_uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: formview_death_confirmation + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: formview_covid_referral_follow_up + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: formview_covid_evacuation_follow_up + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: followup + columns: + - name: form + tests: + - not_null + - accepted_values: + values: + [ + 'family_planning', + 'fp_follow_up_long_term', + 'fp_follow_up_refill', + 'fp_follow_up_prospective', + 'fp_follow_up_short_term', + ] + - name: contactview_family_survey + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('couchdb') + field: doc ->> '_id' + - name: impactview_chw_facility + columns: + - name: chw_uuid + tests: + - not_null + - unique + - relationships: + to: ref('contactview_chp') + field: uuid + - name: impactview_active + columns: + - name: month + tests: + - not_null + - name: epoch + tests: + - not_null + - name: iccmview_active + columns: + - name: month + tests: + - not_null + - name: epoch + tests: + - not_null + - name: fp_referral_cases + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('formview_fp_patient_record') + field: uuid + - name: form + tests: + - not_null + - accepted_values: + values: + [ + 'family_planning', + 'fp_follow_up_long_term', + 'fp_follow_up_refill', + 'fp_follow_up_prospective', + 'fp_follow_up_short_term', + ] + - name: is_referral_case + tests: + - not_null + - accepted_values: + values: [true] + - name: ancview_danger_sign + columns: + - name: uuid + tests: + - not_null + - name: form + tests: + - not_null + - name: ancview_delivery + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('useview_postnatal_care') + field: uuid + - name: form + tests: + - not_null + - name: pregnancy_outcome + tests: + - not_null + - accepted_values: + values: [healthy, still_birth] + - name: ancview_pregnancy_visit + columns: + - name: uuid + tests: + - not_null + - unique + - relationships: + to: ref('useview_visit') + field: uuid + - name: pregnancy_id + tests: + - not_null + - name: form + tests: + - not_null + - name: visit_with_danger_sign + tests: + - not_null + - name: dashboardview_iccm + columns: + - name: month + tests: + - not_null + - name: epoch + tests: + - not_null + - name: dashboardview_iccm_active + columns: + - name: month + tests: + - not_null + - name: epoch + tests: + - not_null diff --git a/models/contact_views/contact_views.yml b/models/contact_views/contact_views.yml index 8ad464b5..65546438 100644 --- a/models/contact_views/contact_views.yml +++ b/models/contact_views/contact_views.yml @@ -14,6 +14,7 @@ models: - name: uuid tests: - not_null + - unique - name: type tests: - not_null diff --git a/models/user_tables/chws.sql b/models/user_tables/chws.sql deleted file mode 100644 index 3f8658e7..00000000 --- a/models/user_tables/chws.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ - config( - materialized = 'table', - indexes=[ - {'columns': ['"@timestamp"'], 'type': 'brin'}, - {'columns': ['"patient_id"'], 'type': 'hash'}, - ] - ) -}} - -SELECT - doc->>'name' AS name, - doc->>'date_of_birth' AS date_of_birth, - doc->>'phone' AS phone, - doc->>'sex' AS sex, - doc->>'reported_date' AS reported_date, - doc->>'patient_id' AS patient_id, - * -FROM {{ ref('couchdb') }} -WHERE - doc->>'type' = 'person' diff --git a/models/user_tables/user_tables.yml b/models/user_tables/user_tables.yml deleted file mode 100644 index 95f6e872..00000000 --- a/models/user_tables/user_tables.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: 2 - -models: - - name: chws - tests: - - dbt_utils.fewer_rows_than: - compare_model: ref('couchdb') - - dbt_utils.equal_rowcount: - compare_model: ref('person')