diff --git a/models/contacts/contact.sql b/models/contacts/contact.sql index 865b818b..74fa5d3d 100644 --- a/models/contacts/contact.sql +++ b/models/contacts/contact.sql @@ -34,6 +34,7 @@ INNER JOIN ON source_table._id = document_metadata.uuid WHERE document_metadata.doc_type IN ('contact', 'clinic', 'district_hospital', 'health_center', 'person') + AND source_table._deleted = false {% if is_incremental() %} AND document_metadata.saved_timestamp >= {{ max_existing_timestamp('saved_timestamp') }} {% endif %} diff --git a/models/contacts/contacts.yml b/models/contacts/contacts.yml index 05120925..9066d59d 100644 --- a/models/contacts/contacts.yml +++ b/models/contacts/contacts.yml @@ -60,6 +60,8 @@ models: data_type: date - name: sex data_type: string + - name: deleted + data_type: boolean - name: place config: contract: @@ -79,6 +81,8 @@ models: data_type: timestamp - name: place_id data_type: string + - name: deleted + data_type: boolean - name: patient config: contract: @@ -98,3 +102,5 @@ models: data_type: timestamp - name: patient_id data_type: string + - name: deleted + data_type: boolean diff --git a/models/contacts/patient.sql b/models/contacts/patient.sql index ac2f72ef..3a2c62e0 100644 --- a/models/contacts/patient.sql +++ b/models/contacts/patient.sql @@ -14,7 +14,8 @@ SELECT uuid, person.saved_timestamp, - couchdb.doc->>'patient_id' as patient_id + couchdb.doc->>'patient_id' AS patient_id, + person.deleted AS deleted FROM {{ ref('person') }} person INNER JOIN {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} couchdb ON couchdb._id = uuid WHERE couchdb.doc->>'patient_id' IS NOT NULL diff --git a/models/contacts/person.sql b/models/contacts/person.sql index c952d150..1ed754e2 100644 --- a/models/contacts/person.sql +++ b/models/contacts/person.sql @@ -13,8 +13,9 @@ SELECT contact.uuid, contact.saved_timestamp, - (couchdb.doc->>'date_of_birth')::date as date_of_birth, - couchdb.doc->>'sex' as sex + (couchdb.doc->>'date_of_birth')::date AS date_of_birth, + couchdb.doc->>'sex' AS sex, + contact.deleted AS deleted FROM {{ ref("contact") }} contact INNER JOIN {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} couchdb ON couchdb._id = uuid WHERE contact.contact_type = 'person' diff --git a/models/contacts/place.sql b/models/contacts/place.sql index 45bf274b..6099fe1a 100644 --- a/models/contacts/place.sql +++ b/models/contacts/place.sql @@ -14,7 +14,8 @@ SELECT uuid, contact.saved_timestamp, - couchdb.doc->>'place_id' as place_id + couchdb.doc->>'place_id' AS place_id, + contact.deleted AS deleted FROM {{ ref('contact') }} contact INNER JOIN {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} couchdb ON couchdb._id = uuid WHERE diff --git a/models/forms/data_record.sql b/models/forms/data_record.sql index 431d5884..d6b63d04 100644 --- a/models/forms/data_record.sql +++ b/models/forms/data_record.sql @@ -47,6 +47,7 @@ INNER JOIN ON source_table._id = document_metadata.uuid WHERE document_metadata.doc_type = 'data_record' + AND source_table._deleted = false {% if is_incremental() %} AND document_metadata.saved_timestamp >= {{ max_existing_timestamp('saved_timestamp') }} {% endif %} diff --git a/test/sqltest/contact.sql b/test/sqltest/contact.sql index 1c6c0440..9143af85 100644 --- a/test/sqltest/contact.sql +++ b/test/sqltest/contact.sql @@ -5,9 +5,7 @@ WHERE couchdb.doc->>'type' IN ('contact', 'clinic', 'district_hospital', 'health_center', 'person') -- TEST CONDITIONS AND ( - -- in couchdb, not in contact table - (contact.uuid IS NULL) - OR -- fields dont match + -- fields dont match contact.parent_uuid <> couchdb.doc->'parent'->>'_id' OR contact.contact_type <> COALESCE(couchdb.doc->>'contact_type', couchdb.doc->>'type') OR contact.phone <> couchdb.doc->>'phone' diff --git a/test/sqltest/data_record.sql b/test/sqltest/data_record.sql index f0c16d84..e8ef30de 100644 --- a/test/sqltest/data_record.sql +++ b/test/sqltest/data_record.sql @@ -5,9 +5,7 @@ WHERE couchdb.doc->>'type' = 'data_record' -- TEST CONDITIONS AND ( - -- in couchdb, not in data_record - (data_record.uuid IS NULL) - OR -- fields dont match + -- fields dont match data_record.from_phone <> couchdb.doc->>'from' OR data_record.form <> couchdb.doc->>'form' OR data_record.patient_id <> couchdb.doc->>'patient_id' OR diff --git a/test/sqltest/patient.sql b/test/sqltest/patient.sql index fa7387e5..a1a85604 100644 --- a/test/sqltest/patient.sql +++ b/test/sqltest/patient.sql @@ -9,6 +9,6 @@ WHERE couchdb.doc->>'patient_id' IS NOT NULL -- TEST CONDITIONS AND ( - -- in couchdb, not in patients - patient.uuid IS NULL + -- deleted is true + (patient.deleted = true) ) diff --git a/test/sqltest/person.sql b/test/sqltest/person.sql index d6c52a92..e9fbf6d1 100644 --- a/test/sqltest/person.sql +++ b/test/sqltest/person.sql @@ -10,8 +10,6 @@ WHERE ) -- TEST CONDITIONS AND ( - -- in couchdb, not in person - person.uuid IS NULL OR - -- a person, but not a contact? - contact.uuid IS NULL + -- deleted is true + (person.deleted = true) ) diff --git a/test/sqltest/place.sql b/test/sqltest/place.sql index c9e78616..ecdf31bd 100644 --- a/test/sqltest/place.sql +++ b/test/sqltest/place.sql @@ -3,15 +3,15 @@ FROM {{ env_var('POSTGRES_SCHEMA') }}.{{ env_var('POSTGRES_TABLE') }} couchdb LEFT JOIN {{ ref('place') }} place ON couchdb._id = place.uuid LEFT JOIN {{ ref('contact') }} contact ON contact.uuid = place.uuid WHERE - -- person conditions + -- place conditions ( (couchdb.doc->>'type' <> 'person') AND (couchdb.doc->>'type' = 'contact' AND couchdb.doc->>'contact_type' <> 'person') ) -- TEST CONDITIONS AND ( - -- in couchdb, not in place - place.uuid IS NULL OR - -- a person, but not a contact? + -- deleted is true + (place.deleted = true) OR + -- a place, but not a contact? contact.uuid IS NULL )