From e7a1fa4f3f9adf5000f998e7ebb2a8b685ff2701 Mon Sep 17 00:00:00 2001 From: Doug Jeffery Date: Wed, 5 Jan 2022 13:49:21 +0000 Subject: [PATCH 1/6] yml changes to dependencies --- dbt_project.yml | 6 ++++-- packages.yml | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 12b35be..b7f1de0 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ name: 'xero' -version: '0.4.0' +version: '0.3.0' config-version: 2 -require-dbt-version: [">=1.0.0", "<2.0.0"] +require-dbt-version: ">=0.20.0" vars: xero: @@ -14,6 +14,8 @@ vars: organization: "{{ ref('stg_xero__organization') }}" credit_note: "{{ ref('stg_xero__credit_note') }}" bank_transaction: "{{ ref('stg_xero__bank_transaction') }}" + purchase_order_line_item: "{{ ref('stg_xero__purchase_order_line_item') }}" + purchase_order: "{{ ref('stg_xero__purchase_order') }}" models: xero: diff --git a/packages.yml b/packages.yml index 336841d..52a6c95 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,2 @@ packages: -- package: fivetran/xero_source - version: [">=0.4.0", "<0.5.0"] +- git: https://github.com/Project-Rouge/dbt_xero_source From b69079c4bac4f3d0e0a59a62663d3fc7a3a52b6b Mon Sep 17 00:00:00 2001 From: Doug Jeffery Date: Thu, 6 Jan 2022 10:18:10 +0000 Subject: [PATCH 2/6] new model for po line items and assoc yml changes --- models/xero.yml | 81 ++++++++++++++++++++++ models/xero__purchase_order_line_items.sql | 61 ++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 models/xero__purchase_order_line_items.sql diff --git a/models/xero.yml b/models/xero.yml index 19684a8..ba49ef5 100644 --- a/models/xero.yml +++ b/models/xero.yml @@ -49,6 +49,8 @@ models: description: Percentage discount being applied to a line item. - name: invoice_id description: The Xero identifier for the invoice the line items belongs to. + tests: + - not null - name: item_code description: User defined item code. - name: line_amount @@ -209,3 +211,82 @@ models: The source where this data was pulled from. If you are making use of the `union_schemas` variable, this will be the source schema. If you are making use of the `union_databases` variable, this will be the source database. If you are not unioining together multiple sources, this will be an empty string. + + + - name: xero__purchase_order_line_items + description: Each record represents a purchase order line item, enriched with account, contact and invoice information. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - line_item_id + - source_relation + columns: + - name: _fivetran_synced + description: "{{ doc('_fivetran_synced') }}" + - name: description + description: The description of the line item + - name: discount_rate + description: Percentage discount being applied to a purchase order line item. + - name: purchase_order_id + description: The Xero identifier for the purchase order the line items belongs to. + tests: + - not_null + - name: item_code + description: User defined item code. + - name: line_amount + description: The line amount reflects the discounted price if a DiscountRate has been used. + - name: line_item_id + description: Xero identifier for the purchase order line item. + tests: + - not_null + - name: quantity + description: The quantity of the purchase order line item. + - name: tax_amount + description: The tax amount is auto calculated as a percentage of the line amount based on the tax rate. + - name: tax_type + description: Used as an override if the default Tax Code for the selected AccountCode is not correct. + - name: unit_amount + description: The unit amount of each purchase order line item. + - name: purchase_order_date + description: The date the purchase order was issued. + - name: updated_date + description: The date the purchase order was last modified. + - name: delivery_date + description: The expected delivery date of the purchase order goods, when set. + - name: currency_code + description: The currency that purchase order has been raised in + - name: currency_rate + description: The currency rate for a multicurrency purchase order + - name: purchase_order_reference + description: Additional reference number to suit the organisation or supplier. + - name: purchase_order_status + description: The status of the purchase order + - name: type + description: The type of the purchase order. + - name: is_discounted + description: Flags if purchase order items have discounts applied + - name: line_amount_types + description: Tax inclusive, exclusive or no tax + - name: sub_total + description: Purchase order amount before tax + - name: total_tax + description: Purchase order tax amount + - name: total + description: Purchase order total amount including tax + - name: account_id + description: The Xero identifier for the related account. + - name: account_name + description: The name of the account. + - name: account_code + description: Customer defined alpha numeric account code e.g 200 or SALES + - name: account_type + description: The type of account, e.g. CURRENT, EQUITY, SALES + - name: account_class + description: The class of account, e.g. ASSET, EQUITY, EXPENSE, LIABILITY, REVENUE + - name: contact_name + description: The name of the associated contact. + - name: source_relation + description: > + The source where this data was pulled from. If you are making use of the `union_schemas` variable, this will be the source schema. + If you are making use of the `union_databases` variable, this will be the source database. If you are not unioining together multiple + sources, this will be an empty string. diff --git a/models/xero__purchase_order_line_items.sql b/models/xero__purchase_order_line_items.sql new file mode 100644 index 0000000..28794bd --- /dev/null +++ b/models/xero__purchase_order_line_items.sql @@ -0,0 +1,61 @@ +with line_items as ( + + select * + from {{ var('purchase_order_line_item') }} + +), purchase_orders as ( + + select * + from {{ var('purchase_order') }} + +), accounts as ( + + select * + from {{ var('account') }} + +), contacts as ( + + select * + from {{ var('contact') }} + +), joined as ( + + select + line_items.*, + + purchase_orders.purchase_order_date, + purchase_orders.updated_date, + purchase_orders.delivery_date, + purchase_orders.sub_total, + purchase_orders.total_tax, + purchase_orders.total, + purchase_orders.currency_code, + purchase_orders.currency_rate, + purchase_orders.purchase_order_number, + purchase_orders.purchase_order_status, + purchase_orders.type, + purchase_orders.reference as purchase_order_reference, + purchase_orders.is_discounted, + purchase_orders.line_amount_types, + accounts.account_id, + accounts.account_name, + accounts.account_type, + accounts.account_class, + + contacts.contact_name + + from line_items + left join purchase_orders + on (line_items.purchase_order_id = purchase_orders.purchase_order_id + and line_items.source_relation = purchase_orders.source_relation) + left join accounts + on (line_items.account_code = accounts.account_code + and line_items.source_relation = accounts.source_relation) + left join contacts + on (purchase_orders.contact_id = contacts.contact_id + and purchase_orders.source_relation = contacts.source_relation) + +) + +select * +from joined \ No newline at end of file From 495c1217d13947562aaf324df7d781b435786b3b Mon Sep 17 00:00:00 2001 From: Doug Jeffery Date: Thu, 6 Jan 2022 17:45:10 +0000 Subject: [PATCH 3/6] fix incorrect test --- models/xero.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/xero.yml b/models/xero.yml index ba49ef5..edecea6 100644 --- a/models/xero.yml +++ b/models/xero.yml @@ -58,7 +58,7 @@ models: - name: line_item_id description: Xero identifier for the line item. tests: - - not_null + - not null - name: quantity description: The quantity of the line item. - name: tax_amount From 4bc771fd4c1a7f0b5048fd27ee5fcf0c7ce53c65 Mon Sep 17 00:00:00 2001 From: Doug Jeffery Date: Fri, 7 Jan 2022 09:16:44 +0000 Subject: [PATCH 4/6] fix tests again --- models/xero.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/xero.yml b/models/xero.yml index edecea6..8d69cc6 100644 --- a/models/xero.yml +++ b/models/xero.yml @@ -50,7 +50,7 @@ models: - name: invoice_id description: The Xero identifier for the invoice the line items belongs to. tests: - - not null + - not_null - name: item_code description: User defined item code. - name: line_amount @@ -58,7 +58,7 @@ models: - name: line_item_id description: Xero identifier for the line item. tests: - - not null + - not_null - name: quantity description: The quantity of the line item. - name: tax_amount From 4cc7c7289431a3d026fea9d28c7bcb132c3d09d5 Mon Sep 17 00:00:00 2001 From: Doug Jeffery Date: Tue, 12 Apr 2022 12:39:01 +0100 Subject: [PATCH 5/6] use acc code not name for unique check --- models/xero.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/xero.yml b/models/xero.yml index 8d69cc6..75baad7 100644 --- a/models/xero.yml +++ b/models/xero.yml @@ -190,7 +190,7 @@ models: combination_of_columns: - source_relation - date_month - - account_name + - account_code columns: - name: date_month description: The month a given record is reporting on. From 06932eafdf43ded72db6465b7693cca5af8fa4e2 Mon Sep 17 00:00:00 2001 From: Doug Jeffery Date: Tue, 12 Apr 2022 12:49:30 +0100 Subject: [PATCH 6/6] Revert "use acc code not name for unique check" --- models/xero.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/xero.yml b/models/xero.yml index 75baad7..8d69cc6 100644 --- a/models/xero.yml +++ b/models/xero.yml @@ -190,7 +190,7 @@ models: combination_of_columns: - source_relation - date_month - - account_code + - account_name columns: - name: date_month description: The month a given record is reporting on.