From 8f1fb2bb9c5b362b666e4930a0afdb08b7bd83df Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 15 Oct 2024 14:01:18 -0600 Subject: [PATCH 1/6] finishing touches across many models. also introduces dim_labels --- .github/workflows/dbt_run_adhoc.yml | 66 +++ .github/workflows/dbt_run_core.yml | 46 +++ .github/workflows/dbt_run_dev_refresh.yml | 45 ++ models/bronze/bronze__transactions.sql | 8 - models/bronze/bronze__transactions_FR.sql | 8 - .../bronze_api__program_mappings.sql | 34 -- models/descriptions/address.md | 5 + models/descriptions/solution_id.md | 5 + models/gold/core/core__dim_labels.sql | 36 ++ models/gold/core/core__dim_labels.yml | 52 +++ models/gold/core/core__dim_programs.sql | 9 +- models/gold/core/core__dim_programs.yml | 9 - .../core/core__fact_block_round_batches.sql | 5 +- .../core/core__fact_block_round_batches.yml | 4 +- .../gold/core/core__fact_block_solutions.sql | 5 +- .../gold/core/core__fact_block_solutions.yml | 82 ++-- models/gold/core/core__fact_blocks.sql | 4 +- models/gold/core/core__fact_blocks.yml | 6 - models/gold/core/core__fact_transactions.sql | 5 +- models/gold/core/core__fact_transitions.sql | 5 +- .../core/programs/silver__custom_programs.sql | 388 ++++++++++++++++++ models/silver/core/silver__blocks.sql | 15 +- models/silver/core/silver__labels.sql | 23 ++ models/silver/core/silver__programs.sql | 1 - models/silver/core/silver__programs.yml | 40 ++ models/silver/core/silver__transactions.sql | 2 +- models/silver/core/silver__transitions.sql | 2 +- models/sources.yml | 6 +- .../streamline__transactions_complete.sql | 55 --- .../streamline__transactions_realtime.sql | 80 ---- 30 files changed, 764 insertions(+), 287 deletions(-) create mode 100644 .github/workflows/dbt_run_adhoc.yml create mode 100644 .github/workflows/dbt_run_core.yml create mode 100644 .github/workflows/dbt_run_dev_refresh.yml delete mode 100644 models/bronze/bronze__transactions.sql delete mode 100644 models/bronze/bronze__transactions_FR.sql delete mode 100644 models/bronze/bronze_api/bronze_api__program_mappings.sql create mode 100644 models/descriptions/address.md create mode 100644 models/descriptions/solution_id.md create mode 100644 models/gold/core/core__dim_labels.sql create mode 100644 models/gold/core/core__dim_labels.yml create mode 100644 models/silver/core/programs/silver__custom_programs.sql create mode 100644 models/silver/core/silver__labels.sql create mode 100644 models/silver/core/silver__programs.yml delete mode 100644 models/streamline/silver/core/complete/streamline__transactions_complete.sql delete mode 100644 models/streamline/silver/core/realtime/streamline__transactions_realtime.sql diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml new file mode 100644 index 0000000..05134ae --- /dev/null +++ b/.github/workflows/dbt_run_adhoc.yml @@ -0,0 +1,66 @@ +name: dbt_run_adhoc +run-name: ${{ inputs.dbt_command }} + +on: + workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: choice + description: DBT run environment + required: true + options: + - dev + - prod + default: dev + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + - DBT_EMERGENCY + default: DBT + dbt_command: + type: string + description: "DBT Run Command" + required: true + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ inputs.warehouse }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_${{ inputs.environment }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + ${{ inputs.dbt_command }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_core.yml b/.github/workflows/dbt_run_core.yml new file mode 100644 index 0000000..7134c44 --- /dev/null +++ b/.github/workflows/dbt_run_core.yml @@ -0,0 +1,46 @@ +name: dbt_run_core +run-name: dbt_run_core + +on: + workflow_dispatch: + # schedule: + # # Runs "at minute 10, every hour" (see https://crontab.guru) + # - cron: '10 * * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m "aleo_models,tag:core" \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml new file mode 100644 index 0000000..45b5edc --- /dev/null +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -0,0 +1,45 @@ +name: dbt_run_dev_refresh +run-name: dbt_run_dev_refresh + +on: + workflow_dispatch: + schedule: + # Runs "at 5:10 UTC" (see https://crontab.guru) + - cron: '10 5 * * *' + +env: + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt workstream-dbt-core + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_create_prod_clone \ No newline at end of file diff --git a/models/bronze/bronze__transactions.sql b/models/bronze/bronze__transactions.sql deleted file mode 100644 index 39afc06..0000000 --- a/models/bronze/bronze__transactions.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ config ( - materialized = 'view', - enabled = false -) }} -{{ streamline_external_table_query_v2( - model = "transactions", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" -) }} diff --git a/models/bronze/bronze__transactions_FR.sql b/models/bronze/bronze__transactions_FR.sql deleted file mode 100644 index 69b4298..0000000 --- a/models/bronze/bronze__transactions_FR.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ config ( - materialized = 'view', - enabled = false -) }} -{{ streamline_external_table_FR_query_v2( - model = 'transactions', - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" -) }} diff --git a/models/bronze/bronze_api/bronze_api__program_mappings.sql b/models/bronze/bronze_api/bronze_api__program_mappings.sql deleted file mode 100644 index a1c33e8..0000000 --- a/models/bronze/bronze_api/bronze_api__program_mappings.sql +++ /dev/null @@ -1,34 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = "program_id", - tags = ['core','full_test'], - enabled = false -) }} - -WITH programs AS ( - - SELECT - program_id - FROM - {{ ref('silver__programs') }} -), -mappings AS ( - SELECT - program_id, - {{ target.database }}.live.udf_api( - 'GET', - '{Service}/program/' || program_id || '/mappings', - OBJECT_CONSTRUCT( - 'Content-Type', - 'application/json' - ),{}, - 'Vault/dev/aleo/mainnet' - ) :data AS mappings - FROM - programs -) -SELECT - program_id, - mappings -FROM - mappings diff --git a/models/descriptions/address.md b/models/descriptions/address.md new file mode 100644 index 0000000..aeed86f --- /dev/null +++ b/models/descriptions/address.md @@ -0,0 +1,5 @@ +{% docs address %} + +The unique address referring to an entity on the blockchain. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/solution_id.md b/models/descriptions/solution_id.md new file mode 100644 index 0000000..5bc0502 --- /dev/null +++ b/models/descriptions/solution_id.md @@ -0,0 +1,5 @@ +{% docs solution_id %} + +A unique identifier for the prover solution. + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/core/core__dim_labels.sql b/models/gold/core/core__dim_labels.sql new file mode 100644 index 0000000..652de2a --- /dev/null +++ b/models/gold/core/core__dim_labels.sql @@ -0,0 +1,36 @@ +{{ config( + materialized = 'incremental', + unique_key = ['dim_labels_id'], + incremental_strategy = 'merge', + cluster_by = 'modified_timestamp::DATE', + merge_exclude_columns = ['inserted_timestamp'], + tags = ['core'] +) }} + +SELECT + 'lava' AS blockchain, + creator, + address, + address_name, + label_type, + label_subtype, + project_name, + {{ dbt_utils.generate_surrogate_key( + [' address '] + ) }} AS dim_labels_id, + SYSDATE() AS inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__labels') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} \ No newline at end of file diff --git a/models/gold/core/core__dim_labels.yml b/models/gold/core/core__dim_labels.yml new file mode 100644 index 0000000..ab2d655 --- /dev/null +++ b/models/gold/core/core__dim_labels.yml @@ -0,0 +1,52 @@ +version: 2 +models: + - name: core__dim_labels + description: A hand curated table containing address names / labels for popular contracts, validators, tokens, etc. + columns: + - name: BLOCKCHAIN + description: The name of the blockchain + tests: + - not_null + - name: CREATOR + description: The name of the creator of the label + tests: + - not_null + - name: ADDRESS + description: Address that the label is for. This is the field that should be used to join other tables with labels. + tests: + - not_null + - name: ADDRESS_NAME + description: The name associated with the address + tests: + - not_null + - name: LABEL_TYPE + description: A high-level category describing the addresses main function or ownership + tests: + - not_null + - name: LABEL_SUBTYPE + description: A sub-category nested within label type providing further detail + tests: + - not_null + - name: PROJECT_NAME + description: Name of the project or entity associated with the address + tests: + - not_null + - name: DIM_LABELS_ID + description: Unique identifier for each label record + tests: + - not_null + - unique + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + tests: + - not_null + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - not_null + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCKCHAIN + - ADDRESS diff --git a/models/gold/core/core__dim_programs.sql b/models/gold/core/core__dim_programs.sql index 8caa508..f7ca147 100644 --- a/models/gold/core/core__dim_programs.sql +++ b/models/gold/core/core__dim_programs.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', - unique_key = ['program_id'], + unique_key = ['dim_program_id'], incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], tags = ['core','full_test'] ) }} @@ -11,7 +12,6 @@ SELECT program_id, edition, program, - {# mappings, #} verifying_keys, {{ dbt_utils.generate_surrogate_key( ['program_id'] @@ -21,3 +21,8 @@ SELECT '{{ invocation_id }}' AS invocation_id FROM {{ ref('silver__programs') }} +UNION ALL +SELECT + * +FROM + {{ ref('silver__custom_programs') }} \ No newline at end of file diff --git a/models/gold/core/core__dim_programs.yml b/models/gold/core/core__dim_programs.yml index 155a41c..c3175cd 100644 --- a/models/gold/core/core__dim_programs.yml +++ b/models/gold/core/core__dim_programs.yml @@ -47,19 +47,10 @@ models: column_type_list: - STRING - VARCHAR - # - name: MAPPINGS - # description: "{{ doc('mappings') }}" - # tests: - # - dbt_expectations.expect_column_to_exist - # - not_null - # - dbt_expectations.expect_column_values_to_be_in_type_list: - # column_type_list: - # - VARIANT - name: VERIFYING_KEYS description: "{{ doc('verifying_keys') }}" tests: - dbt_expectations.expect_column_to_exist - - not_null - name: dim_program_id description: '{{ doc("pk") }}' tests: diff --git a/models/gold/core/core__fact_block_round_batches.sql b/models/gold/core/core__fact_block_round_batches.sql index 2840e8b..bdda3d0 100644 --- a/models/gold/core/core__fact_block_round_batches.sql +++ b/models/gold/core/core__fact_block_round_batches.sql @@ -1,10 +1,11 @@ {{ config( materialized = 'incremental', incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::DATE >= (select min(block_timestamp::DATE) from ' ~ generate_tmp_view_name(this) ~ ')'], - unique_key = "fact_block_round_batches_id", + unique_key = ['fact_block_round_batches_id'], incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], + merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(batch_id,author,committee_id,transmission_ids);", tags = ['core', 'full_test'] ) }} diff --git a/models/gold/core/core__fact_block_round_batches.yml b/models/gold/core/core__fact_block_round_batches.yml index cdb2432..2ad1533 100644 --- a/models/gold/core/core__fact_block_round_batches.yml +++ b/models/gold/core/core__fact_block_round_batches.yml @@ -2,7 +2,7 @@ version: 2 models: - name: core__fact_block_round_batches - description: Records of all rounds and batches for blocks that have occurred on the Aleo network, dating back to the genesis block. + description: Records of rounds and batches within blocks on the Aleo network, capturing the consensus and block production process from genesis onward. tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -83,4 +83,4 @@ models: description: '{{ doc("modified_timestamp") }}' tests: - dbt_expectations.expect_column_to_exist - - not_null \ No newline at end of file + - not_null diff --git a/models/gold/core/core__fact_block_solutions.sql b/models/gold/core/core__fact_block_solutions.sql index ac16306..46df5a4 100644 --- a/models/gold/core/core__fact_block_solutions.sql +++ b/models/gold/core/core__fact_block_solutions.sql @@ -1,10 +1,11 @@ {{ config( materialized = 'incremental', incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::DATE >= (select min(block_timestamp::DATE) from ' ~ generate_tmp_view_name(this) ~ ')'], - unique_key = "fact_block_solutions_id", + unique_key = ['fact_block_solutions_id'], incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], + merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(solution_id,address,epoch_hash);", tags = ['core', 'full_test'] ) }} diff --git a/models/gold/core/core__fact_block_solutions.yml b/models/gold/core/core__fact_block_solutions.yml index 6050041..5fdb0e7 100644 --- a/models/gold/core/core__fact_block_solutions.yml +++ b/models/gold/core/core__fact_block_solutions.yml @@ -2,7 +2,7 @@ version: 2 models: - name: core__fact_block_solutions - description: Records of all prover solutions for each block. + description: Records of all prover solutions for each block, including block details, rewards, and solution-specific information. tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -37,62 +37,40 @@ models: column_type_list: - NUMBER - name: address - # description: "{{ doc('tx_count') }}" - # tests: - # - dbt_expectations.expect_column_to_exist - # - not_null - # - dbt_expectations.expect_column_values_to_be_in_type_list: - # column_type_list: - # - STRING + description: "{{ doc('address') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null - name: counter - # description: "{{ doc('previous_hash') }}" - # tests: - # - dbt_expectations.expect_column_to_exist - # - not_null + description: "A counter value associated with the solution." + tests: + - dbt_expectations.expect_column_to_exist + - not_null - name: epoch_hash - # description: "{{ doc('proving_round') }}" - # tests: - # - dbt_expectations.expect_column_to_exist - # - not_null - # - dbt_expectations.expect_column_values_to_be_in_type_list: - # column_type_list: - # - NUMBER - # - FLOAT + description: "The hash of the epoch in which the solution was found." + tests: + - dbt_expectations.expect_column_to_exist + - not_null - name: solution_id - # description: "{{ doc('prover_rounds') }}" - # tests: - # - dbt_expectations.expect_column_to_exist - # - not_null - # - dbt_expectations.expect_column_values_to_be_in_type_list: - # column_type_list: - # - VARIANT + description: "{{ doc('solution_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null + - unique - name: target - # description: "{{ doc('coinbase_target') }}" - # tests: - # - dbt_expectations.expect_column_to_exist - # - not_null - # - dbt_expectations.expect_column_values_to_be_in_type_list: - # column_type_list: - # - NUMBER - # - FLOAT + description: "The target difficulty for the block solution." + tests: + - dbt_expectations.expect_column_to_exist + - not_null - name: reward_raw - # description: "{{ doc('cumulative_proof_target') }}" - # tests: - # - dbt_expectations.expect_column_to_exist - # - not_null - # - dbt_expectations.expect_column_values_to_be_in_type_list: - # column_type_list: - # - NUMBER - # - FLOAT + description: "The raw value of the reward for solving the block." + tests: + - dbt_expectations.expect_column_to_exist + - not_null - name: reward - # description: "{{ doc('cumulative_weight') }}" - # tests: - # - dbt_expectations.expect_column_to_exist - # - not_null - # - dbt_expectations.expect_column_values_to_be_in_type_list: - # column_type_list: - # - NUMBER - # - FLOAT + description: "The processed or formatted value of the reward for solving the block." + tests: + - dbt_expectations.expect_column_to_exist - name: fact_block_solutions_id description: '{{ doc("pk") }}' tests: @@ -108,4 +86,4 @@ models: description: '{{ doc("modified_timestamp") }}' tests: - dbt_expectations.expect_column_to_exist - - not_null \ No newline at end of file + - not_null diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index 01c288c..099fad3 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -1,9 +1,9 @@ {{ config( materialized = 'incremental', incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::DATE >= (select min(block_timestamp::DATE) from ' ~ generate_tmp_view_name(this) ~ ')'], - unique_key = "fact_blocks_id", + unique_key = ['fact_blocks_id'], incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], + merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['block_timestamp::DATE'], tags = ['core', 'full_test'] ) }} diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index 5d69e23..74f395f 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -67,12 +67,6 @@ models: - FLOAT - name: ROUNDS description: "{{ doc('prover_rounds') }}" - # tests: - # - dbt_expectations.expect_column_to_exist - # - not_null - # - dbt_expectations.expect_column_values_to_be_in_type_list: - # column_type_list: - # - VARIANT - name: coinbase_target description: "{{ doc('coinbase_target') }}" tests: diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index e433ddd..1389546 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -1,10 +1,11 @@ {{ config( materialized = 'incremental', incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::DATE >= (select min(block_timestamp::DATE) from ' ~ generate_tmp_view_name(this) ~ ')'], - unique_key = "tx_id", + unique_key = ['fact_transactions_id'], incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], + merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,fee_msg,execution_msg,deployment_msg,owner_msg,finalize_msg,rejected_msg);", tags = ['core', 'full_test'] ) }} diff --git a/models/gold/core/core__fact_transitions.sql b/models/gold/core/core__fact_transitions.sql index fce03f0..6271b7e 100644 --- a/models/gold/core/core__fact_transitions.sql +++ b/models/gold/core/core__fact_transitions.sql @@ -1,10 +1,11 @@ {{ config( materialized = 'incremental', incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::DATE >= (select min(block_timestamp::DATE) from ' ~ generate_tmp_view_name(this) ~ ')'], - unique_key = "fact_transitions_id", + unique_key = ['fact_transitions_id'], incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], + merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,transition_id,program_id,function,inputs,outputs);", tags = ['core', 'full_test'] ) }} diff --git a/models/silver/core/programs/silver__custom_programs.sql b/models/silver/core/programs/silver__custom_programs.sql new file mode 100644 index 0000000..de3e8f3 --- /dev/null +++ b/models/silver/core/programs/silver__custom_programs.sql @@ -0,0 +1,388 @@ +{{ config( + materialized = 'ephemeral' +) }} + +WITH credits_aleo AS ( + + SELECT + 0 AS deployment_block_id, + '2024-09-04 15:00:00'::TIMESTAMP AS deployment_block_timestamp, + 'credits.aleo' AS program_id, + 0 AS edition, + 'program credits.aleo; + + mapping committee: + key as address.public; + value as committee_state.public; + + struct committee_state: + is_open as boolean; + commission as u8; + + mapping delegated: + key as address.public; + value as u64.public; + + mapping metadata: + key as address.public; + value as u32.public; + + mapping bonded: + key as address.public; + value as bond_state.public; + + struct bond_state: + validator as address; + microcredits as u64; + + mapping unbonding: + key as address.public; + value as unbond_state.public; + + struct unbond_state: + microcredits as u64; + height as u32; + + mapping account: + key as address.public; + value as u64.public; + + mapping withdraw: + key as address.public; + value as address.public; + + record credits: + owner as address.private; + microcredits as u64.private; + + function bond_validator: + input r0 as address.public; + input r1 as u64.public; + input r2 as u8.public; + assert.neq self.signer r0; + gte r1 1000000u64 into r3; + assert.eq r3 true; + gt r2 100u8 into r4; + assert.neq r4 true; + async bond_validator self.signer r0 r1 r2 into r5; + output r5 as credits.aleo/bond_validator.future; + finalize bond_validator: + input r0 as address.public; + input r1 as address.public; + input r2 as u64.public; + input r3 as u8.public; + get.or_use withdraw[r0] r1 into r4; + assert.eq r1 r4; + cast true r3 into r5 as committee_state; + get.or_use committee[r0] r5 into r6; + assert.eq r3 r6.commission; + cast r0 0u64 into r7 as bond_state; + get.or_use bonded[r0] r7 into r8; + assert.eq r8.validator r0; + add r8.microcredits r2 into r9; + cast r0 r9 into r10 as bond_state; + get.or_use delegated[r0] 0u64 into r11; + add r2 r11 into r12; + gte r12 10000000000000u64 into r13; + assert.eq r13 true; + get account[r0] into r14; + sub r14 r2 into r15; + contains committee[r0] into r16; + branch.eq r16 true to validator_in_committee; + set r4 into withdraw[r0]; + gte r2 100000000u64 into r17; + assert.eq r17 true; + get.or_use metadata[aleo1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3ljyzc] 0u32 into r18; + add r18 1u32 into r19; + set r19 into metadata[aleo1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3ljyzc]; + contains unbonding[r0] into r20; + assert.eq r20 false; + position validator_in_committee; + set r6 into committee[r0]; + set r12 into delegated[r0]; + set r10 into bonded[r0]; + set r15 into account[r0]; + + function bond_public: + input r0 as address.public; + input r1 as address.public; + input r2 as u64.public; + gte r2 1000000u64 into r3; + assert.eq r3 true; + assert.neq self.caller r0; + async bond_public self.caller r0 r1 r2 into r4; + output r4 as credits.aleo/bond_public.future; + finalize bond_public: + input r0 as address.public; + input r1 as address.public; + input r2 as address.public; + input r3 as u64.public; + get.or_use withdraw[r0] r2 into r4; + assert.eq r2 r4; + contains bonded[r0] into r5; + branch.eq r5 true to continue_bond_delegator; + set r2 into withdraw[r0]; + cast true 0u8 into r6 as committee_state; + get.or_use committee[r1] r6 into r7; + assert.eq r7.is_open true; + get.or_use metadata[aleo1qgqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqanmpl0] 0u32 into r8; + add r8 1u32 into r9; + lte r9 100000u32 into r10; + assert.eq r10 true; + set r9 into metadata[aleo1qgqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqanmpl0]; + position continue_bond_delegator; + cast r1 0u64 into r11 as bond_state; + get.or_use bonded[r0] r11 into r12; + assert.eq r12.validator r1; + add r12.microcredits r3 into r13; + gte r13 10000000000u64 into r14; + assert.eq r14 true; + cast r1 r13 into r15 as bond_state; + get account[r0] into r16; + sub r16 r3 into r17; + get.or_use delegated[r1] 0u64 into r18; + add r3 r18 into r19; + contains unbonding[r1] into r20; + assert.eq r20 false; + set r15 into bonded[r0]; + set r17 into account[r0]; + set r19 into delegated[r1]; + + function unbond_public: + input r0 as address.public; + input r1 as u64.public; + async unbond_public self.caller r0 r1 into r2; + output r2 as credits.aleo/unbond_public.future; + finalize unbond_public: + input r0 as address.public; + input r1 as address.public; + input r2 as u64.public; + add block.height 360u32 into r3; + cast 0u64 r3 into r4 as unbond_state; + get bonded[r1] into r5; + get withdraw[r1] into r6; + is.eq r0 r6 into r7; + contains withdraw[r5.validator] into r8; + get.or_use withdraw[r5.validator] aleo1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3ljyzc into r9; + is.eq r0 r9 into r10; + and r8 r10 into r11; + or r7 r11 into r12; + assert.eq r12 true; + is.eq r5.validator r1 into r13; + branch.eq r13 true to unbond_validator; + get.or_use unbonding[r1] r4 into r14; + get delegated[r5.validator] into r15; + sub r5.microcredits r2 into r16; + lt r16 10000000000u64 into r17; + or r11 r17 into r18; + ternary r18 r5.microcredits r2 into r19; + add r14.microcredits r19 into r20; + cast r20 r3 into r21 as unbond_state; + set r21 into unbonding[r1]; + sub r15 r19 into r22; + set r22 into delegated[r5.validator]; + branch.eq r18 true to remove_delegator; + cast r5.validator r16 into r23 as bond_state; + set r23 into bonded[r1]; + branch.eq true true to end_unbond_delegator; + position remove_delegator; + remove bonded[r1]; + get metadata[aleo1qgqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqanmpl0] into r24; + sub r24 1u32 into r25; + set r25 into metadata[aleo1qgqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqanmpl0]; + position end_unbond_delegator; + gte r22 10000000000000u64 into r26; + branch.eq r26 true to end; + position unbond_validator; + contains committee[r5.validator] into r27; + nor r13 r27 into r28; + branch.eq r28 true to end; + get committee[r5.validator] into r29; + get bonded[r5.validator] into r30; + get delegated[r5.validator] into r31; + lt r31 10000000000000u64 into r32; + branch.eq r32 true to remove_validator; + sub r31 r2 into r33; + sub r30.microcredits r2 into r34; + gte r34 100000000u64 into r35; + gte r33 10000000000000u64 into r36; + and r35 r36 into r37; + branch.eq r37 false to remove_validator; + get.or_use unbonding[r5.validator] r4 into r38; + add r38.microcredits r2 into r39; + cast r39 r3 into r40 as unbond_state; + set r40 into unbonding[r5.validator]; + set r33 into delegated[r5.validator]; + cast r5.validator r34 into r41 as bond_state; + set r41 into bonded[r5.validator]; + branch.eq true true to end; + position remove_validator; + remove committee[r5.validator]; + get metadata[aleo1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3ljyzc] into r42; + sub r42 1u32 into r43; + set r43 into metadata[aleo1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3ljyzc]; + sub r31 r30.microcredits into r44; + set r44 into delegated[r5.validator]; + remove bonded[r5.validator]; + get.or_use unbonding[r5.validator] r4 into r45; + add r30.microcredits r45.microcredits into r46; + cast r46 r3 into r47 as unbond_state; + set r47 into unbonding[r5.validator]; + position end; + + function claim_unbond_public: + input r0 as address.public; + async claim_unbond_public r0 into r1; + output r1 as credits.aleo/claim_unbond_public.future; + finalize claim_unbond_public: + input r0 as address.public; + get unbonding[r0] into r1; + gte block.height r1.height into r2; + assert.eq r2 true; + get withdraw[r0] into r3; + get.or_use account[r3] 0u64 into r4; + add r1.microcredits r4 into r5; + set r5 into account[r3]; + remove unbonding[r0]; + contains bonded[r0] into r6; + branch.eq r6 true to end; + remove withdraw[r0]; + position end; + + function set_validator_state: + input r0 as boolean.public; + async set_validator_state self.caller r0 into r1; + output r1 as credits.aleo/set_validator_state.future; + finalize set_validator_state: + input r0 as address.public; + input r1 as boolean.public; + get committee[r0] into r2; + cast r1 r2.commission into r3 as committee_state; + set r3 into committee[r0]; + + function transfer_public: + input r0 as address.public; + input r1 as u64.public; + async transfer_public self.caller r0 r1 into r2; + output r2 as credits.aleo/transfer_public.future; + finalize transfer_public: + input r0 as address.public; + input r1 as address.public; + input r2 as u64.public; + get account[r0] into r3; + sub r3 r2 into r4; + set r4 into account[r0]; + get.or_use account[r1] 0u64 into r5; + add r5 r2 into r6; + set r6 into account[r1]; + + function transfer_public_as_signer: + input r0 as address.public; + input r1 as u64.public; + async transfer_public_as_signer self.signer r0 r1 into r2; + output r2 as credits.aleo/transfer_public_as_signer.future; + finalize transfer_public_as_signer: + input r0 as address.public; + input r1 as address.public; + input r2 as u64.public; + get account[r0] into r3; + sub r3 r2 into r4; + set r4 into account[r0]; + get.or_use account[r1] 0u64 into r5; + add r5 r2 into r6; + set r6 into account[r1]; + + function transfer_private: + input r0 as credits.record; + input r1 as address.private; + input r2 as u64.private; + sub r0.microcredits r2 into r3; + cast r1 r2 into r4 as credits.record; + cast r0.owner r3 into r5 as credits.record; + output r4 as credits.record; + output r5 as credits.record; + + function transfer_private_to_public: + input r0 as credits.record; + input r1 as address.public; + input r2 as u64.public; + sub r0.microcredits r2 into r3; + cast r0.owner r3 into r4 as credits.record; + async transfer_private_to_public r1 r2 into r5; + output r4 as credits.record; + output r5 as credits.aleo/transfer_private_to_public.future; + finalize transfer_private_to_public: + input r0 as address.public; + input r1 as u64.public; + get.or_use account[r0] 0u64 into r2; + add r1 r2 into r3; + set r3 into account[r0]; + + function transfer_public_to_private: + input r0 as address.private; + input r1 as u64.public; + cast r0 r1 into r2 as credits.record; + async transfer_public_to_private self.caller r1 into r3; + output r2 as credits.record; + output r3 as credits.aleo/transfer_public_to_private.future; + finalize transfer_public_to_private: + input r0 as address.public; + input r1 as u64.public; + get account[r0] into r2; + sub r2 r1 into r3; + set r3 into account[r0]; + + function join: + input r0 as credits.record; + input r1 as credits.record; + add r0.microcredits r1.microcredits into r2; + cast r0.owner r2 into r3 as credits.record; + output r3 as credits.record; + + function split: + input r0 as credits.record; + input r1 as u64.private; + sub r0.microcredits r1 into r2; + sub r2 10000u64 into r3; + cast r0.owner r1 into r4 as credits.record; + cast r0.owner r3 into r5 as credits.record; + output r4 as credits.record; + output r5 as credits.record; + + function fee_private: + input r0 as credits.record; + input r1 as u64.public; + input r2 as u64.public; + input r3 as field.public; + assert.neq r1 0u64; + assert.neq r3 0field; + add r1 r2 into r4; + sub r0.microcredits r4 into r5; + cast r0.owner r5 into r6 as credits.record; + output r6 as credits.record; + + function fee_public: + input r0 as u64.public; + input r1 as u64.public; + input r2 as field.public; + assert.neq r0 0u64; + assert.neq r2 0field; + add r0 r1 into r3; + async fee_public self.signer r3 into r4; + output r4 as credits.aleo/fee_public.future; + finalize fee_public: + input r0 as address.public; + input r1 as u64.public; + get account[r0] into r2; + sub r2 r1 into r3; + set r3 into account[r0];' AS program, + NULL AS verifying_keys, + {{ dbt_utils.generate_surrogate_key( + ['program_id','edition'] + ) }} AS programs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +) + +SELECT * FROM credits_aleo \ No newline at end of file diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index 284dddf..3513f95 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -32,33 +32,20 @@ FROM WITH base AS ( SELECT DATA, - -- IN FINAL DATA :header AS header, - -- IN FINAL header :metadata :height :: INT AS block_id, - -- IN FINAL header :metadata :timestamp :: datetime AS block_timestamp, DATA :block_hash :: STRING AS block_hash, DATA :previous_hash :: STRING AS previous_hash, - -- IN FINAL COALESCE(ARRAY_SIZE(DATA :transactions) :: NUMBER, 0) AS tx_count, - -- IN FINAL header :metadata :network AS network_id, header :metadata :coinbase_target :: bigint AS coinbase_target, header :metadata :cumulative_proof_target :: bigint AS cumulative_proof_target, header :metadata :cumulative_weight :: bigint AS cumulative_weight, - -- IN FINAL header :metadata :round :: INT AS ROUND, - -- use to identify address of block producer (validator) -- IN FINAL object_keys( DATA :authority :subdag :subdag - ) AS rounds, - -- IN FINAL, REPLACES PROPOSER ADDRESS - {# DATA :transactions AS transactions, - -- IN FINAL - DATA :ratifications AS block_rewards, - -- puzzle rewards (provers) and staker rewards (block reward). puzzle rewards are split by weight -- IN FINAL - DATA :solutions :solutions :solutions AS puzzle_solutions -- target is the proportion of prover rewards #} + ) AS rounds FROM {% if is_incremental() %} diff --git a/models/silver/core/silver__labels.sql b/models/silver/core/silver__labels.sql new file mode 100644 index 0000000..8639f6e --- /dev/null +++ b/models/silver/core/silver__labels.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'view', + tags = ['core'] +) }} + +SELECT + system_created_at, + insert_date, + blockchain, + address, + creator, + label_type, + label_subtype, + address_name, + project_name, + modified_timestamp +FROM + {{ source( + 'crosschain', + 'dim_labels' + ) }} +WHERE + blockchain = 'aleo' \ No newline at end of file diff --git a/models/silver/core/silver__programs.sql b/models/silver/core/silver__programs.sql index 49012d5..96ff447 100644 --- a/models/silver/core/silver__programs.sql +++ b/models/silver/core/silver__programs.sql @@ -46,7 +46,6 @@ SELECT TRY_PARSE_JSON( deployment_msg :verifying_keys ) AS verifying_keys, - {# mappings, #} {{ dbt_utils.generate_surrogate_key( ['program_id','edition'] ) }} AS programs_id, diff --git a/models/silver/core/silver__programs.yml b/models/silver/core/silver__programs.yml new file mode 100644 index 0000000..482c91b --- /dev/null +++ b/models/silver/core/silver__programs.yml @@ -0,0 +1,40 @@ +version: 2 + +models: + - name: silver__programs + description: "This model contains detailed information about programs deployed on the Aleo blockchain." + columns: + - name: deployment_block_id + description: "The ID of the block in which the program was deployed." + tests: + - not_null + + - name: deployment_block_timestamp + description: "The timestamp of the block in which the program was deployed." + tests: + - not_null + + - name: program_id + description: "The unique identifier of the program." + tests: + - not_null + - unique + + - name: edition + description: "The version or edition number of the program." + tests: + - not_null + + - name: program + description: "The full source code of the program." + tests: + - not_null + + - name: verifying_keys + description: "The verifying keys associated with the program, if any." + + - name: programs_id + description: "A unique identifier for each program record." + tests: + - not_null + - unique \ No newline at end of file diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index e50eb4c..1e93443 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key = "tx_id", + unique_key = "transactions_id", incremental_strategy = 'merge', incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], merge_exclude_columns = ["inserted_timestamp"], diff --git a/models/silver/core/silver__transitions.sql b/models/silver/core/silver__transitions.sql index 30211c6..29206e9 100644 --- a/models/silver/core/silver__transitions.sql +++ b/models/silver/core/silver__transitions.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key = "transition_id", + unique_key = "transitions_id", incremental_strategy = 'merge', incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], merge_exclude_columns = ["inserted_timestamp"], diff --git a/models/sources.yml b/models/sources.yml index e15a830..5055eb5 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -5,16 +5,13 @@ sources: database: "{{ 'crosschain' if target.database == 'aleo' else 'crosschain_dev' }}" schema: core tables: - - name: dim_date_hours - - name: address_tags - - name: dim_dates + - name: dim_labels - name: crosschain_silver database: "{{ 'crosschain' if target.database == 'ALEO' else 'crosschain_dev' }}" schema: silver tables: - name: number_sequence - - name: labels_combined - name: bronze_streamline database: streamline @@ -23,6 +20,7 @@ sources: tables: - name: blocks - name: transactions + - name: github_actions database: ALEO schema: github_actions diff --git a/models/streamline/silver/core/complete/streamline__transactions_complete.sql b/models/streamline/silver/core/complete/streamline__transactions_complete.sql deleted file mode 100644 index 683662a..0000000 --- a/models/streamline/silver/core/complete/streamline__transactions_complete.sql +++ /dev/null @@ -1,55 +0,0 @@ -{{ config ( - materialized = "incremental", - incremental_strategy = 'merge', - unique_key = "complete_transactions_id", - enabled = false -) }} --- depends_on: {{ ref('bronze__transactions') }} -WITH transactions AS ( - - SELECT - VALUE :BLOCK_ID_REQUESTED :: INT AS block_id, - DATA :id :: STRING AS transaction_id, - {{ dbt_utils.generate_surrogate_key( - ['VALUE:BLOCK_ID_REQUESTED :: INT', 'DATA :id :: STRING'] - ) }} AS complete_transactions_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - file_name, - '{{ invocation_id }}' AS _invocation_id, - ROW_NUMBER() over ( - PARTITION BY DATA :id - ORDER BY - inserted_timestamp DESC - ) AS rn - FROM - -{% if is_incremental() %} -{{ ref('bronze__transactions') }} -{% else %} - {{ ref('bronze__transactions_FR') }} -{% endif %} - -{% if is_incremental() %} -WHERE - inserted_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) -{% endif %} -) -SELECT - block_id, - transaction_id, - complete_transactions_id, - inserted_timestamp, - modified_timestamp, - file_name, - _invocation_id -FROM - transactions -WHERE - rn = 1 - AND block_id IS NOT NULL diff --git a/models/streamline/silver/core/realtime/streamline__transactions_realtime.sql b/models/streamline/silver/core/realtime/streamline__transactions_realtime.sql deleted file mode 100644 index 891ece9..0000000 --- a/models/streamline/silver/core/realtime/streamline__transactions_realtime.sql +++ /dev/null @@ -1,80 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = fsc_utils.if_data_call_function_v2( - func = 'streamline.udf_bulk_rest_api_v2', - target = "{{this.schema}}.{{this.identifier}}", - params ={ "external_table" :"transactions", - "sql_limit" :"500", - "producer_batch_size" :"100", - "worker_batch_size" :"100", - "sql_source" :"{{this.identifier}}" } - ), - enabled = false -) }} --- depends_on: {{ ref('streamline__transactions_complete') }} -WITH blocks AS ( - - SELECT - block_id, - block_timestamp, - transactions, - tx_count - FROM - {{ ref("silver__blocks") }} - WHERE - tx_count > 0 -), -transaction_ids AS ( - SELECT - b.block_id, - b.block_timestamp, - t.value :transaction :id :: STRING AS transaction_id - FROM - blocks b, - TABLE(FLATTEN(PARSE_JSON(transactions))) t - WHERE - t.value :transaction :id IS NOT NULL), - tx_to_pull AS ( - SELECT - A.* - FROM - transaction_ids A - LEFT JOIN {{ ref('streamline_transactions_complete') }} - ) ( - SELECT - block_id, - block_timestamp, - transaction_id, - {{ target.database }}.live.udf_api( - 'GET', - '{Service}/transaction/' || transaction_id, - OBJECT_CONSTRUCT( - 'Content-Type', - 'application/json' - ),{}, - 'Vault/dev/aleo/mainnet' - ) AS request, - block_id AS block_id_requested - FROM - transaction_ids - ) b - SELECT - ROUND( - block_id, - -4 - ) :: INT AS partition_key, - block_timestamp, - {{ target.database }}.live.udf_api( - 'GET', - '{Service}/transaction/' || transaction_id, - OBJECT_CONSTRUCT( - 'Content-Type', - 'application/json' - ),{}, - 'Vault/dev/aleo/mainnet' - ) AS request, - block_id AS block_id_requested - FROM - transaction_ids - ORDER BY - block_id From 19000ef9c6fe7a01f8d3eb0dc1d589df407a89b7 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 15 Oct 2024 14:19:51 -0600 Subject: [PATCH 2/6] fixes --- models/gold/core/core__dim_labels.sql | 2 +- models/gold/core/core__dim_programs.sql | 57 ++++++++++++++++--------- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/models/gold/core/core__dim_labels.sql b/models/gold/core/core__dim_labels.sql index 652de2a..0a7587b 100644 --- a/models/gold/core/core__dim_labels.sql +++ b/models/gold/core/core__dim_labels.sql @@ -8,7 +8,7 @@ ) }} SELECT - 'lava' AS blockchain, + 'aleo' AS blockchain, creator, address, address_name, diff --git a/models/gold/core/core__dim_programs.sql b/models/gold/core/core__dim_programs.sql index f7ca147..5be07a0 100644 --- a/models/gold/core/core__dim_programs.sql +++ b/models/gold/core/core__dim_programs.sql @@ -6,23 +6,42 @@ tags = ['core','full_test'] ) }} -SELECT - deployment_block_id, - deployment_block_timestamp, - program_id, - edition, - program, - verifying_keys, - {{ dbt_utils.generate_surrogate_key( - ['program_id'] - ) }} AS dim_program_id, - SYSDATE() AS insert_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS invocation_id -FROM - {{ ref('silver__programs') }} +WITH base AS ( + SELECT + deployment_block_id, + deployment_block_timestamp, + program_id, + edition, + program, + verifying_keys, + {{ dbt_utils.generate_surrogate_key( + ['program_id'] + ) }} AS dim_program_id, + SYSDATE() AS insert_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id + FROM + {{ ref('silver__programs') }} + + {% if is_incremental() %} + WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) + {% endif %} +), + +custom_programs AS ( + SELECT + * + FROM + {{ ref('silver__custom_programs') }} +) +SELECT * FROM base UNION ALL -SELECT - * -FROM - {{ ref('silver__custom_programs') }} \ No newline at end of file +SELECT * FROM custom_programs \ No newline at end of file From 52023943623d2be200e8735c58f02d0bb5b38485 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 15 Oct 2024 14:34:45 -0600 Subject: [PATCH 3/6] added workflows and seed file --- .github/workflows/dbt_alter_gha_task.yml | 46 ++++++++++++++++++ .github/workflows/dbt_run_deployment.yml | 47 +++++++++++++++++++ .github/workflows/dbt_run_non_core.yml | 46 ++++++++++++++++++ .../dbt_run_streamline_blocks_realtime.yml | 4 +- .github/workflows/dbt_test_monthly.yml | 45 ++++++++++++++++++ .github/workflows/dbt_test_recent.yml | 47 +++++++++++++++++++ .github/workflows/dbt_test_tasks.yml | 27 +++++++++++ data/github_actions__workflows.csv | 6 +++ 8 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/dbt_alter_gha_task.yml create mode 100644 .github/workflows/dbt_run_deployment.yml create mode 100644 .github/workflows/dbt_run_non_core.yml create mode 100644 .github/workflows/dbt_test_monthly.yml create mode 100644 .github/workflows/dbt_test_recent.yml create mode 100644 .github/workflows/dbt_test_tasks.yml create mode 100644 data/github_actions__workflows.csv diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml new file mode 100644 index 0000000..8f00313 --- /dev/null +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -0,0 +1,46 @@ +name: dbt_alter_gha_task +run-name: dbt_alter_gha_task + +on: + workflow_dispatch: + branches: + - "main" + inputs: + workflow_name: + type: string + description: Name of the workflow to perform the action on, no .yml extension + required: true + task_action: + type: choice + description: Action to perform + required: true + options: + - SUSPEND + - RESUME + default: SUSPEND + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 + with: + workflow_name: | + ${{ inputs.workflow_name }} + task_action: | + ${{ inputs.task_action }} + environment: workflow_prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml new file mode 100644 index 0000000..f69e52b --- /dev/null +++ b/.github/workflows/dbt_run_deployment.yml @@ -0,0 +1,47 @@ +name: dbt_run_deployment +run-name: dbt_run_deployment + +on: + workflow_dispatch: + branches: + - "main" + inputs: + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + - DBT_EMERGENCY + default: DBT + dbt_command: + type: string + description: 'DBT Run Command' + required: true + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ inputs.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@AN-4374/upgrade-dbt-1.7 + with: + dbt_command: | + ${{ inputs.dbt_command }} + environment: workflow_prod + warehouse: ${{ inputs.WAREHOUSE }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_non_core.yml b/.github/workflows/dbt_run_non_core.yml new file mode 100644 index 0000000..b73199a --- /dev/null +++ b/.github/workflows/dbt_run_non_core.yml @@ -0,0 +1,46 @@ +name: dbt_run_non_core +run-name: dbt_run_non_core + +on: + workflow_dispatch: + # schedule: + # # Runs "at minute 7 and 37, every hour" (see https://crontab.guru) + # - cron: '10 * * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m "aleo_models,tag:noncore" diff --git a/.github/workflows/dbt_run_streamline_blocks_realtime.yml b/.github/workflows/dbt_run_streamline_blocks_realtime.yml index 8258583..64ff54c 100644 --- a/.github/workflows/dbt_run_streamline_blocks_realtime.yml +++ b/.github/workflows/dbt_run_streamline_blocks_realtime.yml @@ -5,8 +5,8 @@ on: workflow_dispatch: branches: - "main" - schedule: - - cron: '0 * * * *' + #schedule: + # - cron: '0 * * * *' env: DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" diff --git a/.github/workflows/dbt_test_monthly.yml b/.github/workflows/dbt_test_monthly.yml new file mode 100644 index 0000000..a60098c --- /dev/null +++ b/.github/workflows/dbt_test_monthly.yml @@ -0,0 +1,45 @@ +name: dbt_test_monthly +run-name: dbt_test_monthly + +on: + workflow_dispatch: + schedule: + # Runs 2nd of month at 6AM” (see https://crontab.guru) + - cron: '0 6 2 * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt test -m "aleo_models,tag:full_test" \ No newline at end of file diff --git a/.github/workflows/dbt_test_recent.yml b/.github/workflows/dbt_test_recent.yml new file mode 100644 index 0000000..1be646c --- /dev/null +++ b/.github/workflows/dbt_test_recent.yml @@ -0,0 +1,47 @@ +name: dbt_test_recent +run-name: dbt_test_recent + +on: + workflow_dispatch: + # schedule: + # # Daily at 8:30am UTC + # - cron: '30 8 * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m "aleo_models,tag:recent_test" + dbt test -m "aleo_models,tag:recent_test" \ No newline at end of file diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml new file mode 100644 index 0000000..8adb7f5 --- /dev/null +++ b/.github/workflows/dbt_test_tasks.yml @@ -0,0 +1,27 @@ +name: dbt_test_tasks +run-name: dbt_test_tasks + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7 + secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv new file mode 100644 index 0000000..b0707f4 --- /dev/null +++ b/data/github_actions__workflows.csv @@ -0,0 +1,6 @@ +workflow_name,workflow_schedule +dbt_run_streamline_blocks_realtime,"0,20,40 * * * *" +dbt_run_core,"10 * * * *" +dbt_run_non_core,"30,50 * * * *" +dbt_test_recent,"20 */4 * * *" +dbt_test_tasks,"0,30 * * * *" \ No newline at end of file From acaada31383d2daf5b59ab2515144c480a3dc8a8 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 15 Oct 2024 14:46:09 -0600 Subject: [PATCH 4/6] updated runtimes to 2x/hour --- data/github_actions__workflows.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index b0707f4..b0edfc4 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,6 +1,6 @@ workflow_name,workflow_schedule -dbt_run_streamline_blocks_realtime,"0,20,40 * * * *" -dbt_run_core,"10 * * * *" +dbt_run_streamline_blocks_realtime,"5,35 * * * *" +dbt_run_core,"20,50 * * * *" dbt_run_non_core,"30,50 * * * *" dbt_test_recent,"20 */4 * * *" dbt_test_tasks,"0,30 * * * *" \ No newline at end of file From 090590489ca00ea739833f0e5de902ea5fdf5937 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 15 Oct 2024 15:05:10 -0600 Subject: [PATCH 5/6] updated workflows and added integration testing at AE's request --- .github/workflows/dbt_integration_test.yml | 17 ++++++++ .github/workflows/dbt_run_deployment.yml | 47 ---------------------- .github/workflows/dbt_run_non_core.yml | 46 --------------------- .github/workflows/dbt_test_recent.yml | 47 ---------------------- data/github_actions__workflows.csv | 2 - selectors.yml | 8 ++++ 6 files changed, 25 insertions(+), 142 deletions(-) create mode 100644 .github/workflows/dbt_integration_test.yml delete mode 100644 .github/workflows/dbt_run_deployment.yml delete mode 100644 .github/workflows/dbt_run_non_core.yml delete mode 100644 .github/workflows/dbt_test_recent.yml create mode 100644 selectors.yml diff --git a/.github/workflows/dbt_integration_test.yml b/.github/workflows/dbt_integration_test.yml new file mode 100644 index 0000000..1646d61 --- /dev/null +++ b/.github/workflows/dbt_integration_test.yml @@ -0,0 +1,17 @@ +name: dbt_run_integration_test +run-name: ${{ github.event.inputs.branch }} + +on: + workflow_dispatch: + +concurrency: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt.yml@main + with: + command: > + dbt test --selector 'integration_tests' + environment: ${{ github.ref == 'refs/heads/main' && 'workflow_prod' || 'workflow_dev' }} + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml deleted file mode 100644 index f69e52b..0000000 --- a/.github/workflows/dbt_run_deployment.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: dbt_run_deployment -run-name: dbt_run_deployment - -on: - workflow_dispatch: - branches: - - "main" - inputs: - warehouse: - type: choice - description: Snowflake warehouse - required: true - options: - - DBT - - DBT_CLOUD - - DBT_EMERGENCY - default: DBT - dbt_command: - type: string - description: 'DBT Run Command' - required: true - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ inputs.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: | - ${{ inputs.dbt_command }} - environment: workflow_prod - warehouse: ${{ inputs.WAREHOUSE }} - secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_non_core.yml b/.github/workflows/dbt_run_non_core.yml deleted file mode 100644 index b73199a..0000000 --- a/.github/workflows/dbt_run_non_core.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: dbt_run_non_core -run-name: dbt_run_non_core - -on: - workflow_dispatch: - # schedule: - # # Runs "at minute 7 and 37, every hour" (see https://crontab.guru) - # - cron: '10 * * * *' - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "${{ vars.PYTHON_VERSION }}" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -m "aleo_models,tag:noncore" diff --git a/.github/workflows/dbt_test_recent.yml b/.github/workflows/dbt_test_recent.yml deleted file mode 100644 index 1be646c..0000000 --- a/.github/workflows/dbt_test_recent.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: dbt_test_recent -run-name: dbt_test_recent - -on: - workflow_dispatch: - # schedule: - # # Daily at 8:30am UTC - # - cron: '30 8 * * *' - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "${{ vars.PYTHON_VERSION }}" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -m "aleo_models,tag:recent_test" - dbt test -m "aleo_models,tag:recent_test" \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index b0edfc4..ec08fac 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,6 +1,4 @@ workflow_name,workflow_schedule dbt_run_streamline_blocks_realtime,"5,35 * * * *" dbt_run_core,"20,50 * * * *" -dbt_run_non_core,"30,50 * * * *" -dbt_test_recent,"20 */4 * * *" dbt_test_tasks,"0,30 * * * *" \ No newline at end of file diff --git a/selectors.yml b/selectors.yml new file mode 100644 index 0000000..4561b77 --- /dev/null +++ b/selectors.yml @@ -0,0 +1,8 @@ +selectors: + - name: integration_tests + description: "Selector for integration tests" + definition: + union: + - method: fqn + value: "livequery_models.deploy.core.*" + \ No newline at end of file From 4734a4749aa15f872ddec0979af1bd123b6d037d Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 15 Oct 2024 15:12:02 -0600 Subject: [PATCH 6/6] updated sp clone --- .github/workflows/dbt_run_dev_refresh.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 45b5edc..c5d6bfd 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip install -r requirements.txt workstream-dbt-core + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: |