From 30ce62f6a0b20752551025cadffab67766fcaf56 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 1 Jul 2024 13:27:42 -0600 Subject: [PATCH] AN-4936/Backfill Migration Final (#335) * del cw models * core views * defi views * gov view * nft_views * rm ref to old cw models * del internal cols * nv 13-15 * nv 13-15 * nv 17-18 * nv 20 * del temp logic * upd tests on allday metadata * upd to warn --- .github/workflows/dbt_run_scheduled.yml | 2 +- .../workflows/dbt_run_scheduled_non_core.yml | 2 +- dbt_project.yml | 2 +- .../gold/core/core__dim_contract_labels.sql | 55 +- models/gold/core/core__ez_token_transfers.sql | 79 +-- models/gold/core/core__ez_token_transfers.yml | 1 - models/gold/core/core__fact_blocks.sql | 74 +-- models/gold/core/core__fact_events.sql | 69 +-- models/gold/core/core__fact_transactions.sql | 106 +--- models/gold/core/core__fact_transactions.yml | 1 + .../gold/defi/defi__dim_swap_pool_labels.sql | 60 +-- .../defi/defi__ez_bridge_transactions.sql | 78 +-- models/gold/defi/defi__ez_swaps.sql | 67 +-- models/gold/defi/defi__ez_swaps.yml | 3 +- models/gold/gov/gov__ez_staking_actions.sql | 70 +-- models/gold/nft/nft__dim_allday_metadata.sql | 38 +- models/gold/nft/nft__dim_allday_metadata.yml | 114 +--- models/gold/nft/nft__dim_moment_metadata.sql | 105 +--- models/gold/nft/nft__ez_nft_sales.sql | 76 +-- models/gold/nft/nft__ez_nft_sales.yml | 5 +- .../silver__streamline_transactions_final.sql | 7 +- .../defi/silver__swaps_single_trade_s.sql | 2 +- .../silver/labels/silver__labels_pools_s.sql | 2 +- ...query__topshot_moments_metadata_needed.sql | 4 +- models/silver_cw/README.md | 9 - models/silver_cw/core/silver__blocks.sql | 85 --- models/silver_cw/core/silver__blocks.yml | 36 -- .../core/silver__event_attributes.sql | 170 ------ .../core/silver__event_attributes_https.sql | 154 ------ .../core/silver__event_attributes_https.yml | 114 ---- models/silver_cw/core/silver__events.sql | 100 ---- models/silver_cw/core/silver__events.yml | 116 ---- .../silver_cw/core/silver__events_final.sql | 183 ------- .../silver_cw/core/silver__events_final.yml | 37 -- .../silver_cw/core/silver__transactions.sql | 123 ----- .../silver_cw/core/silver__transactions.yml | 52 -- .../labels/silver__contract_labels.sql | 52 -- .../labels/silver__contract_labels.yml | 16 - .../silver_cw/labels/silver__labels_pools.sql | 66 --- .../silver_cw/labels/silver__labels_pools.yml | 49 -- .../labels/silver__labels_pools_metapier.sql | 120 ----- .../labels/silver__labels_pools_metapier.yml | 50 -- models/silver_cw/nft/silver__nft_giglabs.sql | 337 ------------ models/silver_cw/nft/silver__nft_giglabs.yml | 74 --- .../nft/silver__nft_moment_editions.sql | 50 -- .../nft/silver__nft_moment_editions.yml | 32 -- .../nft/silver__nft_moment_metadata.sql | 75 --- .../nft/silver__nft_moment_metadata.yml | 22 - .../nft/silver__nft_moment_metadata_final.sql | 80 --- .../nft/silver__nft_moment_metadata_final.yml | 39 -- .../nft/silver__nft_moment_minted.sql | 42 -- .../nft/silver__nft_moment_minted.yml | 32 -- .../nft/silver__nft_moment_minted_2.sql | 43 -- .../nft/silver__nft_moment_minted_2.yml | 35 -- .../nft/silver__nft_moment_series.sql | 42 -- .../nft/silver__nft_moment_series.yml | 30 -- .../silver_cw/nft/silver__nft_moment_set.sql | 44 -- .../silver_cw/nft/silver__nft_moment_set.yml | 24 - models/silver_cw/nft/silver__nft_moments.sql | 47 -- models/silver_cw/nft/silver__nft_sales.sql | 116 ---- models/silver_cw/nft/silver__nft_sales.yml | 52 -- .../nft/silver__nft_topshot_pack_sales.sql | 66 --- .../nft/silver__nft_topshot_pack_sales.yml | 100 ---- .../nft/silver__nft_topshot_sales.sql | 158 ------ .../nft/silver__nft_topshot_sales.yml | 139 ----- ...ver__nft_transactions_secondary_market.sql | 509 ------------------ ...ver__nft_transactions_secondary_market.yml | 175 ------ .../staking/silver__staking_actions.sql | 99 ---- .../staking/silver__staking_actions.yml | 37 -- models/silver_cw/swaps/silver__swaps.sql | 318 ----------- models/silver_cw/swaps/silver__swaps.yml | 112 ---- .../silver_cw/swaps/silver__swaps_events.sql | 65 --- .../silver_cw/swaps/silver__swaps_events.yml | 37 -- .../swaps/silver__swaps_single_trade.sql | 174 ------ .../swaps/silver__swaps_single_trade.yml | 129 ----- .../transfers/silver__bridge_blocto.sql | 298 ---------- .../transfers/silver__bridge_blocto.yml | 131 ----- .../transfers/silver__bridge_celer.sql | 165 ------ .../transfers/silver__bridge_celer.yml | 128 ----- .../transfers/silver__token_transfers.sql | 139 ----- .../transfers/silver__token_transfers.yml | 31 -- tests/tests__moment_metadata_allday.sql | 2 +- tests/tests__moment_metadata_topshot.sql | 2 +- 83 files changed, 120 insertions(+), 6664 deletions(-) delete mode 100644 models/silver_cw/README.md delete mode 100644 models/silver_cw/core/silver__blocks.sql delete mode 100644 models/silver_cw/core/silver__blocks.yml delete mode 100644 models/silver_cw/core/silver__event_attributes.sql delete mode 100644 models/silver_cw/core/silver__event_attributes_https.sql delete mode 100644 models/silver_cw/core/silver__event_attributes_https.yml delete mode 100644 models/silver_cw/core/silver__events.sql delete mode 100644 models/silver_cw/core/silver__events.yml delete mode 100644 models/silver_cw/core/silver__events_final.sql delete mode 100644 models/silver_cw/core/silver__events_final.yml delete mode 100644 models/silver_cw/core/silver__transactions.sql delete mode 100644 models/silver_cw/core/silver__transactions.yml delete mode 100644 models/silver_cw/labels/silver__contract_labels.sql delete mode 100644 models/silver_cw/labels/silver__contract_labels.yml delete mode 100644 models/silver_cw/labels/silver__labels_pools.sql delete mode 100644 models/silver_cw/labels/silver__labels_pools.yml delete mode 100644 models/silver_cw/labels/silver__labels_pools_metapier.sql delete mode 100644 models/silver_cw/labels/silver__labels_pools_metapier.yml delete mode 100644 models/silver_cw/nft/silver__nft_giglabs.sql delete mode 100644 models/silver_cw/nft/silver__nft_giglabs.yml delete mode 100644 models/silver_cw/nft/silver__nft_moment_editions.sql delete mode 100644 models/silver_cw/nft/silver__nft_moment_editions.yml delete mode 100644 models/silver_cw/nft/silver__nft_moment_metadata.sql delete mode 100644 models/silver_cw/nft/silver__nft_moment_metadata.yml delete mode 100644 models/silver_cw/nft/silver__nft_moment_metadata_final.sql delete mode 100644 models/silver_cw/nft/silver__nft_moment_metadata_final.yml delete mode 100644 models/silver_cw/nft/silver__nft_moment_minted.sql delete mode 100644 models/silver_cw/nft/silver__nft_moment_minted.yml delete mode 100644 models/silver_cw/nft/silver__nft_moment_minted_2.sql delete mode 100644 models/silver_cw/nft/silver__nft_moment_minted_2.yml delete mode 100644 models/silver_cw/nft/silver__nft_moment_series.sql delete mode 100644 models/silver_cw/nft/silver__nft_moment_series.yml delete mode 100644 models/silver_cw/nft/silver__nft_moment_set.sql delete mode 100644 models/silver_cw/nft/silver__nft_moment_set.yml delete mode 100644 models/silver_cw/nft/silver__nft_moments.sql delete mode 100644 models/silver_cw/nft/silver__nft_sales.sql delete mode 100644 models/silver_cw/nft/silver__nft_sales.yml delete mode 100644 models/silver_cw/nft/silver__nft_topshot_pack_sales.sql delete mode 100644 models/silver_cw/nft/silver__nft_topshot_pack_sales.yml delete mode 100644 models/silver_cw/nft/silver__nft_topshot_sales.sql delete mode 100644 models/silver_cw/nft/silver__nft_topshot_sales.yml delete mode 100644 models/silver_cw/nft/silver__nft_transactions_secondary_market.sql delete mode 100644 models/silver_cw/nft/silver__nft_transactions_secondary_market.yml delete mode 100644 models/silver_cw/staking/silver__staking_actions.sql delete mode 100644 models/silver_cw/staking/silver__staking_actions.yml delete mode 100644 models/silver_cw/swaps/silver__swaps.sql delete mode 100644 models/silver_cw/swaps/silver__swaps.yml delete mode 100644 models/silver_cw/swaps/silver__swaps_events.sql delete mode 100644 models/silver_cw/swaps/silver__swaps_events.yml delete mode 100644 models/silver_cw/swaps/silver__swaps_single_trade.sql delete mode 100644 models/silver_cw/swaps/silver__swaps_single_trade.yml delete mode 100644 models/silver_cw/transfers/silver__bridge_blocto.sql delete mode 100644 models/silver_cw/transfers/silver__bridge_blocto.yml delete mode 100644 models/silver_cw/transfers/silver__bridge_celer.sql delete mode 100644 models/silver_cw/transfers/silver__bridge_celer.yml delete mode 100644 models/silver_cw/transfers/silver__token_transfers.sql delete mode 100644 models/silver_cw/transfers/silver__token_transfers.yml diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index d00d4841..90b6ce41 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -48,7 +48,7 @@ jobs: run: > dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; dbt seed; - dbt run -s tag:scheduled_core tag:streamline_complete "flow_models,models/gold" --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' + dbt run -s tag:scheduled_core tag:streamline_complete "flow_models,models/gold" - name: Store logs uses: actions/upload-artifact@v3 diff --git a/.github/workflows/dbt_run_scheduled_non_core.yml b/.github/workflows/dbt_run_scheduled_non_core.yml index 24f01c27..b9b6b42d 100644 --- a/.github/workflows/dbt_run_scheduled_non_core.yml +++ b/.github/workflows/dbt_run_scheduled_non_core.yml @@ -44,7 +44,7 @@ jobs: run: > dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; dbt seed; - dbt run -s tag:scheduled_non_core --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' + dbt run -s tag:scheduled_non_core - name: Store logs uses: actions/upload-artifact@v3 diff --git a/dbt_project.yml b/dbt_project.yml index 704bf2d5..c8f0559a 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ # and underscores. A good package name should reflect your organization's # name or the intended use of these models name: "flow_models" -version: "1.7.0" +version: "1.8.0" config-version: 2 require-dbt-version: ">=1.7.0" diff --git a/models/gold/core/core__dim_contract_labels.sql b/models/gold/core/core__dim_contract_labels.sql index 6fe0c368..2f9c4be2 100644 --- a/models/gold/core/core__dim_contract_labels.sql +++ b/models/gold/core/core__dim_contract_labels.sql @@ -3,61 +3,16 @@ tags = ['scheduled'] ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS event_contract_id, - event_contract, - contract_name, - account_address, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__contract_labels') }} -), -streamline AS ( - SELECT - event_contract_id, - event_contract, - contract_name, - account_address, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__contract_labels_s') }} -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT + event_contract_id, event_contract, contract_name, account_address, + inserted_timestamp, + modified_timestamp, COALESCE ( event_contract_id, {{ dbt_utils.generate_surrogate_key(['event_contract']) }} - ) AS dim_contract_labels_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + ) AS dim_contract_labels_id FROM - FINAL qualify ROW_NUMBER() over ( - PARTITION BY event_contract - ORDER BY - _inserted_timestamp DESC - ) = 1 + {{ ref('silver__contract_labels_s') }} diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index cdcb304e..dc46fda8 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -3,68 +3,6 @@ tags = ['ez', 'scheduled'] ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS token_transfers_id, - block_height, - block_timestamp, - tx_id, - sender, - recipient, - token_contract, - amount, - tx_succeeded, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__token_transfers') }} - WHERE - token_contract NOT IN ( - 'A.c38aea683c0c4d38.ZelosAccountingToken', - 'A.f1b97c06745f37ad.SwapPair' - ) - AND block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - token_transfers_id, - block_height, - block_timestamp, - tx_id, - sender, - recipient, - token_contract, - amount, - tx_succeeded, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__token_transfers_s') }} - WHERE - token_contract NOT IN ( - 'A.c38aea683c0c4d38.ZelosAccountingToken', - 'A.f1b97c06745f37ad.SwapPair' - ) - AND block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - * - FROM - streamline - UNION ALL - SELECT - * - FROM - chainwalkers -) SELECT block_height, block_timestamp, @@ -80,13 +18,12 @@ SELECT ['tx_id','sender', 'recipient','token_contract', 'amount'] ) }} ) AS ez_token_transfers_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL \ No newline at end of file + {{ ref('silver__token_transfers_s') }} +WHERE + token_contract NOT IN ( + 'A.c38aea683c0c4d38.ZelosAccountingToken', + 'A.f1b97c06745f37ad.SwapPair' + ) diff --git a/models/gold/core/core__ez_token_transfers.yml b/models/gold/core/core__ez_token_transfers.yml index 3e8a3246..09a3db62 100644 --- a/models/gold/core/core__ez_token_transfers.yml +++ b/models/gold/core/core__ez_token_transfers.yml @@ -15,7 +15,6 @@ models: - amount config: severity: warn - error_if: ">10" columns: - name: TX_ID diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index b349778c..f9c355ed 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -3,67 +3,12 @@ tags = ['scheduled'] ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS blocks_id, - block_height, - block_timestamp, - network, - network_version, - chain_id, - tx_count, - id, - parent_id, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__blocks') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - blocks_id, - block_height, - block_timestamp, - 'mainnet' AS network, - network_version, - 'flow' AS chain_id, - tx_count, - id, - parent_id, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__streamline_blocks') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT - blocks_id, - block_height, + block_height :: INT AS block_height, block_timestamp, - network, + 'mainnet' AS network, network_version, - chain_id, + 'flow' AS chain_id, tx_count, id, parent_id, @@ -71,13 +16,8 @@ SELECT blocks_id, {{ dbt_utils.generate_surrogate_key(['block_height']) }} ) AS fact_blocks_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + _inserted_timestamp, + inserted_timestamp, + modified_timestamp FROM - FINAL + {{ ref('silver__streamline_blocks') }} diff --git a/models/gold/core/core__fact_events.sql b/models/gold/core/core__fact_events.sql index 4cf16100..ef5bbd1a 100644 --- a/models/gold/core/core__fact_events.sql +++ b/models/gold/core/core__fact_events.sql @@ -3,64 +3,10 @@ tags = ['scheduled'] ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS streamline_event_id, - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__events_final') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - streamline_event_id, - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__streamline_events') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT tx_id, block_timestamp, - block_height, + block_height :: INT AS block_height, tx_succeeded, event_index, event_contract, @@ -70,13 +16,8 @@ SELECT streamline_event_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS fact_events_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + _inserted_timestamp, + inserted_timestamp, + modified_timestamp FROM - FINAL \ No newline at end of file + {{ ref('silver__streamline_events') }} diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 56d38cde..e37b7573 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -3,88 +3,11 @@ tags = ['scheduled'] ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS streamline_transaction_id, - tx_id, - block_timestamp, - block_height, - chain_id, - tx_index, - proposer, - payer, - authorizers, - count_authorizers, - gas_limit, - NULL AS script, - NULL AS arguments, - transaction_result, - tx_succeeded, - error_msg, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__transactions') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - streamline_transaction_id, - tx_id, - block_timestamp, - block_height, - 'flow' AS chain_id, - NULL AS tx_index, - proposer, - payer, - authorizers, - count_authorizers, - gas_limit, - script, - arguments, - OBJECT_CONSTRUCT( - 'error', - error_message, - 'events', - events, - 'status', - status - ) AS transaction_result, - tx_succeeded, - error_message AS error_msg, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__streamline_transactions_final') }} - WHERE - NOT pending_result_response - AND block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT tx_id, block_timestamp, - block_height, - chain_id, - tx_index, + block_height :: INT AS block_height, + 'flow' AS chain_id, proposer, payer, authorizers, @@ -92,20 +15,23 @@ SELECT gas_limit, script, arguments, - transaction_result, + OBJECT_CONSTRUCT( + 'error', + error_message, + 'events', + events, + 'status', + status + ) AS transaction_result, tx_succeeded, - error_msg, + error_message AS error_msg, COALESCE ( streamline_transaction_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS fact_transactions_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL \ No newline at end of file + {{ ref('silver__streamline_transactions_final') }} +WHERE + NOT pending_result_response diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml index bcf72f96..e7865a7e 100644 --- a/models/gold/core/core__fact_transactions.yml +++ b/models/gold/core/core__fact_transactions.yml @@ -98,6 +98,7 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - VARIANT + - OBJECT - name: TX_SUCCEEDED description: "{{ doc('tx_succeeded') }}" diff --git a/models/gold/defi/defi__dim_swap_pool_labels.sql b/models/gold/defi/defi__dim_swap_pool_labels.sql index 0f46764a..b4f2ae0e 100644 --- a/models/gold/defi/defi__dim_swap_pool_labels.sql +++ b/models/gold/defi/defi__dim_swap_pool_labels.sql @@ -3,40 +3,8 @@ tag = ['scheduled'] ) }} -WITH pairs_cw AS ( +WITH pairs_s AS ( - SELECT - tx_id, - NULL AS labels_pools_metapier_id, - swap_contract, - deployment_timestamp, - token0_contract, - token1_contract, - pool_id, - vault_address, - NULL AS inserted_timestamp, - _inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__labels_pools') }} -), -metapier_cw AS ( - SELECT - tx_id, - NULL AS labels_pools_metapier_id, - swap_contract, - deployment_timestamp, - token0_contract, - token1_contract, - pool_id, - vault_address, - NULL AS inserted_timestamp, - _inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__labels_pools_metapier') }} -), -pairs_s AS ( SELECT tx_id, labels_pools_id AS labels_pools_metapier_id, @@ -47,7 +15,6 @@ pairs_s AS ( pool_id, vault_address, inserted_timestamp, - _inserted_timestamp, modified_timestamp FROM {{ ref('silver__labels_pools_s') }} @@ -63,27 +30,16 @@ metapier_s AS ( pool_id, vault_address, inserted_timestamp, - _inserted_timestamp, modified_timestamp FROM {{ ref('silver__labels_pools_metapier_s') }} ), FINAL AS ( - SELECT - * - FROM - pairs_cw - UNION - SELECT - * - FROM - metapier_cw - UNION SELECT * FROM pairs_s - UNION + UNION ALL SELECT * FROM @@ -100,13 +56,7 @@ SELECT labels_pools_metapier_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS dim_swap_pool_labels_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL \ No newline at end of file + FINAL diff --git a/models/gold/defi/defi__ez_bridge_transactions.sql b/models/gold/defi/defi__ez_bridge_transactions.sql index 32b55233..03098c68 100644 --- a/models/gold/defi/defi__ez_bridge_transactions.sql +++ b/models/gold/defi/defi__ez_bridge_transactions.sql @@ -4,84 +4,20 @@ meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'BRIDGE' }} } ) }} -WITH blocto_cw AS ( - - SELECT - * - FROM - {{ ref('silver__bridge_blocto') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -celer_cw AS ( - SELECT - * - FROM - {{ ref('silver__bridge_celer') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), +WITH blocto_s AS ( SELECT * FROM {{ ref('silver__bridge_blocto_s') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} ), celer_s AS ( SELECT * FROM {{ ref('silver__bridge_celer_s') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} ), combo AS ( - SELECT - NULL AS bridge_id, - tx_id, - block_timestamp, - block_height, - teleport_contract AS bridge_contract, - token_contract, - gross_amount AS amount, - flow_wallet_address, - blockchain, - teleport_direction AS direction, - bridge, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - blocto_cw - UNION ALL - SELECT - NULL AS bridge_id, - tx_id, - block_timestamp, - block_height, - bridge_contract, - token_contract, - amount, - flow_wallet_address, - blockchain, - direction, - bridge, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - celer_cw - UNION ALL SELECT bridge_blocto_id AS bridge_id, tx_id, @@ -94,7 +30,6 @@ combo AS ( blockchain, teleport_direction AS direction, bridge, - _inserted_timestamp, inserted_timestamp, modified_timestamp FROM @@ -112,7 +47,6 @@ combo AS ( blockchain, direction, bridge, - _inserted_timestamp, inserted_timestamp, modified_timestamp FROM @@ -133,13 +67,7 @@ SELECT bridge_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS ez_bridge_transactions_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM combo \ No newline at end of file diff --git a/models/gold/defi/defi__ez_swaps.sql b/models/gold/defi/defi__ez_swaps.sql index a682a379..96a1d781 100644 --- a/models/gold/defi/defi__ez_swaps.sql +++ b/models/gold/defi/defi__ez_swaps.sql @@ -4,67 +4,8 @@ meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }} } ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS swaps_id, - tx_id, - block_timestamp, - block_height, - swap_contract, - swap_index, - trader, - token_out_source, - token_out_contract, - token_out_amount, - token_in_destination, - token_in_contract, - token_in_amount, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__swaps') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - swaps_final_id AS swaps_id, - tx_id, - block_timestamp, - block_height, - swap_contract, - swap_index, - trader, - token_out_source, - token_out_contract, - token_out_amount, - token_in_destination, - token_in_contract, - token_in_amount, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__swaps_final') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT + swaps_final_id AS swaps_id, tx_id, block_timestamp, block_height, @@ -81,9 +22,9 @@ SELECT swaps_id, {{ dbt_utils.generate_surrogate_key(['tx_id', 'swap_index']) }} ) AS ez_swaps_id, - COALESCE(inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL + {{ ref('silver__swaps_final') }} WHERE token_in_contract IS NOT NULL diff --git a/models/gold/defi/defi__ez_swaps.yml b/models/gold/defi/defi__ez_swaps.yml index fd4f0ca2..42063acc 100644 --- a/models/gold/defi/defi__ez_swaps.yml +++ b/models/gold/defi/defi__ez_swaps.yml @@ -34,7 +34,8 @@ models: - name: TRADER description: "{{ doc('trader') }}" tests: - - not_null + - not_null: + severity: warn - name: TOKEN_OUT_SOURCE description: "{{ doc('token_out_source') }}" diff --git a/models/gold/gov/gov__ez_staking_actions.sql b/models/gold/gov/gov__ez_staking_actions.sql index 19b49d47..68e1c5e8 100644 --- a/models/gold/gov/gov__ez_staking_actions.sql +++ b/models/gold/gov/gov__ez_staking_actions.sql @@ -1,65 +1,9 @@ {{ config( materialized = 'view', tags = ['ez', 'scheduled'], - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }}} + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }} } ) }} -WITH chainwalkers AS ( - - SELECT - * - FROM - {{ ref('silver__staking_actions') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - * - FROM - {{ ref('silver__staking_actions_s') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - NULL AS staking_actions_id, - tx_id, - event_index, - block_timestamp, - block_height, - tx_succeeded, - delegator, - action, - amount, - node_id, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - chainwalkers - UNION ALL - SELECT - staking_actions_id, - tx_id, - event_index, - block_timestamp, - block_height, - tx_succeeded, - delegator, - action, - amount, - node_id, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - streamline -) SELECT tx_id, event_index, @@ -74,13 +18,7 @@ SELECT staking_actions_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS ez_staking_actions_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL \ No newline at end of file + {{ ref('silver__staking_actions_s') }} diff --git a/models/gold/nft/nft__dim_allday_metadata.sql b/models/gold/nft/nft__dim_allday_metadata.sql index d85cbfce..3ebe0753 100644 --- a/models/gold/nft/nft__dim_allday_metadata.sql +++ b/models/gold/nft/nft__dim_allday_metadata.sql @@ -6,32 +6,6 @@ WITH allday AS ( - SELECT - NULL AS nft_unique_id, - nft_id, - nft_collection, - nflallday_id, - serial_number, - moment_tier, - total_circulation, - moment_description, - player, - team, - season, - week, - classification, - play_type, - moment_date, - series, - set_name, - video_urls, - moment_stats_full, - _inserted_timestamp, - _inserted_timestamp AS inserted_timestamp, - _inserted_timestamp AS modified_timestamp - FROM - {{ ref('silver__nft_allday_metadata') }} - UNION SELECT nft_allday_metadata_s_id AS nft_unique_id, nft_id, @@ -52,7 +26,6 @@ WITH allday AS ( set_name, video_urls, moment_stats_full, - _inserted_timestamp, inserted_timestamp, modified_timestamp FROM @@ -78,7 +51,6 @@ WITH allday AS ( set_name, video_urls, moment_stats_full, - _inserted_timestamp, _inserted_timestamp AS inserted_timestamp, _inserted_timestamp AS modified_timestamp FROM @@ -107,13 +79,7 @@ SELECT nft_unique_id, {{ dbt_utils.generate_surrogate_key(['nft_id']) }} ) AS dim_allday_metadata_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM allday diff --git a/models/gold/nft/nft__dim_allday_metadata.yml b/models/gold/nft/nft__dim_allday_metadata.yml index 8f5dafaf..270e086f 100644 --- a/models/gold/nft/nft__dim_allday_metadata.yml +++ b/models/gold/nft/nft__dim_allday_metadata.yml @@ -5,99 +5,56 @@ models: description: |- Data for NFL AllDay Moments, including player, team, stats and more. This is produced via API and may differ in structure from metadata available on-chain in the `dim_moment_metadata` table. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - nft_id - severity: error - error_if: ">10" - warn_if: "<10" - columns: - name: NFT_ID description: "{{ doc('nft_id') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" - - unique: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null + - unique - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR - severity: error - error_if: ">10" - warn_if: "<10" + - name: NFT_COLLECTION description: "{{ doc('nft_collection') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR - severity: error - error_if: ">10" - warn_if: "<10" - name: NFLALLDAY_ID description: "{{ doc('nflallday_id') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR - severity: error - error_if: ">10" - warn_if: "<10" - name: SERIAL_NUMBER description: "{{ doc('serial_number') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - severity: error - error_if: ">10" - warn_if: "<10" - name: TOTAL_CIRCULATION description: "{{ doc('total_circulation') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - severity: error - error_if: ">10" - warn_if: "<10" - name: MOMENT_DESCRIPTION description: "{{ doc('moment_description') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -106,10 +63,7 @@ models: - name: PLAYER description: "{{ doc('player') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -118,10 +72,7 @@ models: - name: TEAM description: "{{ doc('team') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -130,10 +81,7 @@ models: - name: SEASON description: "{{ doc('season') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -142,10 +90,7 @@ models: - name: WEEK description: "{{ doc('week') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -154,10 +99,7 @@ models: - name: CLASSIFICATION description: "{{ doc('classification') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -166,10 +108,7 @@ models: - name: PLAY_TYPE description: "{{ doc('play_type') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -178,10 +117,7 @@ models: - name: MOMENT_DATE description: "{{ doc('moment_date') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ @@ -189,10 +125,7 @@ models: - name: SERIES description: "{{ doc('series') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -201,10 +134,7 @@ models: - name: SET_NAME description: "{{ doc('set_name') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -213,10 +143,7 @@ models: - name: VIDEO_URLS description: "{{ doc('video_urls') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - ARRAY @@ -224,10 +151,7 @@ models: - name: MOMENT_STATS_FULL description: "{{ doc('moment_stats_full') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - OBJECT diff --git a/models/gold/nft/nft__dim_moment_metadata.sql b/models/gold/nft/nft__dim_moment_metadata.sql index 47571e8c..de6a520e 100644 --- a/models/gold/nft/nft__dim_moment_metadata.sql +++ b/models/gold/nft/nft__dim_moment_metadata.sql @@ -4,80 +4,8 @@ meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'NFT, ALLDAY, GOLAZOS, TOPSHOT' }} } ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS nft_moment_metadata_id, - event_contract AS nft_collection, - nft_id, - serial_number, - max_mint_size, - play_id, - series_id, - series_name, - set_id, - set_name, - edition_id, - tier, - metadata, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__nft_moment_metadata_final') }} - WHERE - NOT ( - nft_collection = 'A.87ca73a41bb50ad5.Golazos' - AND edition_id = 486 - ) - AND NOT ( - nft_collection = 'A.e4cf4bdc1751c65d.AllDay' - AND edition_id = 1486 - ) -), -streamline AS ( - SELECT - nft_moment_metadata_id, - event_contract AS nft_collection, - nft_id, - serial_number, - max_mint_size, - play_id, - series_id, - series_name, - set_id, - set_name, - edition_id, - tier, - metadata, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__nft_moment_metadata_final_s') }} - WHERE - NOT ( - nft_collection = 'A.87ca73a41bb50ad5.Golazos' - AND edition_id = 486 - ) - AND NOT ( - nft_collection = 'A.e4cf4bdc1751c65d.AllDay' - AND edition_id = 1486 - ) -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT - nft_collection, + event_contract AS nft_collection, nft_id, serial_number, max_mint_size, @@ -95,17 +23,22 @@ SELECT ['nft_collection','edition_id','nft_id'] ) }} ) AS dim_moment_metadata_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL qualify ROW_NUMBER() over ( - PARTITION BY dim_moment_metadata_id - ORDER BY - _inserted_timestamp DESC - ) = 1 + {{ ref('silver__nft_moment_metadata_final_s') }} +WHERE + NOT ( + nft_collection = 'A.87ca73a41bb50ad5.Golazos' + AND edition_id = 486 + ) + AND NOT ( + nft_collection = 'A.e4cf4bdc1751c65d.AllDay' + AND edition_id = 1486 + ) + +qualify ROW_NUMBER() over ( + PARTITION BY dim_moment_metadata_id + ORDER BY + inserted_timestamp DESC +) = 1 diff --git a/models/gold/nft/nft__ez_nft_sales.sql b/models/gold/nft/nft__ez_nft_sales.sql index b70482fe..f53b4332 100644 --- a/models/gold/nft/nft__ez_nft_sales.sql +++ b/models/gold/nft/nft__ez_nft_sales.sql @@ -4,70 +4,6 @@ meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'NFT' }} } ) }} -WITH chainwalkers AS ( - - SELECT - * - FROM - {{ ref('silver__nft_sales') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - * - FROM - {{ ref('silver__nft_sales_s') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - NULL AS nft_sales_id, - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - tokenflow, - counterparties, - NULL AS inserted_timestamp, - _inserted_timestamp, - NULL AS modified_timestamp - FROM - chainwalkers - UNION ALL - SELECT - nft_sales_id, - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - tokenflow, - counterparties, - inserted_timestamp, - _inserted_timestamp, - modified_timestamp - FROM - streamline -) SELECT tx_id, block_height, @@ -86,13 +22,7 @@ SELECT nft_sales_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS ez_nft_sales_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL + {{ ref('silver__nft_sales_s') }} diff --git a/models/gold/nft/nft__ez_nft_sales.yml b/models/gold/nft/nft__ez_nft_sales.yml index ba4f038c..9946b8bb 100644 --- a/models/gold/nft/nft__ez_nft_sales.yml +++ b/models/gold/nft/nft__ez_nft_sales.yml @@ -13,9 +13,8 @@ models: - buyer - nft_collection - nft_id - severity: error - error_if: ">10" - warn_if: "<10" + severity: warn + error_if: ">5000" columns: - name: TX_ID diff --git a/models/silver/core/silver__streamline_transactions_final.sql b/models/silver/core/silver__streamline_transactions_final.sql index 6d619d54..7412789c 100644 --- a/models/silver/core/silver__streamline_transactions_final.sql +++ b/models/silver/core/silver__streamline_transactions_final.sql @@ -30,7 +30,9 @@ %} {% set incr = "" %} {% if is_incremental() %} + {% set incr = """ + WHERE modified_timestamp >= ( SELECT @@ -50,7 +52,7 @@ ) ) """ %} - {% endif %} + {% endif %} {% set run = run_query(query ~ incr) %} {% endif %} @@ -86,6 +88,7 @@ WHERE silver.streamline_transactions_final_intermediate_tmp ) {% endif %} + ), blocks AS ( SELECT @@ -93,6 +96,7 @@ blocks AS ( FROM {{ ref('silver__streamline_blocks') }} + {% if is_incremental() %} WHERE modified_timestamp >= SYSDATE() - INTERVAL '3 days' @@ -103,6 +107,7 @@ WHERE silver.streamline_transactions_final_intermediate_tmp ) {% endif %} + ), FINAL AS ( SELECT diff --git a/models/silver/defi/silver__swaps_single_trade_s.sql b/models/silver/defi/silver__swaps_single_trade_s.sql index 218ef435..bab33201 100644 --- a/models/silver/defi/silver__swaps_single_trade_s.sql +++ b/models/silver/defi/silver__swaps_single_trade_s.sql @@ -129,7 +129,7 @@ trade_data AS ( _inserted_timestamp FROM swaps_single_trade sst - LEFT JOIN {{ ref('silver__contract_labels') }} + LEFT JOIN {{ ref('silver__contract_labels_s') }} l USING (event_contract) WHERE event_type = 'Trade' diff --git a/models/silver/labels/silver__labels_pools_s.sql b/models/silver/labels/silver__labels_pools_s.sql index ffb63afa..5f7bafed 100644 --- a/models/silver/labels/silver__labels_pools_s.sql +++ b/models/silver/labels/silver__labels_pools_s.sql @@ -36,7 +36,7 @@ pair_labels AS ( SELECT * FROM - {{ ref('silver__contract_labels') }} + {{ ref('silver__contract_labels_s') }} WHERE contract_name ILIKE '%swappair%' ), diff --git a/models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql b/models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql index 17376790..cae9d64a 100644 --- a/models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql +++ b/models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql @@ -9,7 +9,7 @@ WITH mints AS ( event_contract, event_data :momentID :: STRING AS moment_id FROM - {{ ref('silver__nft_moments') }} + {{ ref('silver__nft_moments_s') }} WHERE event_contract = 'A.0b2a3299cc857e29.TopShot' AND event_type = 'MomentMinted' @@ -19,7 +19,7 @@ sales AS ( nft_collection AS event_contract, nft_id AS moment_id FROM - {{ ref('silver__nft_sales') }} + {{ ref('silver__nft_sales_s') }} WHERE nft_collection ILIKE '%topshot%' ), diff --git a/models/silver_cw/README.md b/models/silver_cw/README.md deleted file mode 100644 index 612189f4..00000000 --- a/models/silver_cw/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# SILVER_CW / The Streamline Migration - -This directory is a copy of the silver directory with models downstream of the Chainwalkers source. Chainwalkers is being de-commissioned in October 2023, but the Streamline backfill will take some time to finish. These models are thus copied over as an archive of the Chainwalkers data, and to keep the new (streamline) models separate and organized. - -This entire directory may be deleted once Chainwalkers is fully deprecated. - -New models that have been migrated to use Streamline as a source are suffixed with `_s`. No suffix was added for models that do not require a migration (prices, labels, external APIs, etc.). While I would prefer the new models have no suffix, the decision was made to append `_s` to the new streamline models (instead of copying data into new tables and appending `_cw` to the deprecating data) to minimize the touchpoints on prod data. A decision may be made once the CW data is fully deprecated to drop the suffix from all models and rebuild in cleanly named tables. - -The `scheduled` tag will remain on all models that should run on an hourly schedule. 2 new tags have been added for org purposes: `streamline_scheduled` and `chainwalkers_scheduled` to enable clear model selection. These tags were added only where a model was already tagged with `scheduled` to limit net-new. diff --git a/models/silver_cw/core/silver__blocks.sql b/models/silver_cw/core/silver__blocks.sql deleted file mode 100644 index 570d70f3..00000000 --- a/models/silver_cw/core/silver__blocks.sql +++ /dev/null @@ -1,85 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'block_height', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH bronze_blocks AS ( - - SELECT - * - FROM - {{ ref('bronze__blocks') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} - -qualify ROW_NUMBER() over ( - PARTITION BY block_id - ORDER BY - _ingested_at DESC -) = 1 -), -silver_blocks AS ( - SELECT - block_id AS block_height, - block_timestamp, - network, - chain_id, - tx_count, - COALESCE( - header :block_id, - header :block_header :block_id, - header :id - ) :: STRING AS id, - COALESCE( - header :parent_id, - header :parentId, - header :block_header :parent_id - ) :: STRING AS parent_id, - _ingested_at, - _inserted_timestamp - FROM - bronze_blocks -), -network_version AS ( - SELECT - root_height, - network_version, - COALESCE(LAG(root_height) over ( - ORDER BY - network_version DESC) - 1, 'inf' :: FLOAT) AS end_height - FROM - {{ ref('seeds__network_version') }} -), -add_version AS ( - SELECT - block_height, - block_timestamp, - network, - v.network_version, - chain_id, - tx_count, - id, - parent_id, - _ingested_at, - _inserted_timestamp - FROM - silver_blocks b - LEFT JOIN network_version v - ON b.block_height BETWEEN v.root_height - AND v.end_height -) -SELECT - * -FROM - add_version diff --git a/models/silver_cw/core/silver__blocks.yml b/models/silver_cw/core/silver__blocks.yml deleted file mode 100644 index bf704c31..00000000 --- a/models/silver_cw/core/silver__blocks.yml +++ /dev/null @@ -1,36 +0,0 @@ -version: 2 - -models: - - name: silver__blocks - description: Information about blocks on the FLOW network and corresponding metadata. - - columns: - - name: block_height - description: "{{ doc('block_height') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: network - description: "{{ doc('network') }}" - - - name: network_version - description: "{{ doc('network_version') }}" - - - name: chain_id - description: "{{ doc('chain_id') }}" - - - name: tx_count - description: "{{ doc('tx_count') }}" - - - name: id - description: "{{ doc('id') }}" - - - name: parent_id - description: "{{ doc('parent_id') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/core/silver__event_attributes.sql b/models/silver_cw/core/silver__event_attributes.sql deleted file mode 100644 index 7bb506cf..00000000 --- a/models/silver_cw/core/silver__event_attributes.sql +++ /dev/null @@ -1,170 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'attribute_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events') }} - WHERE - _inserted_timestamp :: DATE < '2022-07-18' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -events_data AS ( - SELECT - event_id, - tx_id, - block_timestamp, - event_index, - event_contract, - event_type, - _event_data_type, - _event_data_fields, - _ingested_at, - COALESCE( - _event_data_type :fields, - _event_data_type :Fields - ) AS event_data_type_fields, - _inserted_timestamp - FROM - events -), -attributes AS ( - SELECT - event_id, - tx_id, - block_timestamp, - event_index, - event_contract, - event_type, - COALESCE( - VALUE :identifier, - VALUE :Identifier - ) :: STRING AS attribute_key, - COALESCE( - _event_data_fields [index] :Value, - _event_data_fields [index] - ) AS attribute_value, - concat_ws( - '-', - event_id, - INDEX - ) AS attribute_id, - INDEX AS attribute_index, - _ingested_at, - _inserted_timestamp - FROM - events_data, - LATERAL FLATTEN( - input => event_data_type_fields - ) -), -handle_address_arrays AS ( - SELECT - attribute_id, - b.index, - LPAD(TRIM(to_char(b.value :: INT, 'XXXXXXX')) :: STRING, 2, '0') AS hex - FROM - attributes A, - TABLE(FLATTEN(attribute_value, recursive => TRUE)) b - WHERE - IS_ARRAY(attribute_value) = TRUE - ORDER BY - 1, - 2 -), -recombine_address AS ( - SELECT - attribute_id, - CONCAT( - '0x', - ARRAY_TO_STRING(ARRAY_AGG(hex) within GROUP ( - ORDER BY - INDEX ASC), '') - ) AS decoded_address - FROM - handle_address_arrays - GROUP BY - 1 -), -replace_arrays AS ( - SELECT - A.attribute_id, - event_id, - tx_id, - block_timestamp, - event_index, - attribute_index, - event_contract, - event_type, - attribute_key, - attribute_value, - decoded_address, - COALESCE( - decoded_address, - attribute_value - ) :: STRING AS attribute_value_adj, - _ingested_at, - _inserted_timestamp - FROM - attributes A - LEFT JOIN recombine_address USING (attribute_id) -), -address_adjustment AS ( - SELECT - attribute_id, - LENGTH(attribute_value_adj) AS ava_len, - CONCAT( - '0x', - LPAD(SPLIT(attribute_value_adj, '0x') [1], 16, '0') :: STRING - ) AS address_adj - FROM - replace_arrays - WHERE - attribute_value_adj LIKE '0x%' - AND ava_len < 19 -), -FINAL AS ( - SELECT - A.attribute_id, - event_id, - tx_id, - block_timestamp, - event_index, - attribute_index, - event_contract, - event_type, - attribute_key, - decoded_address, - attribute_value, - REPLACE( - COALESCE( - address_adj, - attribute_value_adj - ), - '"' - ) AS attribute_value_adj, - _ingested_at, - _inserted_timestamp - FROM - replace_arrays A - LEFT JOIN address_adjustment USING (attribute_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/core/silver__event_attributes_https.sql b/models/silver_cw/core/silver__event_attributes_https.sql deleted file mode 100644 index 08c1d033..00000000 --- a/models/silver_cw/core/silver__event_attributes_https.sql +++ /dev/null @@ -1,154 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'attribute_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events') }} - WHERE - _inserted_timestamp :: DATE >= '2022-07-18' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -payload_method AS ( - SELECT - * - FROM - events - WHERE - _try_parse_payload IS NOT NULL -), -events_data AS ( - SELECT - event_id, - tx_id, - block_timestamp, - event_index, - event_contract, - event_type, - _event_data_type, - _event_data_fields, - _ingested_at, - COALESCE( - _event_data_type :fields, - _event_data_type :Fields - ) AS event_data_type_fields, - _inserted_timestamp - FROM - events - WHERE - _try_parse_payload IS NULL -), -attributes AS ( - SELECT - event_id, - tx_id, - block_timestamp, - event_index, - event_contract, - event_type, - COALESCE( - VALUE :identifier, - VALUE :Identifier - ) :: STRING AS attribute_key, - COALESCE( - _event_data_fields [index] :fields, - _event_data_fields [index] :staticType :typeID, - _event_data_fields [index] :value :value, - _event_data_fields [index] :value, - _event_data_fields [index] :Value, - _event_data_fields [index] - ) :: STRING AS attribute_value, - concat_ws( - '-', - event_id, - INDEX - ) AS attribute_id, - INDEX AS attribute_index, - _ingested_at, - _inserted_timestamp, - 'attributes' AS _cte - FROM - events_data, - LATERAL FLATTEN( - input => event_data_type_fields - ) -), -attributes_2 AS ( - SELECT - event_id, - tx_id, - block_timestamp, - event_index, - event_contract, - event_type, - VALUE :name :: STRING AS attribute_key, - COALESCE( - VALUE :value :value :fields, - VALUE :value :value :staticType, - VALUE :value :value :value :value, - VALUE :value :value :value, - VALUE :value :value - ) :: STRING AS attribute_value, - concat_ws( - '-', - event_id, - INDEX - ) AS attribute_id, - INDEX AS attribute_index, - _ingested_at, - _inserted_timestamp, - 'attributes_2' AS _cte - FROM - payload_method, - LATERAL FLATTEN( - _try_parse_payload :value :fields - ) -), -combo AS ( - SELECT - * - FROM - attributes - UNION - SELECT - * - FROM - attributes_2 -), -FINAL AS ( - SELECT - attribute_id, - event_id, - tx_id, - block_timestamp, - event_index, - attribute_index, - event_contract, - event_type, - attribute_key, - attribute_value, - _ingested_at, - _inserted_timestamp, - _cte - FROM - combo -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/core/silver__event_attributes_https.yml b/models/silver_cw/core/silver__event_attributes_https.yml deleted file mode 100644 index e1f08470..00000000 --- a/models/silver_cw/core/silver__event_attributes_https.yml +++ /dev/null @@ -1,114 +0,0 @@ -version: 2 - -models: - - name: silver__event_attributes_https - description: |- - This table cleans and transform attributes from each event in the events table. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - event_index - - attribute_index - - columns: - - name: attribute_id - description: "{{ doc('attribute_id') }}" - tests: - - not_null - - unique - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: event_id - description: "{{ doc('event_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: event_index - description: "{{ doc('event_index') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: attribute_index - description: "{{ doc('attribute_index') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: event_contract - description: "{{ doc('event_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: event_type - description: "{{ doc('event_type') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: attribute_key - description: "{{ doc('attribute_key') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: decoded_address - description: "{{ doc('decoded_address') }}" - - - name: attribute_value - description: "{{ doc('attribute_value') }}" - - - name: attribute_value_adj - description: "{{ doc('attribute_value_adj') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/core/silver__events.sql b/models/silver_cw/core/silver__events.sql deleted file mode 100644 index 98285c7e..00000000 --- a/models/silver_cw/core/silver__events.sql +++ /dev/null @@ -1,100 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'event_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH transactions AS ( - - SELECT - * - FROM - {{ ref('silver__transactions') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -events AS ( - SELECT - VALUE, - tx_id, - block_timestamp, - block_height, - tx_succeeded, - INDEX AS _index_from_flatten, - COALESCE( - VALUE :event_index, - VALUE :eventIndex - ) :: NUMBER AS event_index, - VALUE :value :: variant AS event_data, - COALESCE( - VALUE :value :EventType, - VALUE :value :eventType - ) :: variant AS event_data_type, - COALESCE( - VALUE :value :Fields, - VALUE :value :fields - ) :: variant AS event_data_fields, - SPLIT( - IFF( - ( - event_data_type :qualifiedIdentifier LIKE 'A.%' - OR event_data_type :qualifiedIdentifier LIKE 'flow%' - ), - event_data_type :qualifiedIdentifier, - VALUE :type - ), - '.' - ) AS type_split, - ARRAY_TO_STRING( - ARRAY_SLICE(type_split, 0, ARRAY_SIZE(type_split) -1), - '.') AS event_contract, - type_split [array_size(type_split)-1] :: STRING AS event_type, - concat_ws( - '-', - tx_id, - event_index - ) AS event_id, - TRY_PARSE_JSON(BASE64_DECODE_STRING(VALUE :payload)) AS try_parse_payload, - _ingested_at, - _inserted_timestamp - FROM - transactions, - LATERAL FLATTEN( - input => transaction_result :events - ) - ), - FINAL AS ( - SELECT - event_id, - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - event_data_type AS _event_data_type, - event_data_fields AS _event_data_fields, - try_parse_payload AS _try_parse_payload, - _event_data_type :fields AS _attribute_fields, - _index_from_flatten, - _ingested_at, - _inserted_timestamp - FROM - events - ) - SELECT - * - FROM - FINAL diff --git a/models/silver_cw/core/silver__events.yml b/models/silver_cw/core/silver__events.yml deleted file mode 100644 index c703cc77..00000000 --- a/models/silver_cw/core/silver__events.yml +++ /dev/null @@ -1,116 +0,0 @@ -version: 2 - -models: - - name: silver__events - description: |- - This table records events from each transaction on the FLOW blockchain. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - event_index - - columns: - - name: event_id - description: "{{ doc('event_id') }}" - tests: - - not_null - - unique - - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: event_index - description: "{{ doc('event_index') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: event_contract - description: "{{ doc('event_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: event_type - description: "{{ doc('event_type') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: event_data - description: "{{ doc('event_data') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - - - name: _event_data_type - description: "{{ doc('_event_data_type') }}" - tests: - - not_null: - name: event_data_type__not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - name: event_data_type__expected_type - - - name: _event_data_fields - description: "{{ doc('_event_data_fields') }}" - tests: - - not_null: - name: event_data_fields__not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - name: event_data_fields__expected_type - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/core/silver__events_final.sql b/models/silver_cw/core/silver__events_final.sql deleted file mode 100644 index 91e1218a..00000000 --- a/models/silver_cw/core/silver__events_final.sql +++ /dev/null @@ -1,183 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::date'], - unique_key = "CONCAT_WS('-', tx_id, event_index)", - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH silver_events AS ( - - SELECT - * - FROM - {{ ref('silver__events') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -silver_event_attributes AS ( - SELECT - * - FROM - {{ ref('silver__event_attributes') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -silver_event_attributes_https AS ( - SELECT - * - FROM - {{ ref('silver__event_attributes_https') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -all_event_attributes AS ( - SELECT - attribute_id, - event_id, - tx_id, - block_timestamp, - event_index, - attribute_index, - event_contract, - event_type, - attribute_key, - attribute_value_adj AS attribute_value, - _ingested_at, - _inserted_timestamp - FROM - silver_event_attributes - UNION - SELECT - attribute_id, - event_id, - tx_id, - block_timestamp, - event_index, - attribute_index, - event_contract, - event_type, - attribute_key, - attribute_value, - _ingested_at, - _inserted_timestamp - FROM - silver_event_attributes_https -), -objs AS ( - SELECT - event_id, - OBJECT_AGG( - attribute_key, - IFF( - attribute_value IS NULL, - 'null', - attribute_value - ) :: variant - ) AS event_data - FROM - all_event_attributes - GROUP BY - 1 -), -location_object AS ( - SELECT - event_id, - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - COALESCE( - _event_data_type :location, - _event_data_type :Location - ) AS event_data, - _ingested_at, - _inserted_timestamp - FROM - silver_events - WHERE - _event_data_fields = '[]' - AND tx_id NOT IN ( - SELECT - tx_id - FROM - silver_event_attributes_https - ) -), -gold_events AS ( - SELECT - e.event_id, - e.tx_id, - e.block_timestamp, - e.block_height, - e.tx_succeeded, - e.event_index, - e.event_contract, - e.event_type, - A.event_data, - e._ingested_at, - e._inserted_timestamp - FROM - objs A - LEFT JOIN silver_events e USING (event_id) -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - _ingested_at, - _inserted_timestamp - FROM - gold_events - UNION - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - _ingested_at, - _inserted_timestamp - FROM - location_object -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/core/silver__events_final.yml b/models/silver_cw/core/silver__events_final.yml deleted file mode 100644 index e2964736..00000000 --- a/models/silver_cw/core/silver__events_final.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 2 - -models: - - name: silver__events_final - description: |- - This table records events from each transaction on the FLOW blockchain. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: event_index - description: "{{ doc('event_index') }}" - - - name: event_contract - description: "{{ doc('event_contract') }}" - - - name: event_type - description: "{{ doc('event_type') }}" - - - name: event_data - description: "{{ doc('event_attributes') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/core/silver__transactions.sql b/models/silver_cw/core/silver__transactions.sql deleted file mode 100644 index 4eb73e7d..00000000 --- a/models/silver_cw/core/silver__transactions.sql +++ /dev/null @@ -1,123 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH bronze_txs AS ( - - SELECT - * - FROM - {{ ref('bronze__transactions') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} - -qualify ROW_NUMBER() over ( - PARTITION BY tx_id - ORDER BY - _ingested_at DESC -) = 1 -), -silver_txs AS ( - SELECT - tx_id, - block_timestamp, - block_id AS block_height, - chain_id, - tx_block_index AS tx_index, - COALESCE( - tx :proposal_key :Address, - tx :proposalKeyAddress - ) :: STRING AS proposer, - tx :payer :: STRING AS payer, - tx :authorizers :: ARRAY AS authorizers, - ARRAY_SIZE(authorizers) AS count_authorizers, - COALESCE( - tx :gas_limit, - tx :gasLimit - ) :: NUMBER AS gas_limit, - COALESCE( - tx :transaction_result, - tx :result - ) :: variant AS transaction_result, - CASE - WHEN transaction_result :error = '' THEN TRUE - WHEN transaction_result :error :: STRING IS NULL THEN TRUE - ELSE FALSE - END AS tx_succeeded, - COALESCE( - transaction_result :error, - '' - ) :: STRING AS error_msg, - _ingested_at, - _inserted_timestamp - FROM - bronze_txs -), -concat_authorizers AS ( - SELECT - tx_id, - CONCAT( - '0x', - VALUE - ) AS x_auth - FROM - silver_txs, - LATERAL FLATTEN ( - input => authorizers - ) -), -authorizers_array AS ( - SELECT - tx_id, - ARRAY_AGG(x_auth) AS authorizers - FROM - concat_authorizers - GROUP BY - 1 -), -FINAL AS ( - SELECT - t.tx_id, - block_timestamp, - block_height, - chain_id, - tx_index, - CONCAT( - '0x', - proposer - ) AS proposer, - CONCAT( - '0x', - payer - ) AS payer, - COALESCE( - aa.authorizers, - t.authorizers - ) AS authorizers, - count_authorizers, - gas_limit, - transaction_result, - tx_succeeded, - error_msg, - _ingested_at, - _inserted_timestamp - FROM - silver_txs t - LEFT JOIN authorizers_array aa USING (tx_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/core/silver__transactions.yml b/models/silver_cw/core/silver__transactions.yml deleted file mode 100644 index 5a7d9db7..00000000 --- a/models/silver_cw/core/silver__transactions.yml +++ /dev/null @@ -1,52 +0,0 @@ -version: 2 - -models: - - name: silver__transactions - description: |- - This table records all the transactions of the FLOW blockchain. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: chain_id - description: "{{ doc('chain_id') }}" - - - name: tx_index - description: tbd - - - name: proposer - description: "{{ doc('proposer') }}" - - - name: payer - description: "{{ doc('payer') }}" - - - name: authorizers - description: "{{ doc('authorizers') }}" - - - name: count_authorizers - description: "{{ doc('count_authorizers') }}" - - - name: gas_limit - description: "{{ doc('gas_limit') }}" - - - name: transaction_result - description: "{{ doc('transaction_result') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: error_msg - description: tbd - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/labels/silver__contract_labels.sql b/models/silver_cw/labels/silver__contract_labels.sql deleted file mode 100644 index 076597b7..00000000 --- a/models/silver_cw/labels/silver__contract_labels.sql +++ /dev/null @@ -1,52 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['event_contract'], - unique_key = 'event_contract', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH splt AS ( - - SELECT - event_contract, - SPLIT( - event_contract, - '.' - ) AS ec_s, - _inserted_timestamp - FROM - {{ ref('silver__events') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -FINAL AS ( - SELECT - DISTINCT event_contract, - ec_s [array_size(ec_s)-1] :: STRING AS contract_name, - CONCAT( - '0x', - ec_s [array_size(ec_s)-2] :: STRING - ) AS account_address, - _inserted_timestamp - FROM - splt - WHERE - ec_s [0] != 'flow' -) -SELECT - * -FROM - FINAL qualify ROW_NUMBER() over ( - PARTITION BY event_contract - ORDER BY - _inserted_timestamp DESC - ) = 1 diff --git a/models/silver_cw/labels/silver__contract_labels.yml b/models/silver_cw/labels/silver__contract_labels.yml deleted file mode 100644 index 1a4ac8c5..00000000 --- a/models/silver_cw/labels/silver__contract_labels.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: 2 - -models: - - name: silver__contract_labels - description: |- - This table extracts all contract labels referenced in the events item of a Flow transaction. - - columns: - - name: event_contract - description: "{{ doc('event_contract') }}" - - - name: contract_name - description: "{{ doc('contract_name') }}" - - - name: account_address - description: "{{ doc('account_address') }}" diff --git a/models/silver_cw/labels/silver__labels_pools.sql b/models/silver_cw/labels/silver__labels_pools.sql deleted file mode 100644 index 2d229c00..00000000 --- a/models/silver_cw/labels/silver__labels_pools.sql +++ /dev/null @@ -1,66 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -pair_labels AS ( - SELECT - * - FROM - {{ ref('silver__contract_labels') }} - WHERE - contract_name ILIKE '%swappair%' -), -pair_creation AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :numPairs :: NUMBER AS pair_number, - event_data :pairAddress :: STRING AS account_address, - event_data :token0Key :: STRING AS token0_contract, - event_data :token1Key :: STRING AS token1_contract, - _inserted_timestamp - FROM - events - WHERE - event_type = 'PairCreated' -), -FINAL AS ( - SELECT - tx_id, - block_timestamp AS deployment_timestamp, - pair_number as pool_id, - p.account_address as vault_address, - token0_contract, - token1_contract, - l.event_contract AS swap_contract, - _inserted_timestamp - FROM - pair_creation p - LEFT JOIN pair_labels l USING (account_address) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/labels/silver__labels_pools.yml b/models/silver_cw/labels/silver__labels_pools.yml deleted file mode 100644 index 9a53f2ba..00000000 --- a/models/silver_cw/labels/silver__labels_pools.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: 2 - -models: - - name: silver__labels_pools - description: |- - Looks for new PairCreated events from the Swap pair factory and records info. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - unique - - - name: deployment_timestamp - description: "{{ doc('deployment_timestamp') }}" - tests: - - not_null - - - name: token0_contract - description: "{{ doc('token0_contract') }}" - tests: - - not_null - - - name: token1_contract - description: "{{ doc('token1_contract') }}" - tests: - - not_null - - - name: pool_id - description: "{{ doc('pool_id') }}" - tests: - - not_null - - - name: vault_address - description: "{{ doc('vault_address') }}" - tests: - - not_null - - unique - - - name: swap_contract - description: "{{ doc('swap_contract') }}" - tests: - - not_null - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null diff --git a/models/silver_cw/labels/silver__labels_pools_metapier.sql b/models/silver_cw/labels/silver__labels_pools_metapier.sql deleted file mode 100644 index fb4f6b18..00000000 --- a/models/silver_cw/labels/silver__labels_pools_metapier.sql +++ /dev/null @@ -1,120 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -pierpools AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_type, - event_data :poolId :: STRING AS pool_id, - _inserted_timestamp - FROM - events - WHERE - event_contract = 'A.609e10301860b683.PierSwapFactory' - AND event_type = 'NewPoolCreated' -), -pier_events AS ( - SELECT - * - FROM - events - WHERE - tx_id IN ( - SELECT - DISTINCT tx_id - FROM - pierpools - ) -), -token_withdraws AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_index, - _inserted_timestamp - FROM - pier_events - WHERE - event_type = 'TokensWithdrawn' - AND event_data :amount :: DOUBLE = 0 -), -pairs AS ( - SELECT - tx_id, - block_timestamp AS deployment_timestamp, - event_contract AS token0_contract, - LAG(event_contract) over ( - PARTITION BY tx_id - ORDER BY - event_index - ) AS token1_contract, - _inserted_timestamp - FROM - token_withdraws -), -escrow AS ( - SELECT - tx_id, - event_data :address :: STRING AS vault_address - FROM - pier_events - WHERE - event_contract = 'flow' - AND event_type = 'AccountCreated' -), -pool_addr AS ( - SELECT - tx_id, - event_contract AS swap_contract - FROM - pier_events - WHERE - event_type = 'Mint' -), -FINAL AS ( - SELECT - C.tx_id, - C.deployment_timestamp, - C.token0_contract, - C.token1_contract, - p.pool_id, - e.vault_address, - pa.swap_contract, - C._inserted_timestamp - FROM - pairs C - LEFT JOIN pierpools p USING (tx_id) - LEFT JOIN escrow e USING (tx_id) - LEFT JOIN pool_addr pa USING (tx_id) - WHERE - token1_contract IS NOT NULL -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/labels/silver__labels_pools_metapier.yml b/models/silver_cw/labels/silver__labels_pools_metapier.yml deleted file mode 100644 index be1a85d8..00000000 --- a/models/silver_cw/labels/silver__labels_pools_metapier.yml +++ /dev/null @@ -1,50 +0,0 @@ -version: 2 - -models: - - name: silver__labels_pools_metapier - description: |- - Looks for new PoolCreated events from the Metapier factory and recordes info. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - unique - - - name: deployment_timestamp - description: "{{ doc('deployment_timestamp') }}" - tests: - - not_null - - - name: token0_contract - description: "{{ doc('token0_contract') }}" - tests: - - not_null - - - name: token1_contract - description: "{{ doc('token1_contract') }}" - tests: - - not_null - - - name: pool_id - description: "{{ doc('pool_id') }}" - tests: - - not_null - - unique - - - name: vault_address - description: "{{ doc('vault_address') }}" - tests: - - not_null - - unique - - - name: swap_contract - description: "{{ doc('swap_contract') }}" - tests: - - not_null - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null diff --git a/models/silver_cw/nft/silver__nft_giglabs.sql b/models/silver_cw/nft/silver__nft_giglabs.sql deleted file mode 100644 index d3daa13b..00000000 --- a/models/silver_cw/nft/silver__nft_giglabs.sql +++ /dev/null @@ -1,337 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -mapped_sales AS ( - SELECT - DISTINCT tx_id - FROM - {{ ref('silver__nft_transactions_secondary_market') }} - UNION - SELECT - DISTINCT tx_id - FROM - {{ ref('silver__nft_topshot_sales') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -duc AS ( - SELECT - DISTINCT tx_id - FROM - events - WHERE - event_contract = 'A.ead892083b3e2c6c.DapperUtilityCoin' -), -duc_events AS ( - SELECT - * - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - duc - ) - AND tx_id NOT IN ( - SELECT - tx_id - FROM - mapped_sales - ) -), -duc_transfers AS ( - SELECT - _inserted_timestamp, - tx_id, - COUNT(event_type) AS event_count, - MAX( - event_index + 1 - ) AS max_index - FROM - duc_events - WHERE - event_type IN ( - 'TokensDeposited', - 'TokensWithdrawn', - 'FeesDeducted', - 'ForwardedDeposit' - ) - GROUP BY - _inserted_timestamp, - tx_id - HAVING - event_count = max_index -), -gig_nfts AS ( - SELECT - * - FROM - duc_events - WHERE - tx_id NOT IN ( - SELECT - DISTINCT tx_id - FROM - duc_transfers - ) - AND event_contract ILIKE 'A.329feb3ab062d289%' - AND event_type IN ( - 'Withdraw', - 'Deposit' - ) -), -gig_sales_events AS ( - SELECT - * - FROM - events - WHERE - tx_id IN ( - SELECT - DISTINCT tx_id - FROM - gig_nfts - ) -), -missing_contract AS ( - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - _inserted_timestamp, - event_contract AS currency, - event_data :amount :: DOUBLE AS amount, - event_data :from :: STRING AS forwarded_from, - TRUE AS missing - FROM - gig_sales_events - WHERE - event_index = 0 - AND event_type = 'TokensWithdrawn' -), -purchase_amt AS ( - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - _inserted_timestamp, - 'A.ead892083b3e2c6c.DapperUtilityCoin' AS currency, - event_data :amount :: DOUBLE AS amount, - event_data :from :: STRING AS forwarded_from, - FALSE AS missing - FROM - gig_sales_events - WHERE - event_type = 'ForwardedDeposit' - AND tx_id NOT IN ( - SELECT - tx_id - FROM - missing_contract - ) -), -triage AS ( - SELECT - * - FROM - missing_contract - UNION - SELECT - * - FROM - purchase_amt -), -withdraw_event AS ( - SELECT - tx_id, - block_timestamp, - block_height, - _inserted_timestamp, - event_contract AS nft_collection, - event_data :from :: STRING AS seller, - event_data :id :: STRING AS nft_id - FROM - gig_sales_events - WHERE - event_type = 'Withdraw' - AND event_data :from :: STRING != 'null' -), -deposit_event AS ( - SELECT - tx_id, - block_timestamp, - block_height, - _inserted_timestamp, - event_contract AS nft_collection, - event_data :to :: STRING AS buyer, - event_data :id :: STRING AS nft_id - FROM - gig_sales_events - WHERE - event_type = 'Deposit' - AND event_data :to :: STRING != 'null' -), -gl_sales AS ( - SELECT - p.tx_id, - p.block_timestamp, - p.block_height, - p.tx_succeeded, - p._inserted_timestamp, - 'Gigantik Primary Market' AS marketplace, - p.missing, - p.currency, - p.amount, - p.forwarded_from, - w.seller, - d.buyer, - w.nft_collection, - w.nft_id AS withdraw_nft_id, - d.nft_id AS deposit_nft_id, - w.nft_collection = d.nft_collection AS collection_check, - w.nft_id = d.nft_id AS nft_id_check - FROM - triage p - LEFT JOIN withdraw_event w USING ( - tx_id, - block_timestamp, - block_height, - _inserted_timestamp - ) - LEFT JOIN deposit_event d USING ( - tx_id, - block_timestamp, - block_height, - _inserted_timestamp - ) -), -multi AS ( - SELECT - tx_id, - COUNT( - DISTINCT deposit_nft_id - ) AS nfts - FROM - gl_sales - WHERE - nft_id_check - GROUP BY - 1 -), -giglabs_final AS ( - SELECT - s.tx_id, - block_timestamp, - block_height, - marketplace, - currency, - amount / m.nfts AS price, - seller, - buyer, - nft_collection, - withdraw_nft_id AS nft_id, - m.nfts, - tx_succeeded, - _inserted_timestamp - FROM - gl_sales s - LEFT JOIN multi m USING (tx_id) - WHERE - nft_id_check -), -step_data AS ( - SELECT - tx_id, - event_index, - event_type, - event_data - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - giglabs_final - ) - AND event_type IN ( - 'TokensWithdrawn', - 'TokensDeposited', - 'ForwardedDeposit' - ) -), -counterparty_data AS ( - SELECT - tx_id, - ARRAY_AGG(OBJECT_CONSTRUCT(event_type, event_data)) within GROUP ( - ORDER BY - event_index - ) AS tokenflow, - ARRAY_AGG(COALESCE(event_data :to, event_data :from) :: STRING) within GROUP ( - ORDER BY - event_index - ) AS counterparties - FROM - step_data - GROUP BY - 1 -), -FINAL AS ( - SELECT - s.tx_id, - block_timestamp, - block_height, - marketplace, - currency, - price, - seller, - buyer, - nft_collection, - nft_id, - nfts, - tokenflow, - counterparties, - tx_succeeded, - _inserted_timestamp - FROM - giglabs_final s - LEFT JOIN counterparty_data C USING (tx_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/nft/silver__nft_giglabs.yml b/models/silver_cw/nft/silver__nft_giglabs.yml deleted file mode 100644 index f25b6964..00000000 --- a/models/silver_cw/nft/silver__nft_giglabs.yml +++ /dev/null @@ -1,74 +0,0 @@ -version: 2 - -models: - - name: silver__nft_giglabs - description: |- - NFT primary market sales for Giglabs marketplaces. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - seller - - buyer - - nft_collection - - nft_id - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: marketplace - description: "{{ doc('marketplace') }}" - tests: - - not_null - - - name: nft_collection - description: "{{ doc('nft_collection') }}" - tests: - - not_null - - - name: nft_id - description: "{{ doc('nft_id') }}" - tests: - - not_null - - - name: buyer - description: "{{ doc('buyer') }}" - tests: - - not_null - - - name: seller - description: "{{ doc('seller') }}" - tests: - - not_null - - - name: price - description: "{{ doc('price') }}" - tests: - - not_null - - - name: currency - description: "{{ doc('currency') }}" - tests: - - not_null - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: tokenflow - description: "{{ doc('tokenflow') }}" - - - name: counterparties - description: "{{ doc('counterparties') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_editions.sql b/models/silver_cw/nft/silver__nft_moment_editions.sql deleted file mode 100644 index cdc071f7..00000000 --- a/models/silver_cw/nft/silver__nft_moment_editions.sql +++ /dev/null @@ -1,50 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-', event_contract, edition_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'EditionCreated' - AND event_contract IN ( - 'A.e4cf4bdc1751c65d.AllDay', - 'A.b715b81853fef53f.AllDay', - 'A.87ca73a41bb50ad5.Golazos' - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :id :: STRING AS edition_id, - event_data :maxMintSize :: STRING AS max_mint_size, - event_data :playID :: STRING AS play_id, - event_data :seriesID :: STRING AS series_id, - event_data :setID :: STRING AS set_id, - event_data :tier :: STRING AS tier, - _inserted_timestamp - FROM - events -) -SELECT - * -FROM - org diff --git a/models/silver_cw/nft/silver__nft_moment_editions.yml b/models/silver_cw/nft/silver__nft_moment_editions.yml deleted file mode 100644 index 76b4c000..00000000 --- a/models/silver_cw/nft/silver__nft_moment_editions.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_editions - description: |- - Cleaned EditionCreated events. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - edition_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - - name: edition_id - description: "{{ doc('edition_id') }}" - - name: max_mint_size - description: "{{ doc('max_mint_size') }}" - - name: play_id - description: "{{ doc('play_id') }}" - - name: series_id - description: "{{ doc('series_id') }}" - - name: set_id - description: "{{ doc('set_id') }}" - - name: tier - description: "{{ doc('tier') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_metadata.sql b/models/silver_cw/nft/silver__nft_moment_metadata.sql deleted file mode 100644 index 8634c40e..00000000 --- a/models/silver_cw/nft/silver__nft_moment_metadata.sql +++ /dev/null @@ -1,75 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['play_id'], - unique_key = "concat_ws('-', event_contract, play_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH play_creation AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'PlayCreated' - -{# -currently includes the following contracts -A.c38aea683c0c4d38.Eternal -A.b715b81853fef53f.AllDay -A.67af7ecf76556cd3.ABD -A.0b2a3299cc857e29.TopShot -A.5c0992b465832a94.TKNZ -A.e4cf4bdc1751c65d.AllDay -A.87ca73a41bb50ad5.Golazos - #} - - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -play_metadata AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :id :: NUMBER AS play_id, - VALUE :key :value :: STRING AS column_header, - VALUE :value :value :: STRING AS column_value, - _inserted_timestamp - FROM - play_creation, - LATERAL FLATTEN(input => TRY_PARSE_JSON(event_data :metadata)) -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - play_id, - _inserted_timestamp, - OBJECT_AGG( - column_header :: variant, - column_value :: variant - ) AS metadata - FROM - play_metadata - GROUP BY - 1, - 2, - 3, - 4, - 5 -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/nft/silver__nft_moment_metadata.yml b/models/silver_cw/nft/silver__nft_moment_metadata.yml deleted file mode 100644 index 2d13a406..00000000 --- a/models/silver_cw/nft/silver__nft_moment_metadata.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_metadata - description: |- - Cleaned PlayCreated events, which is what records the metadata on-chain. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - play_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - - name: play_id - description: "{{ doc('play_id') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_metadata_final.sql b/models/silver_cw/nft/silver__nft_moment_metadata_final.sql deleted file mode 100644 index 2982e517..00000000 --- a/models/silver_cw/nft/silver__nft_moment_metadata_final.sql +++ /dev/null @@ -1,80 +0,0 @@ -{{ config( - materialized = 'table', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-',event_contract,edition_id,nft_id)", - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH moments AS ( - - SELECT - * - FROM - {{ ref('silver__nft_moment_minted') }} -), -metadata AS ( - SELECT - * - FROM - {{ ref('silver__nft_moment_metadata') }} -), -editions AS ( - SELECT - * - FROM - {{ ref('silver__nft_moment_editions') }} -), -series AS ( - SELECT - * - FROM - {{ ref('silver__nft_moment_series') }} -), -set_nm AS ( - SELECT - * - FROM - {{ ref('silver__nft_moment_set') }} -), -FINAL AS ( - SELECT - -- tx id and block timestamp don't matter for the final table - m.tx_id, - m.block_timestamp, - m.event_contract, - m.nft_id, - m.serial_number, - e.max_mint_size, - e.play_id, - e.series_id, - s.series_name, - e.set_id, - sn.set_name, - e.edition_id, - e.tier, - pl.metadata, - m._inserted_timestamp, - sn._inserted_timestamp AS _inserted_timestamp_set - FROM - moments m - LEFT JOIN editions e USING ( - event_contract, - edition_id - ) - LEFT JOIN metadata pl USING ( - event_contract, - play_id - ) - LEFT JOIN series s USING ( - event_contract, - series_id - ) - LEFT JOIN set_nm sn USING ( - event_contract, - set_id - ) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/nft/silver__nft_moment_metadata_final.yml b/models/silver_cw/nft/silver__nft_moment_metadata_final.yml deleted file mode 100644 index f7e04782..00000000 --- a/models/silver_cw/nft/silver__nft_moment_metadata_final.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_metadata_final - description: |- - Cleaned PlayCreated events. - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - - name: nft_id - description: "{{ doc('nft_id') }}" - - name: serial_number - description: "{{ doc('serial_number') }}" - - name: max_mint_size - description: "{{ doc('max_mint_size') }}" - - name: play_id - description: "{{ doc('play_id') }}" - - name: series_id - description: "{{ doc('series_id') }}" - - name: series_name - description: "{{ doc('series_name') }}" - - name: set_id - description: "{{ doc('set_id') }}" - - name: set_name - description: "{{ doc('set_name') }}" - - name: edition_id - description: "{{ doc('edition_id') }}" - - name: tier - description: "{{ doc('tier') }}" - - name: metadata - description: "{{ doc('metadata') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null diff --git a/models/silver_cw/nft/silver__nft_moment_minted.sql b/models/silver_cw/nft/silver__nft_moment_minted.sql deleted file mode 100644 index 879f2b49..00000000 --- a/models/silver_cw/nft/silver__nft_moment_minted.sql +++ /dev/null @@ -1,42 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-', event_contract, edition_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'MomentNFTMinted' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :editionID :: STRING AS edition_id, - event_data :id :: STRING AS nft_id, - event_data :serialNumber :: STRING AS serial_number, - _inserted_timestamp - FROM - events -) -SELECT - * -FROM - org diff --git a/models/silver_cw/nft/silver__nft_moment_minted.yml b/models/silver_cw/nft/silver__nft_moment_minted.yml deleted file mode 100644 index a994c458..00000000 --- a/models/silver_cw/nft/silver__nft_moment_minted.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_minted - description: |- - Cleaned MomentNFTMinted events. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - nft_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - tests: - - accepted_values: - values: - - "A.e4cf4bdc1751c65d.AllDay" - - "A.b715b81853fef53f.AllDay" - - "A.87ca73a41bb50ad5.Golazos" - - name: serial_number - description: "{{ doc('serial_number') }}" - - name: edition_id - description: "{{ doc('edition_id') }}" - - name: nft_id - description: "{{ doc('nft_id') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_minted_2.sql b/models/silver_cw/nft/silver__nft_moment_minted_2.sql deleted file mode 100644 index 1974c65a..00000000 --- a/models/silver_cw/nft/silver__nft_moment_minted_2.sql +++ /dev/null @@ -1,43 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-', event_contract, moment_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'MomentMinted' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :momentID :: STRING AS moment_id, - event_data :serialNumber :: STRING AS serial_number, - event_data :seriesID :: STRING AS series_id, - event_data :setID :: STRING AS set_id, - _inserted_timestamp - FROM - events -) -SELECT - * -FROM - org diff --git a/models/silver_cw/nft/silver__nft_moment_minted_2.yml b/models/silver_cw/nft/silver__nft_moment_minted_2.yml deleted file mode 100644 index 88b75c81..00000000 --- a/models/silver_cw/nft/silver__nft_moment_minted_2.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_minted_2 - description: |- - Cleaned MomentNFTMinted events. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - moment_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - tests: - - accepted_values: - values: - - "A.c38aea683c0c4d38.Eternal" - - "A.d4ad4740ee426334.Moments" - - "A.67af7ecf76556cd3.ABD" - - "A.0b2a3299cc857e29.TopShot" - - name: moment_id - description: "{{ doc('moment_id') }}" - - name: serial_number - description: "{{ doc('serial_number') }}" - - name: set_id - description: "{{ doc('set_id') }}" - - name: series_id - description: "{{ doc('series_id') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_series.sql b/models/silver_cw/nft/silver__nft_moment_series.sql deleted file mode 100644 index 36dddb0e..00000000 --- a/models/silver_cw/nft/silver__nft_moment_series.sql +++ /dev/null @@ -1,42 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-', event_contract, series_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'SeriesCreated' - AND ARRAY_CONTAINS('name' :: variant, object_keys(event_data)) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :id :: STRING AS series_id, - event_data :name :: STRING AS series_name, - _inserted_timestamp - FROM - events -) -SELECT - * -FROM - org diff --git a/models/silver_cw/nft/silver__nft_moment_series.yml b/models/silver_cw/nft/silver__nft_moment_series.yml deleted file mode 100644 index 915a0b1e..00000000 --- a/models/silver_cw/nft/silver__nft_moment_series.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_series - description: |- - Cleaned SeriesCreated events. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - series_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - tests: - - accepted_values: - values: - - "A.e4cf4bdc1751c65d.AllDay" - - "A.b715b81853fef53f.AllDay" - - "A.87ca73a41bb50ad5.Golazos" - - name: series_id - description: "{{ doc('series_id') }}" - - name: series_name - description: "{{ doc('series_name') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_set.sql b/models/silver_cw/nft/silver__nft_moment_set.sql deleted file mode 100644 index d96e9ec3..00000000 --- a/models/silver_cw/nft/silver__nft_moment_set.sql +++ /dev/null @@ -1,44 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-', event_contract, set_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'SetCreated' - AND ARRAY_CONTAINS('name' :: variant, object_keys(event_data)) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :id :: STRING AS set_id, - event_data :name :: STRING AS set_name, - _inserted_timestamp - FROM - events - WHERE - set_id IS NOT NULL -) -SELECT - * -FROM - org diff --git a/models/silver_cw/nft/silver__nft_moment_set.yml b/models/silver_cw/nft/silver__nft_moment_set.yml deleted file mode 100644 index a86e2f26..00000000 --- a/models/silver_cw/nft/silver__nft_moment_set.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_set - description: |- - Cleaned SetCreated events. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - set_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - - name: set_id - description: "{{ doc('set_id') }}" - - name: set_name - description: "{{ doc('set_name') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moments.sql b/models/silver_cw/nft/silver__nft_moments.sql deleted file mode 100644 index 93705cfd..00000000 --- a/models/silver_cw/nft/silver__nft_moments.sql +++ /dev/null @@ -1,47 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = "CONCAT_WS('-', tx_id, event_index)", - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -moment_events AS ( - SELECT - * - FROM - events - WHERE - event_type IN ( - 'MomentPurchased', - 'MomentLocked', - 'MomentCreated', - 'MomentNFTBurned', - 'MomentListed', - 'MomentDestroyed', - 'MomentWithdrawn', - 'MomentMinted', - 'MomentNFTMinted' - ) -) -SELECT - * -FROM - moment_events diff --git a/models/silver_cw/nft/silver__nft_sales.sql b/models/silver_cw/nft/silver__nft_sales.sql deleted file mode 100644 index f86e2af2..00000000 --- a/models/silver_cw/nft/silver__nft_sales.sql +++ /dev/null @@ -1,116 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH topshot AS ( - - SELECT - * - FROM - {{ ref('silver__nft_topshot_sales') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -secondary_mkts AS ( - SELECT - * - FROM - {{ ref('silver__nft_transactions_secondary_market') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -giglabs AS ( - SELECT - * - FROM - {{ ref('silver__nft_giglabs') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -combo AS ( - SELECT - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - _inserted_timestamp, - tokenflow, - counterparties - FROM - topshot - UNION - SELECT - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - _inserted_timestamp, - tokenflow, - counterparties - FROM - secondary_mkts - UNION - SELECT - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - _inserted_timestamp, - tokenflow, - counterparties - FROM - giglabs -) -SELECT - * -FROM - combo diff --git a/models/silver_cw/nft/silver__nft_sales.yml b/models/silver_cw/nft/silver__nft_sales.yml deleted file mode 100644 index 5a148695..00000000 --- a/models/silver_cw/nft/silver__nft_sales.yml +++ /dev/null @@ -1,52 +0,0 @@ -version: 2 - -models: - - name: silver__nft_sales - description: |- - NFT market sales on the Flow blockchain. This table will only contain successful transactions, as failed transactions will not have the requisite events to determine it was an attempted NFT purchase. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: marketplace - description: "{{ doc('marketplace') }}" - - - name: nft_collection - description: "{{ doc('nft_collection') }}" - - - name: nft_id - description: "{{ doc('nft_id') }}" - - - name: buyer - description: "{{ doc('buyer') }}" - - - name: seller - description: "{{ doc('seller') }}" - - - name: price - description: "{{ doc('price') }}" - - - name: currency - description: "{{ doc('currency') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: tokenflow - description: "{{ doc('tokenflow') }}" - - - name: counterparties - description: "{{ doc('counterparties') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_topshot_pack_sales.sql b/models/silver_cw/nft/silver__nft_topshot_pack_sales.sql deleted file mode 100644 index 4fe99fc1..00000000 --- a/models/silver_cw/nft/silver__nft_topshot_pack_sales.sql +++ /dev/null @@ -1,66 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'merge', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = ['tx_id','nft_id'], - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH silver_events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -nft_txs AS ( - SELECT - tx_id, - event_index, - block_height, - block_timestamp, - tx_succeeded, - event_data :id :: INT AS nft_id, - _inserted_timestamp - FROM - silver_events - WHERE - event_data :from = '0xe1f2a091f7bb5245' - AND event_contract = 'A.0b2a3299cc857e29.TopShot' -) -SELECT - A.tx_id, - A.block_height, - A.block_timestamp, - 'topshot pack' AS marketplace, - NULL AS nft_collection, - A.nft_id, - b.event_data :to :: STRING buyer, - NULL AS seller, - NULL price, - NULL currency, - A.tx_succeeded, - NULL AS tokenflow, - NULL AS counterparties, - MD5( - CAST(COALESCE(CAST(A.tx_id AS VARCHAR), '') AS VARCHAR) - ) AS pack_id, - A._inserted_timestamp -FROM - nft_txs A - JOIN silver_events b - ON A.tx_id = b.tx_id - AND A.nft_id :: STRING = b.event_data :id :: STRING -WHERE - event_data :to IS NOT NULL - AND A.event_index <> b.event_index diff --git a/models/silver_cw/nft/silver__nft_topshot_pack_sales.yml b/models/silver_cw/nft/silver__nft_topshot_pack_sales.yml deleted file mode 100644 index 7a2bbe17..00000000 --- a/models/silver_cw/nft/silver__nft_topshot_pack_sales.yml +++ /dev/null @@ -1,100 +0,0 @@ -version: 2 - -models: - - name: silver__nft_topshot_pack_sales - description: TopShot pack sales - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: marketplace - description: "{{ doc('marketplace') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: nft_collection - description: "{{ doc('nft_collection') }}" - - - name: nft_id - description: "{{ doc('nft_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - NUMBER - - - name: buyer - description: "{{ doc('buyer') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: seller - description: "{{ doc('seller') }}" - - - name: price - description: "{{ doc('price') }}" - - - name: currency - description: "{{ doc('currency') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: tokenflow - description: "{{ doc('tokenflow') }}" - - - name: counterparties - description: "{{ doc('counterparties') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/nft/silver__nft_topshot_sales.sql b/models/silver_cw/nft/silver__nft_topshot_sales.sql deleted file mode 100644 index 05e35451..00000000 --- a/models/silver_cw/nft/silver__nft_topshot_sales.sql +++ /dev/null @@ -1,158 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH silver_events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -- WHERE - -- event_data :: STRING != '{}' - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -moment_data AS ( - SELECT - block_height, - block_timestamp, - tx_id, - event_contract :: STRING AS marketplace, - event_data :id :: STRING AS nft_id, - event_data :price :: DOUBLE AS price, - event_data :seller :: STRING AS seller, - tx_succeeded, - _ingested_at, - _inserted_timestamp - FROM - silver_events - WHERE - event_type = 'MomentPurchased' - AND event_contract LIKE 'A.c1e4f4f4c4257510%' -- topshot -), -currency_data AS ( - SELECT - tx_id, - event_contract :: STRING AS currency - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - moment_data - ) - AND event_index = 0 -), -nft_data AS ( - SELECT - tx_id, - event_contract :: STRING AS nft_collection, - event_data :to :: STRING AS buyer - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - moment_data - ) - AND event_type = 'Deposit' -), -combo AS ( - SELECT - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - _ingested_at, - _inserted_timestamp - FROM - moment_data - LEFT JOIN currency_data USING (tx_id) - LEFT JOIN nft_data USING (tx_id) -), -step_data AS ( - SELECT - tx_id, - event_index, - event_type, - event_data - FROM - {{ ref('silver__events_final') }} - WHERE - tx_id IN ( - SELECT - tx_id - FROM - combo - ) - AND event_type IN ( - 'TokensWithdrawn', - 'TokensDeposited', - 'ForwardedDeposit' - ) -), -counterparty_data AS ( - SELECT - tx_id, - ARRAY_AGG(OBJECT_CONSTRUCT(event_type, event_data)) within GROUP ( - ORDER BY - event_index - ) AS tokenflow, - ARRAY_AGG(COALESCE(event_data :to, event_data :from) :: STRING) within GROUP ( - ORDER BY - event_index - ) AS counterparties - FROM - step_data - GROUP BY - 1 -), -FINAL AS ( - SELECT - C.tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - _ingested_at, - _inserted_timestamp, - cd.tokenflow, - cd.counterparties - FROM - combo C - LEFT JOIN counterparty_data cd USING (tx_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/nft/silver__nft_topshot_sales.yml b/models/silver_cw/nft/silver__nft_topshot_sales.yml deleted file mode 100644 index 395f2e60..00000000 --- a/models/silver_cw/nft/silver__nft_topshot_sales.yml +++ /dev/null @@ -1,139 +0,0 @@ -version: 2 - -models: - - name: silver__nft_topshot_sales - description: |- - TopShot direct market sales on the Flow blockchain. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: marketplace - description: "{{ doc('marketplace') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: nft_collection - description: "{{ doc('nft_collection') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: nft_id - description: "{{ doc('nft_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - NUMBER - - - name: buyer - description: "{{ doc('buyer') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: seller - description: "{{ doc('seller') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: price - description: "{{ doc('price') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: currency - description: "{{ doc('currency') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: tokenflow - description: "{{ doc('tokenflow') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: counterparties - description: "{{ doc('counterparties') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/nft/silver__nft_transactions_secondary_market.sql b/models/silver_cw/nft/silver__nft_transactions_secondary_market.sql deleted file mode 100644 index a08a6a87..00000000 --- a/models/silver_cw/nft/silver__nft_transactions_secondary_market.sql +++ /dev/null @@ -1,509 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH silver_events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -sale_trigger AS ( - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_contract AS marketplace, - event_data, - COALESCE( - event_data :purchased :: BOOLEAN, - event_data :accepted :: BOOLEAN, - IFF( - event_data :status = 'sold' - OR event_data :status IS NULL, - TRUE, - FALSE - ), - TRUE - ) AS is_purchased, - _ingested_at, - _inserted_timestamp - FROM - silver_events - WHERE - is_purchased - AND -- each market uses a slightly different sale trigger - ( - ( - event_contract = 'A.30cf5dcf6ea8d379.AeraPack' - AND event_type = 'Purchased' - ) - OR ( - event_contract = 'A.8f9231920da9af6d.AFLPack' - AND event_type = 'PackBought' - ) - OR ( - event_contract = 'A.e2e1689b53e92a82.AniqueMarket' - AND event_type = 'CollectiblePurchased' - ) - OR ( - event_contract = 'A.9969d64233d69723.BlockleteMarket_NFT_V2' - AND event_type = 'BlockletePurchased' - ) - OR ( - event_contract = 'A.64f83c60989ce555.ChainmonstersMarketplace' - AND event_type = 'CollectionRemovedSaleOffer' - ) - OR ( - event_contract = 'A.c8c340cebd11f690.DarkCountryMarket' - AND event_type = 'SaleOfferAccepted' - ) - OR ( - event_contract = 'A.921ea449dffec68a.FlovatarMarketplace' - AND event_type IN ( - 'FlovatarPurchased', - 'FlovatarComponentPurchased' - ) - ) - OR ( - event_contract = 'A.09e03b1f871b3513.TheFabricantMarketplace' - AND event_type = 'NFTPurchased' - ) - OR ( - event_contract = 'A.097bafa4e0b48eef.FindMarketAuctionEscrow' - AND event_type = 'EnglishAuction' - ) - OR ( - event_contract = 'A.097bafa4e0b48eef.FindMarketDirectOfferEscrow' - AND event_type = 'DirectOffer' - ) - OR ( - event_contract = 'A.097bafa4e0b48eef.FindMarketSale' - AND event_type = 'Sale' - ) - OR ( - event_contract = 'A.097bafa4e0b48eef.FindPack' - AND event_type = 'Purchased' - ) - OR ( - event_contract = 'A.8b148183c28ff88f.GaiaOrder' - AND event_type = 'OrderClosed' - ) - OR ( - event_contract = 'A.abda6627c70c7f52.GeniaceMarketplace' - AND event_type = 'SaleOfferCompleted' - ) - OR ( - event_contract = 'A.82ed1b9cba5bb1b3.KaratNFTMarket' - AND event_type = 'SaleOfferAccepted' - ) - OR ( - event_contract = 'A.2162bbe13ade251e.MatrixMarketOpenOffer' - AND event_type = 'OfferCompleted' - ) - OR ( - event_contract = 'A.49b8e5d4d66ae880.MintStoreMarketFactory' - AND event_type = 'MintStoreItemPurchased' - ) - OR ( - event_contract = 'A.a49cc0ee46c54bfb.MotoGPNFTStorefront' - AND event_type = 'SaleOfferCompleted' - ) - OR ( - event_contract = 'A.b8ea91944fd51c43.Offers' - AND event_type = 'OfferCompleted' - ) - OR ( - event_contract = 'A.b8ea91944fd51c43.OffersV2' - AND event_type = 'OfferCompleted' - ) - OR ( - event_contract = 'A.856bd81e73e6752b.PonsNftMarketContract' - AND event_type = 'PonsNFTSold' - ) - OR ( - event_contract = 'A.52cbea4e6f616b8e.PublishedNFTStorefront' - AND event_type = 'ListingCompleted' - ) - OR ( - event_contract = 'A.489fcc527edc21cf.TuneGOMarket' - AND event_type = 'SaleOfferAccepted' - ) - OR ( - event_contract = 'A.4eb8a10cb9f87357.NFTStorefront' -- general storefront - AND event_type = 'ListingCompleted' - ) - OR ( - event_contract = 'A.4eb8a10cb9f87357.NFTStorefrontV2' -- funds move in 2ND TOKEN MVMT not FIRST - AND event_type = 'ListingCompleted' - ) - OR ( - event_contract = 'A.85b8bbf926dcddfa.NFTStoreFront' - AND event_type = 'ListingSold' - ) - OR ( - event_contract = 'A.85b075e08d13f697.OlympicPinMarket' - AND event_type = 'PiecePurchased' - ) - OR ( - event_contract = 'A.5b82f21c0edf76e3.StarlyCardMarket' - AND event_type = 'CollectionRemovedSaleOffer' - ) - OR ( - event_contract = 'A.62b3063fbe672fc8.ZeedzMarketplace' - AND event_type = 'RemovedListing' - ) - ) -), -num_triggers AS ( - SELECT - tx_id, - -- storing the marketplace contract interactions - ARRAY_AGG(marketplace) within GROUP ( - ORDER BY - marketplace - ) AS marketplaces, - -- compare total sales (by listing id) with distinct to eliminate the case where 2 marketplaces - -- (general & gaia) are called for 1 sale - ARRAY_AGG( - COALESCE( - event_data :orderId, - -- general - event_data :listingResourceID, - -- gaia, zeedz - event_data :saleItemID, - --chainmonster - event_data :itemID, - --starly, darkcountry - event_data :id, - -- olympic pin, flovatar, fina - event_data :saleOfferResourceID, - -- moto gp - event_data :bidId, - -- matrix - event_data :listingID, - -- fabricant - event_data :templateId, - -- AFLPack - event_data :saleOfferId, - -- tunego - event_data :offerId, - -- OffersV2 - event_data :nftId, - -- pons doesn't do order ids - event_data :packId -- find pack, aera - ) - ) AS sale_ids, - ARRAY_AGG( - DISTINCT COALESCE( - event_data :orderId, - -- general - event_data :listingResourceID, - -- gaia, zeedz - event_data :saleItemID, - --chainmonster - event_data :itemID, - --starly, darkcountry - event_data :id, - -- olympic pin, flovatar, fina - event_data :saleOfferResourceID, - -- moto gp - event_data :bidId, - -- matrix - event_data :listingID, - -- fabricant - event_data :templateId, - -- AFLPack - event_data :saleOfferId, - -- tunego - event_data :offerId, - -- OffersV2 - event_data :nftId, - -- pons doesn't do order ids - event_data :packId -- find pack, aera - ) - ) AS dist_sale_ids, - COUNT(1) AS sale_trigger_count, - ARRAY_SIZE(dist_sale_ids) AS num_sales - FROM - sale_trigger - GROUP BY - 1 -), -omit_nft_nontransfers AS ( - SELECT - tx_id, - ARRAY_AGG( - DISTINCT event_type - ) AS events, - -- don't forget to update below if adding any new movement method ! - ARRAY_SIZE( - array_intersection( - ['Deposit', 'Withdraw', 'FlovatarSaleWithdrawn', 'FlovatarComponentSaleWithdrawn'], - events - ) - ) = 2 AS nft_transferred, - count_if( - event_type = 'Deposit' - ) AS nft_deposits - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - num_triggers - WHERE - num_sales < 2 - ) - GROUP BY - 1 - HAVING - nft_deposits = 1 -), -first_token_withdraw AS ( - SELECT - tx_id, - MIN(event_index) AS min_index - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - omit_nft_nontransfers - WHERE - nft_transferred - ) - AND event_type = 'TokensWithdrawn' - GROUP BY - 1 -), --- 3 most important events are the first TokenWithdraw, then Withdraw and Deposit (NFT movement) -token_withdraw_event AS ( - SELECT - tx_id, - event_contract AS currency, - event_data :amount :: DOUBLE AS amount, - event_data :from :: STRING AS buyer_purchase, - min_index - FROM - silver_events - LEFT JOIN first_token_withdraw USING (tx_id) - WHERE - tx_id IN ( - SELECT - tx_id - FROM - omit_nft_nontransfers - WHERE - nft_transferred - ) - AND event_index = min_index - AND event_type = 'TokensWithdrawn' -), -nft_withdraw_event_seller AS ( - SELECT - tx_id, - event_index AS event_index_seller, - event_contract AS nft_collection_seller, - COALESCE( - event_data :from, - event_data :address - ) :: STRING AS seller, - COALESCE( - event_data :id, - event_data :tokenId - ) :: STRING AS nft_id_seller - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - omit_nft_nontransfers - WHERE - nft_transferred - ) - AND event_type IN ( - 'Withdraw', - 'FlovatarSaleWithdrawn', - 'FlovatarComponentSaleWithdrawn' -- if adding anything new, don't forget about omit_nft_nontransfers check! - ) -), -nft_deposit_event_buyer AS ( - SELECT - tx_id, - event_contract AS nft_collection_deposit, - event_data :id :: STRING AS nft_id_deposit, - event_data :to :: STRING AS buyer_deposit - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - omit_nft_nontransfers - WHERE - nft_transferred - ) - AND event_type = 'Deposit' -), -nft_sales AS ( - SELECT - e.tx_id, - e.block_timestamp, - e.block_height, - e.tx_succeeded, - e.is_purchased, - e.marketplace, - w.currency, - IFF( - e.marketplace = 'A.4eb8a10cb9f87357.NFTStorefrontV2', - e.event_data :salePrice :: DOUBLE, - w.amount - ) AS amount, - w.buyer_purchase, - s.nft_collection_seller, - s.seller, - s.nft_id_seller, - b.nft_collection_deposit, - b.nft_id_deposit, - b.buyer_deposit, - e._ingested_at, - e._inserted_timestamp - FROM - sale_trigger e - LEFT JOIN token_withdraw_event w USING (tx_id) - LEFT JOIN nft_withdraw_event_seller s USING (tx_id) - LEFT JOIN nft_deposit_event_buyer b USING (tx_id) - WHERE - tx_id IN ( - SELECT - tx_id - FROM - omit_nft_nontransfers - WHERE - nft_transferred - ) -), -step_data AS ( - SELECT - tx_id, - event_index, - event_type, - event_data - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - nft_sales - ) - AND event_type IN ( - 'TokensWithdrawn', - 'TokensDeposited', - 'ForwardedDeposit', - 'RoyaltyDeposited' - ) -), -counterparty_data AS ( - SELECT - tx_id, - ARRAY_AGG(OBJECT_CONSTRUCT(event_type, event_data)) within GROUP ( - ORDER BY - event_index - ) AS tokenflow, - ARRAY_SIZE(tokenflow) AS steps, - ARRAY_AGG(event_type) within GROUP ( - ORDER BY - event_index - ) AS action, - ARRAY_AGG(event_data) within GROUP ( - ORDER BY - event_index - ) AS step_data, - ARRAY_AGG(COALESCE(event_data :to, event_data :from) :: STRING) within GROUP ( - ORDER BY - event_index - ) AS counterparties - FROM - step_data - GROUP BY - 1 -), -FINAL AS ( - SELECT - ns.tx_id, - block_timestamp, - block_height, - marketplace, - nft_collection_deposit AS nft_collection, - nft_id_seller AS nft_id, - currency, - amount AS price, - seller, - buyer_deposit AS buyer, - cd.tokenflow, - cd.steps AS num_steps, - cd.action AS step_action, - cd.step_data, - cd.counterparties, - tx_succeeded, - _ingested_at, - _inserted_timestamp - FROM - nft_sales ns - LEFT JOIN counterparty_data cd USING (tx_id) -), -dedupe_gaia AS ( - SELECT - * - FROM - FINAL - WHERE - tx_id IN ( - SELECT - tx_id - FROM - num_triggers - WHERE - sale_trigger_count = 2 - AND num_sales = 1 - ) qualify ROW_NUMBER() over ( - PARTITION BY tx_id - ORDER BY - marketplace - ) = 1 -) -SELECT - * -FROM - FINAL -EXCEPT -SELECT - * -FROM - dedupe_gaia diff --git a/models/silver_cw/nft/silver__nft_transactions_secondary_market.yml b/models/silver_cw/nft/silver__nft_transactions_secondary_market.yml deleted file mode 100644 index a45fbf4f..00000000 --- a/models/silver_cw/nft/silver__nft_transactions_secondary_market.yml +++ /dev/null @@ -1,175 +0,0 @@ -version: 2 - -models: - - name: silver__nft_transactions_secondary_market - description: |- - This table filters all NFT sales that interact with the general purpose contract `A.4eb8a10cb9f87357.NFTStorefront`. Transactions that purchase multiple NFTs in a single transaction are presently excluded from this table! - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - seller - - buyer - - nft_collection - - nft_id - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: marketplace - description: "{{ doc('marketplace') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: nft_collection - description: "{{ doc('nft_collection') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: nft_id - description: "{{ doc('nft_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - NUMBER - - - name: currency - description: "{{ doc('currency') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: price - description: "{{ doc('price') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: seller - description: "{{ doc('seller') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: buyer - description: "{{ doc('buyer') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: tokenflow - description: "{{ doc('tokenflow') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: num_steps - description: "{{ doc('num_steps') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: step_action - description: "{{ doc('step_action') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: step_data - description: "{{ doc('step_data') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: counterparties - description: "{{ doc('counterparties') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/staking/silver__staking_actions.sql b/models/silver_cw/staking/silver__staking_actions.sql deleted file mode 100644 index b230afcb..00000000 --- a/models/silver_cw/staking/silver__staking_actions.sql +++ /dev/null @@ -1,99 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH silver_events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -- WHERE - -- event_data :: STRING != '{}' - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -flow_staking AS ( - SELECT - tx_id, - event_index, - block_timestamp, - block_height, - tx_succeeded, - event_contract, - event_type AS action, - event_data :amount :: FLOAT AS amount, - event_data :delegatorID :: STRING AS delegator_id, - event_data :nodeID :: STRING AS node_id, - _inserted_timestamp - FROM - silver_events - WHERE - event_contract = 'A.8624b52f9ddcd04a.FlowIDTableStaking' - AND event_type IN ( - 'DelegatorTokensCommitted', - 'DelegatorRewardTokensWithdrawn', - 'DelegatorUnstakedTokensWithdrawn', - 'TokensCommitted', - 'RewardTokensWithdrawn', - 'UnstakedTokensWithdrawn' - ) -), -add_auth AS ( - SELECT - tx_id, - COALESCE( - authorizers [1], - authorizers [0] - ) :: STRING AS primary_authorizer - FROM - {{ ref('silver__transactions') }} - WHERE - tx_id IN ( - SELECT - tx_id - FROM - flow_staking - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -FINAL AS ( - SELECT - s.tx_id, - event_index, - block_timestamp, - block_height, - tx_succeeded, - primary_authorizer AS delegator, - action, - amount, - node_id, - _inserted_timestamp - FROM - flow_staking s - LEFT JOIN add_auth A USING (tx_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/staking/silver__staking_actions.yml b/models/silver_cw/staking/silver__staking_actions.yml deleted file mode 100644 index d770ffa7..00000000 --- a/models/silver_cw/staking/silver__staking_actions.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 2 - -models: - - name: silver__staking_actions - description: |- - This table provides transaction-level info on FLOW staking activities. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: event_index - description: "{{ doc('event_index') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: delegator - description: "{{ doc('delegator') }}" - - - name: action - description: "{{ doc('action') }}" - - - name: amount - description: "{{ doc('amount') }}" - - - name: node_id - description: "{{ doc('node_id') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/swaps/silver__swaps.sql b/models/silver_cw/swaps/silver__swaps.sql deleted file mode 100644 index 086a7375..00000000 --- a/models/silver_cw/swaps/silver__swaps.sql +++ /dev/null @@ -1,318 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = "CONCAT_WS('-', tx_id, swap_index)", - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__swaps_events') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -swap_events AS ( - SELECT - * - FROM - events - WHERE - tx_id NOT IN ( - SELECT - DISTINCT tx_id - FROM - events - WHERE - event_type = 'RewardTokensWithdrawn' - OR event_type = 'NFTReceived' - ) -- PierPair needs a bespoke model as it does not deposit traded token to Pool contract - AND tx_id NOT IN ( - SELECT - DISTINCT tx_id - FROM - events - WHERE - event_contract LIKE '%PierPair%' - ) -), -pool_info AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_index, - event_type, - RANK() over ( - PARTITION BY tx_id - ORDER BY - event_index - ) - 1 AS swap_index, - event_contract AS pool_contract, - CASE - WHEN object_keys(event_data) [0] :: STRING = 'side' THEN 'Blocto' - WHEN object_keys(event_data) [0] :: STRING = 'direction' THEN 'Increment' - WHEN object_keys(event_data) [3] :: STRING = 'swapAForB' THEN 'Metapier' - ELSE 'Other' - END AS likely_dex, - COALESCE( - event_data :direction :: NUMBER, - event_data :side :: NUMBER - 1, - event_data :swapAForB :: BOOLEAN :: NUMBER - ) AS direction, - COALESCE( - event_data :inTokenAmount, - event_data :token1Amount, - event_data :amountIn - ) :: DOUBLE AS in_token_amount, - COALESCE( - event_data :outTokenAmount, - event_data :token2Amount, - event_data :amountOut - ) :: DOUBLE AS out_token_amount, - _inserted_timestamp - FROM - swap_events - WHERE - event_type IN ( - 'Trade', - 'Swap' - ) -), -token_withdraws AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_index, - RANK() over ( - PARTITION BY tx_id - ORDER BY - event_index - ) - 1 AS token_index, - RANK() over ( - PARTITION BY CONCAT( - tx_id, - event_data :amount :: STRING, - event_data :from :: STRING - ) - ORDER BY - event_index - ) - 1 AS unique_order, - event_contract, - event_data, - _inserted_timestamp - FROM - swap_events - WHERE - event_type = 'TokensWithdrawn' - AND tx_id IN ( - SELECT - DISTINCT tx_id - FROM - pool_info - ) -), -token_deposits AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_index, - RANK() over ( - PARTITION BY tx_id - ORDER BY - event_index - ) - 1 AS token_index, - RANK() over ( - PARTITION BY CONCAT( - tx_id, - event_data :amount :: STRING, - event_data :to :: STRING - ) - ORDER BY - event_index - ) - 1 AS unique_order, - event_contract, - event_data, - _inserted_timestamp - FROM - swap_events - WHERE - event_type = 'TokensDeposited' - AND tx_id IN ( - SELECT - DISTINCT tx_id - FROM - pool_info - ) -), -link_token_movement AS ( - SELECT - w.tx_id, - w.block_timestamp, - w.block_height, - w._inserted_timestamp, - -- set transfer index based on execution via deposit, not withdraw, event - RANK() over ( - PARTITION BY w.tx_id - ORDER BY - d.event_index - ) - 1 AS transfer_index, - w.event_data :from :: STRING AS withdraw_from, - d.event_data :to :: STRING AS deposit_to, - w.event_data :amount :: DOUBLE AS amount, - w.event_contract AS token_contract, - w.event_contract = d.event_contract AS contract_check - FROM - token_withdraws w - LEFT JOIN token_deposits d - ON w.tx_id = d.tx_id - AND w.event_contract = d.event_contract - AND w.event_data :amount :: STRING = d.event_data :amount :: STRING - AND w.unique_order = d.unique_order -), -restructure AS ( - SELECT - t.tx_id, - t.transfer_index, - p.swap_index, - RANK() over ( - PARTITION BY t.tx_id, - swap_index - ORDER BY - transfer_index - ) - 1 AS token_position, - t.withdraw_from, - t.deposit_to, - CONCAT('0x', SPLIT(pool_contract, '.') [1]) AS pool_address, - sub.trader, - ARRAYS_OVERLAP(ARRAY_CONSTRUCT(t.withdraw_from, t.deposit_to), ARRAY_CONSTRUCT(pool_address, sub.trader)) AS transfer_involve_pool_or_trader, - t.amount, - t.token_contract, - p.pool_contract, - p.direction, - p.in_token_amount, - p.out_token_amount - FROM - link_token_movement t - LEFT JOIN pool_info p - ON p.tx_id = t.tx_id - AND ( - p.in_token_amount = t.amount -- blocto takes a 0.3% fee - OR ROUND((p.in_token_amount / 0.997) - t.amount) = 0 - OR p.out_token_amount = t.amount - OR ROUND((p.out_token_amount / 0.997) - t.amount) = 0 - ) - AND transfer_index >= swap_index - LEFT JOIN ( - SELECT - tx_id, - withdraw_from AS trader - FROM - link_token_movement - WHERE - transfer_index = 0 - ) sub - ON t.tx_id = sub.tx_id - WHERE - swap_index IS NOT NULL -- exclude the network fee token movement - AND transfer_involve_pool_or_trader -), --- there are some cases where the same token is transferred multiple times in a single swap --- causing an error with object agg -excl_dup_transfer AS ( - SELECT - tx_id, - pool_contract, - swap_index, - CONCAT( - 'token', - token_position - ), - COUNT(1) AS identical_transfer_count - FROM - restructure - GROUP BY - 1, - 2, - 3, - 4 - HAVING - identical_transfer_count > 1 -), -pool_token_alignment AS ( - SELECT - tx_id, - pool_contract, - swap_index, - OBJECT_AGG(CONCAT('token', token_position), token_contract :: variant) AS tokens, - OBJECT_AGG(CONCAT('amount', token_position), amount) AS amounts, - OBJECT_AGG(CONCAT('from', token_position), withdraw_from :: variant) AS withdraws, - OBJECT_AGG(CONCAT('to', token_position), deposit_to :: variant) AS deposits - FROM - restructure - WHERE - tx_id NOT IN ( - SELECT - DISTINCT tx_id - FROM - excl_dup_transfer - ) - GROUP BY - 1, - 2, - 3 -), -boilerplate AS ( - SELECT - tx_id, - block_timestamp, - block_height, - _inserted_timestamp, - withdraw_from AS trader - FROM - link_token_movement - WHERE - transfer_index = 0 - AND tx_id NOT IN ( - SELECT - DISTINCT tx_id - FROM - excl_dup_transfer - ) -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - block_height, - pool_contract AS swap_contract, - swap_index, - trader, - withdraws :from0 :: STRING AS token_out_source, - tokens :token0 :: STRING AS token_out_contract, - amounts :amount0 :: DOUBLE AS token_out_amount, - deposits :to1 :: STRING AS token_in_destination, - tokens :token1 :: STRING AS token_in_contract, - amounts :amount1 :: DOUBLE AS token_in_amount, - _inserted_timestamp - FROM - boilerplate - LEFT JOIN pool_token_alignment USING (tx_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/swaps/silver__swaps.yml b/models/silver_cw/swaps/silver__swaps.yml deleted file mode 100644 index 8929059f..00000000 --- a/models/silver_cw/swaps/silver__swaps.yml +++ /dev/null @@ -1,112 +0,0 @@ -version: 2 - -models: - - name: silver__swaps - description: |- - This table records asset swaps on the Flow blockchain. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - trader - - swap_index - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: swap_contract - description: "{{ doc('swap_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: swap_index - description: "{{ doc('swap_index') }}" - - - name: trader - description: "{{ doc('trader') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: token_out_source - description: "{{ doc('token_out_source') }}" - - - name: token_out_amount - description: "{{ doc('token_out_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: token_out_contract - description: "{{ doc('token_out_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: token_in_destination - description: "{{ doc('token_in_destination') }}" - - - name: token_in_amount - description: "{{ doc('token_in_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: token_in_contract - description: "{{ doc('token_in_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/swaps/silver__swaps_events.sql b/models/silver_cw/swaps/silver__swaps_events.sql deleted file mode 100644 index a7d454b3..00000000 --- a/models/silver_cw/swaps/silver__swaps_events.sql +++ /dev/null @@ -1,65 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = "CONCAT_WS('-', tx_id, event_index)", - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH swap_contracts AS ( - - SELECT - * - FROM - {{ ref('silver__contract_labels') }} - WHERE - contract_name LIKE '%SwapPair%' -), -swaps_txs AS ( - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_contract IN ( - SELECT - event_contract - FROM - swap_contracts - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -swap_events AS ( - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - tx_id IN ( - SELECT - tx_id - FROM - swaps_txs - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -) -SELECT - * -FROM - swap_events diff --git a/models/silver_cw/swaps/silver__swaps_events.yml b/models/silver_cw/swaps/silver__swaps_events.yml deleted file mode 100644 index 880ba173..00000000 --- a/models/silver_cw/swaps/silver__swaps_events.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 2 - -models: - - name: silver__swaps_events - description: |- - This table records events from each Swap related transaction on the FLOW blockchain, as determined by an interaction with a SwapPair contract. This is intended to be internal only for downstream usage. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: event_index - description: "{{ doc('event_index') }}" - - - name: event_contract - description: "{{ doc('event_contract') }}" - - - name: event_type - description: "{{ doc('event_type') }}" - - - name: event_data - description: "{{ doc('event_attributes') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/swaps/silver__swaps_single_trade.sql b/models/silver_cw/swaps/silver__swaps_single_trade.sql deleted file mode 100644 index 1400ba57..00000000 --- a/models/silver_cw/swaps/silver__swaps_single_trade.sql +++ /dev/null @@ -1,174 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH swaps_events AS ( - - SELECT - * - FROM - {{ ref('silver__swaps_events') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -action_ct AS ( - SELECT - tx_id, - event_type, - COUNT(1) AS n - FROM - swaps_events - WHERE - event_type IN ( - 'Trade', - 'Swap' - ) - GROUP BY - 1, - 2 -), -step_ct AS ( - SELECT - tx_id, - OBJECT_AGG( - event_type, - n - ) AS ob - FROM - action_ct - GROUP BY - 1 -), -single_trade AS ( - SELECT - tx_id - FROM - step_ct - WHERE - ob :Trade = 1 - AND ob :Swap IS NULL -), -swaps_single_trade AS ( - SELECT - * - FROM - swaps_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - single_trade - ) -), -index_id AS ( - SELECT - tx_id, - event_type, - MIN(event_index) AS event_index - FROM - swaps_single_trade - WHERE - event_type = 'TokensWithdrawn' - GROUP BY - 1, - 2 -), -token_out_data AS ( - SELECT - sst.tx_id, - block_timestamp, - block_height, - event_contract AS event_contract_token_out, - event_data AS event_data_token_out, - event_data :amount :: DOUBLE AS token_amount_token_out, - LOWER( - event_data :from :: STRING - ) AS trader_token_out, - _ingested_at, - _inserted_timestamp - FROM - index_id ii - LEFT JOIN swaps_single_trade sst USING ( - tx_id, - event_index - ) -), -trade_data AS ( - SELECT - tx_id, - block_timestamp, - event_type, - event_contract AS event_contract_trade, - event_data AS event_data_trade, - event_data :side :: NUMBER AS swap_side, - event_data :token1Amount :: DOUBLE AS token_1_amount, - -- note some are decimal adjusted, some are not. identify by contract - event_data :token2Amount :: DOUBLE AS token_2_amount, - l.account_address AS swap_account, - _ingested_at, - _inserted_timestamp - FROM - swaps_single_trade sst - LEFT JOIN {{ ref('silver__contract_labels') }} - l USING (event_contract) - WHERE - event_type = 'Trade' -), -token_in_data AS ( - SELECT - sst.tx_id, - sst.block_timestamp, - sst.event_contract AS event_contract_token_in, - sst.event_data AS event_data_token_in, - sst.event_data :amount :: DOUBLE AS amount_token_in - FROM - trade_data t - LEFT JOIN swaps_single_trade sst - ON sst.tx_id = t.tx_id - AND t.swap_account = LOWER( - sst.event_data :from :: STRING - ) - WHERE - sst.event_type = 'TokensWithdrawn' -), -combo AS ( - SELECT - tod.tx_id, - tod.block_timestamp, - tod.block_height, - td.event_contract_trade AS swap_contract, - LOWER( - tod.trader_token_out - ) AS trader, - tod.token_amount_token_out AS token_out_amount, - tod.event_contract_token_out AS token_out_contract, - tid.amount_token_in AS token_in_amount, - tid.event_contract_token_in AS token_in_contract, - -- keep these next 3 columns bc i can derive fees from the difference in token_out_amount and token_[n]_amount where n = swap_side - td.swap_side, - td.token_1_amount, - td.token_2_amount, - tod._ingested_at, - tod._inserted_timestamp - FROM - token_out_data tod - LEFT JOIN trade_data td USING (tx_id) - LEFT JOIN token_in_data tid USING (tx_id) -) -SELECT - * -FROM - combo diff --git a/models/silver_cw/swaps/silver__swaps_single_trade.yml b/models/silver_cw/swaps/silver__swaps_single_trade.yml deleted file mode 100644 index 21d76662..00000000 --- a/models/silver_cw/swaps/silver__swaps_single_trade.yml +++ /dev/null @@ -1,129 +0,0 @@ -version: 2 - -models: - - name: silver__swaps_single_trade - description: |- - This table records single-pool asset swaps on the Flow blockchain. A single-pool asset swap is one where a liquidity pool exists for the trade such that no routing is required. (i.e. Swapping FUSD to USDT using `A.87f3f233f34b0733.FusdUsdtSwapPair`) - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - trader - - swap_contract - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: swap_contract - description: "{{ doc('swap_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: trader - description: "{{ doc('trader') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: token_out_amount - description: "{{ doc('token_out_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: token_out_contract - description: "{{ doc('token_out_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: token_in_amount - description: "{{ doc('token_in_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: token_in_contract - description: "{{ doc('token_in_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: swap_side - description: "{{ doc('swap_side') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: token_1_amount - description: "{{ doc('token_1_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: token_2_amount - description: "{{ doc('token_2_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/transfers/silver__bridge_blocto.sql b/models/silver_cw/transfers/silver__bridge_blocto.sql deleted file mode 100644 index 60957d7d..00000000 --- a/models/silver_cw/transfers/silver__bridge_blocto.sql +++ /dev/null @@ -1,298 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::date'], - unique_key = 'tx_id', - tags = ['bridge', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -- WHERE - -- event_data :: STRING != '{}' - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -teleport_events AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_contract AS teleport_contract_fee, - event_data :amount :: DOUBLE AS amount_fee, - event_data :type :: NUMBER AS teleport_direction, - _ingested_at, - _inserted_timestamp - FROM - events - WHERE - event_type = 'FeeCollected' - AND event_contract LIKE '%Teleport%' -), -teleports_in AS ( - SELECT - tx_id, - event_contract AS event_contract_teleport, - event_data :amount :: DOUBLE AS amount_teleport, - event_data :from AS from_teleport, - STRTOK_TO_ARRAY( - REPLACE(REPLACE(event_data :from :: STRING, '['), ']'), - ', ' - ) :: ARRAY AS from_teleport_array, - COALESCE( - event_data :hash, - event_data :txHash - ) :: STRING AS hash_teleport, - _ingested_at, - _inserted_timestamp - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - teleport_events - WHERE - teleport_direction = 1 - ) - AND event_index = 0 - AND event_contract LIKE '%Teleport%' - AND event_type IN ( - 'TokensTeleportedIn', - 'Unlocked' - ) - AND from_teleport :: STRING NOT LIKE '%{\"ArrayType%' -), -deposits AS ( - SELECT - tx_id, - event_contract, - event_index, - event_data :amount :: DOUBLE AS amount_deposits, - ROW_NUMBER() over ( - PARTITION BY tx_id - ORDER BY - amount_deposits DESC - ) AS rn, - event_data :to :: STRING AS to_deposits - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - teleports_in - ) - AND event_type = 'TokensDeposited' - ORDER BY - tx_id, - amount_deposits DESC -), -blocto_inbound AS ( - SELECT - t.tx_id, - f.block_timestamp, - f.block_height, - t.event_contract_teleport AS teleport_contract, - d.event_contract AS token_contract, - t.amount_teleport AS gross_amount, - f.amount_fee, - d.amount_deposits AS net_amount, - d.to_deposits AS flow_wallet_address, - f.teleport_direction, - 'blocto' AS bridge, - f._ingested_at, - f._inserted_timestamp - FROM - teleports_in t - LEFT JOIN deposits d USING (tx_id) - LEFT JOIN teleport_events f USING (tx_id) - WHERE - d.rn = 1 -), -teleports_out_withdraw_non_fiat AS ( - SELECT - tx_id, - event_contract, - event_data :amount :: DOUBLE AS amount_withdraw, - event_data :from :: STRING AS from_withdraw - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - teleport_events - WHERE - teleport_direction = 0 - ) - AND event_index = 0 - AND event_contract != 'A.b19436aae4d94622.FiatToken' -), -teleports_out_withdraw_fiat AS ( - SELECT - tx_id, - event_contract, - event_data :amount :: DOUBLE AS amount_withdraw, - event_data :from :: STRING AS from_withdraw - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - teleport_events - WHERE - teleport_direction = 0 - ) - AND event_index = 1 - AND event_contract = 'A.b19436aae4d94622.FiatToken' -), -teleports_out_withdraw AS ( - SELECT - tx_id, - event_contract, - amount_withdraw, - from_withdraw - FROM - teleports_out_withdraw_non_fiat - UNION - SELECT - tx_id, - event_contract, - amount_withdraw, - from_withdraw - FROM - teleports_out_withdraw_fiat -), -teleports_out AS ( - SELECT - tx_id, - event_contract AS event_contract_teleport, - event_data :amount :: DOUBLE AS amount_teleport, - event_data :to AS to_teleport, - STRTOK_TO_ARRAY( - REPLACE(REPLACE(event_data :to :: STRING, '['), ']'), - ', ' - ) :: ARRAY AS to_teleport_array - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - teleport_events - WHERE - teleport_direction = 0 - ) - AND event_type IN ( - 'TokensTeleportedOut', - 'Locked' - ) -), -blocto_outbound AS ( - SELECT - t.tx_id, - f.block_timestamp, - f.block_height, - t.event_contract_teleport AS teleport_contract, - w.event_contract AS token_contract, - w.amount_withdraw AS gross_amount, - f.amount_fee, - t.amount_teleport AS net_amount, - w.from_withdraw AS flow_wallet_address, - f.teleport_direction, - 'blocto' AS bridge, - f._ingested_at, - f._inserted_timestamp - FROM - teleports_out t - LEFT JOIN teleports_out_withdraw w USING (tx_id) - LEFT JOIN teleport_events f USING (tx_id) -), -tbl_union AS ( - SELECT - tx_id, - block_timestamp, - block_height, - teleport_contract, - token_contract, - gross_amount, - amount_fee, - net_amount, - flow_wallet_address, - teleport_direction, - bridge, - _ingested_at, - _inserted_timestamp - FROM - blocto_inbound - UNION - SELECT - tx_id, - block_timestamp, - block_height, - teleport_contract, - token_contract, - gross_amount, - amount_fee, - net_amount, - flow_wallet_address, - teleport_direction, - bridge, - _ingested_at, - _inserted_timestamp - FROM - blocto_outbound -), -tele_labels AS ( - SELECT - teleport_contract, - blockchain - FROM - {{ ref('seeds__blocto_teleport_labels') }} -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - block_height, - t.teleport_contract, - token_contract, - gross_amount, - amount_fee, - net_amount, - flow_wallet_address, - CASE - WHEN teleport_direction = 0 THEN 'outbound' - ELSE 'inbound' - END AS teleport_direction, - l.blockchain, - bridge, - _ingested_at, - _inserted_timestamp - FROM - tbl_union t - LEFT JOIN tele_labels l USING (teleport_contract) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/transfers/silver__bridge_blocto.yml b/models/silver_cw/transfers/silver__bridge_blocto.yml deleted file mode 100644 index d42121e1..00000000 --- a/models/silver_cw/transfers/silver__bridge_blocto.yml +++ /dev/null @@ -1,131 +0,0 @@ -version: 2 - -models: - - name: silver__bridge_blocto - description: |- - This table parses transactions where tokens are bridged to or from the Flow network via Blocto Teleport. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 3 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: teleport_contract - description: "{{ doc('teleport_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: token_contract - description: "{{ doc('token_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: gross_amount - description: "{{ doc('gross_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - DOUBLE - - FLOAT - - - name: amount_fee - description: "{{ doc('amount_fee') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - DOUBLE - - FLOAT - - - name: net_amount - description: "{{ doc('net_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - DOUBLE - - FLOAT - - - name: flow_wallet_address - description: "{{ doc('flow_wallet_address') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: teleport_direction - description: "{{ doc('teleport_direction') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: blockchain - description: "{{ doc('blockchain') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: bridge - description: "{{ doc('bridge') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/transfers/silver__bridge_celer.sql b/models/silver_cw/transfers/silver__bridge_celer.sql deleted file mode 100644 index 50fa8c68..00000000 --- a/models/silver_cw/transfers/silver__bridge_celer.sql +++ /dev/null @@ -1,165 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::date'], - unique_key = 'tx_id', - tags = ['bridge', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -- WHERE - -- event_data :: STRING != '{}' - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -cbridge_txs AS ( - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - _ingested_at, - _inserted_timestamp - FROM - events - WHERE - event_contract = 'A.08dd120226ec2213.PegBridge' -), -inbound AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_contract AS bridge_contract, - REPLACE(REPLACE(event_data :token :: STRING, '.Vault'), '"') AS token_contract, - event_data :amount :: DOUBLE AS amount, - event_data :receiver :: STRING AS flow_wallet_address, - REPLACE(CONCAT('0x', event_data :depositor) :: STRING, '"') AS counterparty, - event_data :refChId :: NUMBER AS chain_id, - 'inbound' AS direction, - 'cbridge' AS bridge, - _ingested_at, - _inserted_timestamp - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - cbridge_txs - ) - AND event_type = 'Mint' -), -outbound AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_contract AS bridge_contract, - REPLACE(REPLACE(event_data :token :: STRING, '.Vault'), '"') AS token_contract, - event_data :amount :: DOUBLE AS amount, - event_data :burner :: STRING AS flow_wallet_address, - REPLACE( - event_data :toAddr :: STRING, - '"' - ) AS counterparty, - event_data :toChain :: NUMBER AS chain_id, - 'outbound' AS direction, - 'cbridge' AS bridge, - _ingested_at, - _inserted_timestamp - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - cbridge_txs - ) - AND event_type = 'Burn' -), -tbl_union AS ( - SELECT - tx_id, - block_timestamp, - block_height, - bridge_contract, - token_contract, - amount, - flow_wallet_address, - counterparty, - chain_id, - direction, - bridge, - _ingested_at, - _inserted_timestamp - FROM - inbound - UNION - SELECT - tx_id, - block_timestamp, - block_height, - bridge_contract, - token_contract, - amount, - flow_wallet_address, - counterparty, - chain_id, - direction, - bridge, - _ingested_at, - _inserted_timestamp - FROM - outbound -), -chain_ids AS ( - SELECT - chain_id, - blockchain - FROM - {{ ref('seeds__celer_chain_ids') }} -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - block_height, - bridge_contract, - token_contract, - amount, - flow_wallet_address, - counterparty, - t.chain_id, - l.blockchain, - direction, - bridge, - _ingested_at, - _inserted_timestamp - FROM - tbl_union t - LEFT JOIN chain_ids l USING (chain_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/transfers/silver__bridge_celer.yml b/models/silver_cw/transfers/silver__bridge_celer.yml deleted file mode 100644 index 363ea98f..00000000 --- a/models/silver_cw/transfers/silver__bridge_celer.yml +++ /dev/null @@ -1,128 +0,0 @@ -version: 2 - -models: - - name: silver__bridge_celer - description: |- - This table parses transactions where tokens are bridged to or from the Flow network using the Celer cBridge. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 3 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: bridge_contract - description: "{{ doc('bridge_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: token_contract - description: "{{ doc('token_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: amount - description: "{{ doc('amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - DOUBLE - - FLOAT - - - name: flow_wallet_address - description: "{{ doc('flow_wallet_address') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: counterparty - description: "{{ doc('counterparty') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: chain_id - description: "{{ doc('chain_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: blockchain - description: "{{ doc('blockchain') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: direction - description: "{{ doc('direction') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: bridge - description: "{{ doc('bridge') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/transfers/silver__token_transfers.sql b/models/silver_cw/transfers/silver__token_transfers.sql deleted file mode 100644 index d384fae1..00000000 --- a/models/silver_cw/transfers/silver__token_transfers.sql +++ /dev/null @@ -1,139 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::date'], - unique_key = "CONCAT_WS('-', tx_id, sender, recipient, token_contract, amount)", - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -- WHERE - -- event_data :: STRING != '{}' - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -transfers AS ( - SELECT - _inserted_timestamp, - tx_id, - event_contract, - COUNT(event_type) AS event_count, - MAX( - event_index + 1 - ) AS max_index - FROM - events - WHERE - event_type IN ( - 'TokensDeposited', - 'TokensWithdrawn', - 'FeesDeducted' - ) - GROUP BY - _inserted_timestamp, - tx_id, - event_contract - HAVING - event_count = max_index - OR event_contract = 'A.b19436aae4d94622.FiatToken' -), -withdraws AS ( - SELECT - block_height, - _inserted_timestamp, - block_timestamp, - tx_id, - event_data :from :: STRING AS sender, - event_contract AS token_contract, - event_data :amount :: FLOAT AS amount, - tx_succeeded - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - transfers - ) - AND event_type = 'TokensWithdrawn' - GROUP BY - block_height, - _inserted_timestamp, - block_timestamp, - tx_id, - sender, - token_contract, - amount, - tx_succeeded -), -deposits AS ( - SELECT - tx_id, - _inserted_timestamp, - event_data :to :: STRING AS recipient, - event_contract AS token_contract, - event_data :amount :: FLOAT AS amount - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - transfers - ) - AND event_type = 'TokensDeposited' - GROUP BY - tx_id, - _inserted_timestamp, - recipient, - token_contract, - amount -), -FINAL AS ( - SELECT - block_height, - w._inserted_timestamp AS _inserted_timestamp, - block_timestamp, - w.tx_id, - sender, - recipient, - w.token_contract, - SUM(COALESCE(d.amount, w.amount)) AS amount, - tx_succeeded - FROM - withdraws w - LEFT JOIN deposits d - ON w.tx_id = d.tx_id - AND w.token_contract = d.token_contract - AND w.amount = d.amount - WHERE - sender IS NOT NULL - GROUP BY - block_height, - w._inserted_timestamp, - block_timestamp, - w.tx_id, - sender, - recipient, - w.token_contract, - tx_succeeded -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/transfers/silver__token_transfers.yml b/models/silver_cw/transfers/silver__token_transfers.yml deleted file mode 100644 index a9a11a9e..00000000 --- a/models/silver_cw/transfers/silver__token_transfers.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: 2 - -models: - - name: silver__token_transfers - description: |- - This table records all token transfers on the FLOW blockchain. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: sender - description: "{{ doc('sender') }}" - - - name: recipient - description: "{{ doc('recipient') }}" - - - name: token_contract - description: "{{ doc('token_contract') }}" - - - name: amount - description: "{{ doc('amount') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" diff --git a/tests/tests__moment_metadata_allday.sql b/tests/tests__moment_metadata_allday.sql index c0b08c8c..2d7bba0e 100644 --- a/tests/tests__moment_metadata_allday.sql +++ b/tests/tests__moment_metadata_allday.sql @@ -2,7 +2,7 @@ WITH mint_events AS ( SELECT MAX(block_timestamp) :: DATE AS last_mint_date FROM - {{ ref('silver__nft_moments') }} + {{ ref('silver__nft_moments_s') }} WHERE event_contract = 'A.e4cf4bdc1751c65d.AllDay' AND event_type = 'MomentNFTMinted' diff --git a/tests/tests__moment_metadata_topshot.sql b/tests/tests__moment_metadata_topshot.sql index c81cf0bd..11a5532c 100644 --- a/tests/tests__moment_metadata_topshot.sql +++ b/tests/tests__moment_metadata_topshot.sql @@ -2,7 +2,7 @@ WITH mint_events AS ( SELECT MAX(block_timestamp) :: DATE AS last_mint_date FROM - {{ ref('silver__nft_moments') }} + {{ ref('silver__nft_moments_s') }} WHERE event_contract = 'A.0b2a3299cc857e29.TopShot' AND event_type = 'MomentMinted'