diff --git a/dbt_project.yml b/dbt_project.yml index 04adf17f..4714daad 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -50,6 +50,7 @@ query-comment: models: +copy_grants: true + +on_schema_change: "append_new_columns" # In this example config, we tell dbt to build all models in the example/ directory # as tables. These settings can be overridden in the individual model files diff --git a/models/bronze/ethereum/bronze__state_hashes.sql b/models/bronze/ethereum/bronze__state_hashes.sql index b9745765..e4170770 100644 --- a/models/bronze/ethereum/bronze__state_hashes.sql +++ b/models/bronze/ethereum/bronze__state_hashes.sql @@ -17,7 +17,13 @@ SELECT state_prev_total_elements, state_min_block, state_max_block, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['state_block_number'] + ) }} AS state_hashes_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ source( 'ethereum_silver', diff --git a/models/bronze/ethereum/bronze__submission_hashes.sql b/models/bronze/ethereum/bronze__submission_hashes.sql index 028fa3dc..acd42a56 100644 --- a/models/bronze/ethereum/bronze__submission_hashes.sql +++ b/models/bronze/ethereum/bronze__submission_hashes.sql @@ -15,7 +15,13 @@ SELECT l1_submission_prev_total_elements, sub_min_block, sub_max_block, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['l1_submission_block_number'] + ) }} AS submission_hashes_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ source( 'ethereum_silver', diff --git a/models/doc_descriptions/general/deprecation.md b/models/doc_descriptions/general/deprecation.md index 9da8ab69..701b7f39 100644 --- a/models/doc_descriptions/general/deprecation.md +++ b/models/doc_descriptions/general/deprecation.md @@ -1,12 +1,11 @@ -{% docs deprecation %} +{% docs internal_column %} -Deprecating soon: This is a notice that we're only removing the below columns. Please migrate queries using these columns to `fact_decoded_event_logs`, `ez_decoded_event_logs` or use manual parsing of topics and data. The following columns will be deprecated on 7/16/23: +Deprecated. This column is no longer used. Please remove from your query by Jan. 31 2024.' -`Fact_event_logs` Columns: -- `event_name` -- `event_inputs` -- `contract_name` +{% enddocs %} + +{% docs amount_deprecation %} + +This column is being deprecated for standardization purposes on Jan. 31 2024. Please use the equivalent column without the native asset prefix. For example, use `amount` instead of `eth_amount`. -`Fact_transactions` Columns: -- `tx_json` {% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/export_columns.md b/models/doc_descriptions/general/export_columns.md new file mode 100644 index 00000000..848af449 --- /dev/null +++ b/models/doc_descriptions/general/export_columns.md @@ -0,0 +1,19 @@ +{% docs pk %} + +The unique identifier for each row in the table. + +{% enddocs %} + +{% docs inserted_timestamp %} + +The utc timestamp at which the row was inserted into the table. + +{% enddocs %} + +{% docs modified_timestamp %} + +The utc timestamp at which the row was last modified. + +{% enddocs %} + + diff --git a/models/doc_descriptions/nft/nft_intra_event_index.md b/models/doc_descriptions/nft/nft_intra_event_index.md new file mode 100644 index 00000000..4ce24fc7 --- /dev/null +++ b/models/doc_descriptions/nft/nft_intra_event_index.md @@ -0,0 +1,5 @@ +{% docs nft_intra_event_index %} + +The order of events within a single event index. This is primarily used for ERC1155 NFT batch transfer events. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/opt_traces_index.md b/models/doc_descriptions/traces/opt_traces_index.md new file mode 100644 index 00000000..1f462629 --- /dev/null +++ b/models/doc_descriptions/traces/opt_traces_index.md @@ -0,0 +1,5 @@ +{% docs opt_trace_index %} + +The index of the trace within the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/core/core__dim_contracts.sql b/models/gold/core/core__dim_contracts.sql index 30bdd92c..8c65ce7c 100644 --- a/models/gold/core/core__dim_contracts.sql +++ b/models/gold/core/core__dim_contracts.sql @@ -12,7 +12,15 @@ SELECT c0.block_number AS created_block_number, c0.block_timestamp AS created_block_timestamp, c0.tx_hash AS created_tx_hash, - c0.creator_address AS creator_address + c0.creator_address AS creator_address, + COALESCE ( + c0.created_contracts_id, + {{ dbt_utils.generate_surrogate_key( + ['c0.created_contract_address'] + ) }} + ) AS dim_contracts_id, + GREATEST(COALESCE(c0.inserted_timestamp, '2000-01-01'), COALESCE(c1.inserted_timestamp, '2000-01-01')) AS inserted_timestamp, + GREATEST(COALESCE(c0.modified_timestamp, '2000-01-01'), COALESCE(c1.modified_timestamp, '2000-01-01')) AS modified_timestamp FROM {{ ref('silver__created_contracts') }} c0 diff --git a/models/gold/core/core__dim_contracts.yml b/models/gold/core/core__dim_contracts.yml index 0ee8ea38..82dae5a5 100644 --- a/models/gold/core/core__dim_contracts.yml +++ b/models/gold/core/core__dim_contracts.yml @@ -20,3 +20,9 @@ models: description: 'The transaction hash when the contract was created' - name: CREATOR_ADDRESS description: 'The address of the creator of the contract' + - name: DIM_CONTRACTS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__dim_labels.sql b/models/gold/core/core__dim_labels.sql index e21ee86c..00a2ba14 100644 --- a/models/gold/core/core__dim_labels.sql +++ b/models/gold/core/core__dim_labels.sql @@ -11,6 +11,20 @@ SELECT address_name, label_type, label_subtype, - project_name + project_name, + COALESCE ( + labels_id, + {{ dbt_utils.generate_surrogate_key( + ['address'] + ) }} + ) AS dim_labels_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__labels') }} diff --git a/models/gold/core/core__dim_labels.yml b/models/gold/core/core__dim_labels.yml index bdc41e56..06cf8f5d 100644 --- a/models/gold/core/core__dim_labels.yml +++ b/models/gold/core/core__dim_labels.yml @@ -52,4 +52,9 @@ models: column_type_list: - STRING - VARCHAR - + - name: DIM_LABELS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__ez_decoded_event_logs.sql b/models/gold/core/core__ez_decoded_event_logs.sql index cb48a474..1e1e3f64 100644 --- a/models/gold/core/core__ez_decoded_event_logs.sql +++ b/models/gold/core/core__ez_decoded_event_logs.sql @@ -20,7 +20,16 @@ SELECT topics, DATA, event_removed, - tx_status + tx_status, + COALESCE ( + decoded_logs_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS ez_decoded_event_logs_id, + GREATEST(COALESCE(l.inserted_timestamp, '2000-01-01'), COALESCE(C.inserted_timestamp, '2000-01-01')) AS inserted_timestamp, + GREATEST(COALESCE(l.modified_timestamp, '2000-01-01'), COALESCE(C.modified_timestamp, '2000-01-01')) AS modified_timestamp FROM - {{ ref('silver__decoded_logs') }} - LEFT JOIN {{ ref('silver__contracts') }} using (contract_address) \ No newline at end of file + {{ ref('silver__decoded_logs') }} + l + LEFT JOIN {{ ref('silver__contracts') }} C USING (contract_address) diff --git a/models/gold/core/core__ez_decoded_event_logs.yml b/models/gold/core/core__ez_decoded_event_logs.yml index 5b286a8b..c3ea8a87 100644 --- a/models/gold/core/core__ez_decoded_event_logs.yml +++ b/models/gold/core/core__ez_decoded_event_logs.yml @@ -56,4 +56,10 @@ models: - name: EVENT_REMOVED description: '{{ doc("opt_event_removed") }}' - name: TX_STATUS - description: '{{ doc("opt_tx_status") }}' \ No newline at end of file + description: '{{ doc("opt_tx_status") }}' + - name: EZ_DECODED_EVENT_LOGS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__ez_eth_transfers.sql b/models/gold/core/core__ez_eth_transfers.sql index b8f2f34a..32d3565f 100644 --- a/models/gold/core/core__ez_eth_transfers.sql +++ b/models/gold/core/core__ez_eth_transfers.sql @@ -1,104 +1,26 @@ {{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'block_number', - cluster_by = ['block_timestamp::DATE'], - tags = ['core','non_realtime','reorg'], + materialized = 'view', persist_docs ={ "relation": true, "columns": true } ) }} -WITH eth_base AS ( - - SELECT - tx_hash, - block_number, - block_timestamp, - identifier, - from_address, - to_address, - eth_value, - _call_id, - _inserted_timestamp, - eth_value_precise_raw, - eth_value_precise, - tx_position, - trace_index - FROM - {{ ref('silver__traces') }} - WHERE - eth_value > 0 - AND tx_status = 'SUCCESS' - AND trace_status = 'SUCCESS' - AND TYPE NOT IN ( - 'DELEGATECALL', - 'STATICCALL' - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '72 hours' - FROM - {{ this }} -) -{% endif %} -), -tx_table AS ( - SELECT - block_number, - tx_hash, - from_address AS origin_from_address, - to_address AS origin_to_address, - origin_function_signature - FROM - {{ ref('silver__transactions') }} - WHERE - tx_hash IN ( - SELECT - DISTINCT tx_hash - FROM - eth_base - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '72 hours' - FROM - {{ this }} -) -{% endif %} -) SELECT - tx_hash AS tx_hash, - block_number AS block_number, - block_timestamp AS block_timestamp, - identifier AS identifier, + tx_hash, + block_number, + block_timestamp, + identifier, origin_from_address, origin_to_address, origin_function_signature, from_address AS eth_from_address, to_address AS eth_to_address, - eth_value AS amount, - eth_value_precise_raw AS amount_precise_raw, - eth_value_precise AS amount_precise, - ROUND( - eth_value * price, - 2 - ) AS amount_usd, + amount, + amount_precise_raw, + amount_precise, + amount_usd, _call_id, _inserted_timestamp, tx_position, trace_index FROM - eth_base A - LEFT JOIN {{ ref('silver__hourly_prices_priority_eth') }} - ON DATE_TRUNC( - 'hour', - A.block_timestamp - ) = HOUR - JOIN tx_table USING ( - tx_hash, - block_number - ) + {{ ref('silver__native_transfers') }} diff --git a/models/gold/core/core__ez_eth_transfers.yml b/models/gold/core/core__ez_eth_transfers.yml index b786074b..5922a3b2 100644 --- a/models/gold/core/core__ez_eth_transfers.yml +++ b/models/gold/core/core__ez_eth_transfers.yml @@ -1,7 +1,7 @@ version: 2 models: - name: core__ez_eth_transfers - description: '{{ doc("opt_ez_eth_transfers_table_doc") }}' + description: 'Deprecating soon! Migrate to `core.ez_native_transfers` by Jan. 31 2024.' columns: - name: BLOCK_NUMBER diff --git a/models/gold/core/core__ez_native_transfers.sql b/models/gold/core/core__ez_native_transfers.sql new file mode 100644 index 00000000..14d2cac4 --- /dev/null +++ b/models/gold/core/core__ez_native_transfers.sql @@ -0,0 +1,38 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + tx_hash, + block_number, + block_timestamp, + tx_position, + trace_index, + identifier, + origin_from_address, + origin_to_address, + origin_function_signature, + from_address, + to_address, + amount, + amount_precise_raw, + amount_precise, + amount_usd, + COALESCE ( + native_transfers_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} + ) AS ez_native_transfers_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp +FROM + {{ ref('silver__native_transfers') }} diff --git a/models/gold/core/core__ez_native_transfers.yml b/models/gold/core/core__ez_native_transfers.yml new file mode 100644 index 00000000..7af13f80 --- /dev/null +++ b/models/gold/core/core__ez_native_transfers.yml @@ -0,0 +1,42 @@ +version: 2 +models: + - name: core__ez_native_transfers + description: '{{ doc("opt_ez_eth_transfers_table_doc") }}' + + columns: + - name: TX_HASH + description: '{{ doc("opt_transfer_tx_hash") }}' + - name: BLOCK_NUMBER + description: '{{ doc("opt_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("opt_block_timestamp") }}' + - name: TX_POSITION + description: '{{ doc("opt_tx_position") }}' + - name: TRACE_INDEX + description: '{{ doc("opt_trace_index") }}' + - name: IDENTIFIER + description: '{{ doc("opt_traces_identifier") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("opt_origin_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("opt_origin_to") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("opt_origin_sig") }}' + - name: FROM_ADDRESS + description: '{{ doc("opt_transfer_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("opt_transfer_to_address") }}' + - name: AMOUNT + description: '{{ doc("opt_eth_amount") }}' + - name: AMOUNT_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: AMOUNT_PRECISE + description: '{{ doc("precise_amount_adjusted") }}' + - name: AMOUNT_USD + description: '{{ doc("opt_eth_amount_usd") }}' + - name: EZ_NATIVE_TRANSFERS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index 37ed8eb1..11def735 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -26,6 +26,20 @@ SELECT has_decimal, has_price, _log_id, - _inserted_timestamp + _inserted_timestamp, + COALESCE ( + transfers_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS ez_token_transfers_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__transfers') }} diff --git a/models/gold/core/core__ez_token_transfers.yml b/models/gold/core/core__ez_token_transfers.yml index dd4eafd3..140d2280 100644 --- a/models/gold/core/core__ez_token_transfers.yml +++ b/models/gold/core/core__ez_token_transfers.yml @@ -47,4 +47,13 @@ models: - name: HAS_PRICE description: '{{ doc("opt_transfer_has_price") }}' - name: _LOG_ID - description: '{{ doc("opt_log_id_transfers") }}' \ No newline at end of file + description: '{{ doc("internal_column") }}' + - name: _INSERTED_TIMESTAMP + description: '{{ doc("internal_column") }}' + - name: EZ_TOKEN_TRANSFERS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + \ No newline at end of file diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index c4156fb3..64153fa4 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -80,7 +80,49 @@ SELECT l1_submission_batch_root, 'l1_submission_tx_hash', l1_submission_tx_hash - ) AS l1_submission_details + ) AS l1_submission_details, + COALESCE ( + blocks_id, + {{ dbt_utils.generate_surrogate_key( + ['a.block_number'] + ) }} + ) AS fact_blocks_id, + GREATEST( + COALESCE( + A.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + b.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + C.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + d.inserted_timestamp, + '2000-01-01' + ) + ) AS inserted_timestamp, + GREATEST( + COALESCE( + A.modified_timestamp, + '2000-01-01' + ), + COALESCE( + b.modified_timestamp, + '2000-01-01' + ), + COALESCE( + C.modified_timestamp, + '2000-01-01' + ), + COALESCE( + d.modified_timestamp, + '2000-01-01' + ) + ) AS modified_timestamp FROM {{ ref('silver__blocks') }} A LEFT JOIN {{ ref('silver__state_hashes') }} diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index debff70e..db04b5ed 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -47,4 +47,10 @@ models: - name: L1_SUBMISSION_BATCH_INDEX description: '{{ doc("op_l1_sub_batch") }}' - name: L1_SUBMISSION_DETAILS - description: Variant column that includes details regarding state and submission root, index and tx_hash information for when the block is submitted to Ethereum Mainnet. \ No newline at end of file + description: Variant column that includes details regarding state and submission root, index and tx_hash information for when the block is submitted to Ethereum Mainnet. + - name: FACT_BLOCKS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_decoded_event_logs.sql b/models/gold/core/core__fact_decoded_event_logs.sql index e153816f..aab1f0cf 100644 --- a/models/gold/core/core__fact_decoded_event_logs.sql +++ b/models/gold/core/core__fact_decoded_event_logs.sql @@ -12,6 +12,20 @@ SELECT contract_address, event_name, decoded_flat AS decoded_log, - decoded_data AS full_decoded_log + decoded_data AS full_decoded_log, + COALESCE ( + decoded_logs_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS fact_decoded_event_logs_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM - {{ ref('silver__decoded_logs') }} \ No newline at end of file + {{ ref('silver__decoded_logs') }} diff --git a/models/gold/core/core__fact_decoded_event_logs.yml b/models/gold/core/core__fact_decoded_event_logs.yml index 2f116042..b9642c0a 100644 --- a/models/gold/core/core__fact_decoded_event_logs.yml +++ b/models/gold/core/core__fact_decoded_event_logs.yml @@ -41,3 +41,9 @@ models: description: 'The flattened decoded log, where the keys are the names of the event parameters, and the values are the values of the event parameters.' - name: FULL_DECODED_LOG description: 'The full decoded log, including the event name, the event parameters, and the data type of the event parameters.' + - name: FACT_DECODED_EVENT_LOGS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_event_logs.sql b/models/gold/core/core__fact_event_logs.sql index f57c5bf9..e21eda0e 100644 --- a/models/gold/core/core__fact_event_logs.sql +++ b/models/gold/core/core__fact_event_logs.sql @@ -17,6 +17,20 @@ SELECT DATA, event_removed, tx_status, - _log_id + _log_id, + COALESCE ( + logs_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS fact_event_logs_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__logs') }} diff --git a/models/gold/core/core__fact_event_logs.yml b/models/gold/core/core__fact_event_logs.yml index 4613c7f1..e44a302c 100644 --- a/models/gold/core/core__fact_event_logs.yml +++ b/models/gold/core/core__fact_event_logs.yml @@ -21,7 +21,7 @@ models: - name: EVENT_REMOVED description: '{{ doc("opt_event_removed") }}' - name: _LOG_ID - description: '{{ doc("opt_log_id_events") }}' + description: '{{ doc("internal_column") }}' - name: TX_STATUS description: '{{ doc("opt_tx_status") }}' - name: ORIGIN_FUNCTION_SIGNATURE @@ -30,3 +30,9 @@ models: description: '{{ doc("opt_origin_from") }}' - name: ORIGIN_TO_ADDRESS description: '{{ doc("opt_origin_to") }}' + - name: FACT_EVENT_LOGS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_l1_state_root_submissions.sql b/models/gold/core/core__fact_l1_state_root_submissions.sql index 614f53ba..83aa275c 100644 --- a/models/gold/core/core__fact_l1_state_root_submissions.sql +++ b/models/gold/core/core__fact_l1_state_root_submissions.sql @@ -8,11 +8,31 @@ SELECT state_tx_hash AS l1_state_root_tx_hash, state_block_number AS l1_block_number, state_block_timestamp AS l1_block_timestamp, - COALESCE(state_batch_index,bedrock_state_batch_index) AS l1_state_root_batch_index, - COALESCE(state_batch_root,bedrock_state_batch_root) AS batch_root, + COALESCE( + state_batch_index, + bedrock_state_batch_index + ) AS l1_state_root_batch_index, + COALESCE( + state_batch_root, + bedrock_state_batch_root + ) AS batch_root, state_batch_size AS batch_size, state_prev_total_elements AS prev_total_elements, state_min_block AS op_min_block, - state_max_block AS op_max_block + state_max_block AS op_max_block, + COALESCE( + state_hashes_id, + {{ dbt_utils.generate_surrogate_key( + ['state_block_number'] + ) }} + ) AS fact_l1_state_root_submissions_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('bronze__state_hashes') }} diff --git a/models/gold/core/core__fact_l1_state_root_submissions.yml b/models/gold/core/core__fact_l1_state_root_submissions.yml index cfb0387c..218338bb 100644 --- a/models/gold/core/core__fact_l1_state_root_submissions.yml +++ b/models/gold/core/core__fact_l1_state_root_submissions.yml @@ -22,3 +22,9 @@ models: description: '{{ doc("op_min_block") }}' - name: OP_MAX_BLOCK description: '{{ doc("op_max_block") }}' + - name: FACT_L1_STATE_ROOT_SUBMISSIONS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__fact_l1_submissions.sql b/models/gold/core/core__fact_l1_submissions.sql index 67483e3f..187d6e7b 100644 --- a/models/gold/core/core__fact_l1_submissions.sql +++ b/models/gold/core/core__fact_l1_submissions.sql @@ -13,6 +13,20 @@ SELECT l1_submission_size AS batch_size, l1_submission_prev_total_elements AS prev_total_elements, sub_min_block AS op_min_block, - sub_max_block AS op_max_block + sub_max_block AS op_max_block, + COALESCE( + submission_hashes_id, + {{ dbt_utils.generate_surrogate_key( + ['l1_submission_block_number'] + ) }} + ) AS fact_l1_state_root_submissions_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('bronze__submission_hashes') }} diff --git a/models/gold/core/core__fact_l1_submissions.yml b/models/gold/core/core__fact_l1_submissions.yml index 2db751c4..13983781 100644 --- a/models/gold/core/core__fact_l1_submissions.yml +++ b/models/gold/core/core__fact_l1_submissions.yml @@ -21,4 +21,10 @@ models: - name: OP_MIN_BLOCK description: '{{ doc("op_min_block") }}' - name: OP_MAX_BLOCK - description: '{{ doc("op_max_block") }}' \ No newline at end of file + description: '{{ doc("op_max_block") }}' + - name: FACT_L1_SUBMISSIONS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_token_transfers.sql b/models/gold/core/core__fact_token_transfers.sql index 9dffa5a8..8908c546 100644 --- a/models/gold/core/core__fact_token_transfers.sql +++ b/models/gold/core/core__fact_token_transfers.sql @@ -17,6 +17,20 @@ SELECT to_address, raw_amount, raw_amount_precise, - _log_id + _log_id, + COALESCE ( + transfers_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS fact_token_transfers_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__transfers') }} diff --git a/models/gold/core/core__fact_token_transfers.yml b/models/gold/core/core__fact_token_transfers.yml index a9f0902d..3fb07029 100644 --- a/models/gold/core/core__fact_token_transfers.yml +++ b/models/gold/core/core__fact_token_transfers.yml @@ -29,4 +29,10 @@ models: - name: RAW_AMOUNT_PRECISE description: '{{ doc("opt_transfer_raw_amount_precise") }}' - name: _LOG_ID - description: '{{ doc("opt_log_id_transfers") }}' \ No newline at end of file + description: '{{ doc("internal_column") }}' + - name: FACT_TOKEN_TRANSFERS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql index ed3211b5..7fe02b09 100644 --- a/models/gold/core/core__fact_traces.sql +++ b/models/gold/core/core__fact_traces.sql @@ -10,9 +10,9 @@ SELECT block_timestamp, from_address, to_address, - eth_value, - eth_value_precise_raw, - eth_value_precise, + eth_value AS VALUE, + eth_value_precise_raw AS value_precise_raw, + eth_value_precise AS value_precise, gas, gas_used, input, @@ -24,6 +24,23 @@ SELECT sub_traces, trace_status, error_reason, - trace_index + trace_index, + COALESCE ( + traces_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} + ) AS fact_traces_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp, + eth_value, + eth_value_precise_raw, + eth_value_precise FROM {{ ref('silver__traces') }} diff --git a/models/gold/core/core__fact_traces.yml b/models/gold/core/core__fact_traces.yml index f3a9d474..5a4c75cd 100644 --- a/models/gold/core/core__fact_traces.yml +++ b/models/gold/core/core__fact_traces.yml @@ -15,10 +15,16 @@ models: - name: TO_ADDRESS description: '{{ doc("opt_traces_to") }}' - name: ETH_VALUE - description: '{{ doc("opt_traces_value") }}' + description: '{{ doc("amount_deprecation") }}' - name: ETH_VALUE_PRECISE_RAW - description: '{{ doc("precise_amount_unadjusted") }}' + description: '{{ doc("amount_deprecation") }}' - name: ETH_VALUE_PRECISE + description: '{{ doc("amount_deprecation") }}' + - name: VALUE + description: '{{ doc("opt_traces_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: VALUE_PRECISE description: '{{ doc("precise_amount_adjusted") }}' - name: GAS description: '{{ doc("opt_traces_gas") }}' @@ -44,7 +50,9 @@ models: description: The reason for the trace failure, if any. - name: TRACE_INDEX description: The index of the trace within the transaction. - - - - \ No newline at end of file + - name: FACT_TRACES_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 295452b2..0101dab7 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -14,118 +14,62 @@ SELECT origin_function_signature, from_address, to_address, - eth_value, - eth_value_precise_raw, - eth_value_precise, + VALUE, + value_precise_raw, + value_precise, tx_fee, tx_fee_precise, gas_price, effective_gas_price, - gas_limit, + gas AS gas_limit, gas_used, l1_gas_price, l1_gas_used, l1_fee_scalar, - l1_fee, l1_fee_precise, + l1_fee, l1_submission_batch_index, l1_submission_tx_hash, - l1_state_root_batch_index, - l1_state_root_tx_hash, - l1_submission_details, + state_batch_index AS l1_state_root_batch_index, + state_tx_hash AS l1_state_root_tx_hash, + OBJECT_CONSTRUCT( + 'l1_bedrock_state_batch_index', + bedrock_state_batch_index, + 'l1_bedrock_state_batch_root', + bedrock_state_batch_root, + 'l1_state_root_batch_index', + state_batch_index, + 'l1_state_root_tx_hash', + state_tx_hash, + 'l1_submission_batch_index', + l1_submission_batch_index, + 'l1_submission_batch_root', + l1_submission_batch_root, + 'l1_submission_tx_hash', + l1_submission_tx_hash + ) AS l1_submission_details, cumulative_gas_used, max_fee_per_gas, max_priority_fee_per_gas, input_data, - status, + tx_status AS status, r, s, v, - tx_type + tx_type, + COALESCE ( + transactions_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} + ) AS fact_transactions_id, + GREATEST(COALESCE(A.inserted_timestamp, '2000-01-01'), COALESCE(b.inserted_timestamp, '2000-01-01'), COALESCE(C.inserted_timestamp, '2000-01-01')) AS inserted_timestamp, + GREATEST(COALESCE(A.modified_timestamp, '2000-01-01'), COALESCE(b.modified_timestamp, '2000-01-01'), COALESCE(C.modified_timestamp, '2000-01-01')) AS modified_timestamp, + VALUE AS eth_value, + value_precise_raw AS eth_value_precise_raw, + value_precise AS eth_value_precise FROM - ( - SELECT - block_number, - block_timestamp, - block_hash, - tx_hash, - nonce, - POSITION, - origin_function_signature, - from_address, - to_address, - VALUE AS eth_value, - tx_fee, - tx_fee_precise, - gas_price, - effective_gas_price, - gas AS gas_limit, - gas_used, - l1_gas_price, - l1_gas_used, - l1_fee_scalar, - IFF(LENGTH(l1_fee_precise_raw) > 18, LEFT(l1_fee_precise_raw, LENGTH(l1_fee_precise_raw) - 18) || '.' || RIGHT(l1_fee_precise_raw, 18), '0.' || LPAD(l1_fee_precise_raw, 18, '0')) AS rough_conversion_l1fee, - IFF( - POSITION( - '.000000000000000000' IN rough_conversion_l1fee - ) > 0, - LEFT( - rough_conversion_l1fee, - LENGTH(rough_conversion_l1fee) - 19 - ), - REGEXP_REPLACE( - rough_conversion_l1fee, - '0*$', - '' - ) - ) AS l1_fee_precise, - (l1_fee_precise_raw / pow(10, 18)) :: FLOAT AS l1_fee, - l1_submission_batch_index, - l1_submission_tx_hash, - state_batch_index AS l1_state_root_batch_index, - state_tx_hash AS l1_state_root_tx_hash, - OBJECT_CONSTRUCT( - 'l1_bedrock_state_batch_index', - bedrock_state_batch_index, - 'l1_bedrock_state_batch_root', - bedrock_state_batch_root, - 'l1_state_root_batch_index', - state_batch_index, - 'l1_state_root_tx_hash', - state_tx_hash, - 'l1_submission_batch_index', - l1_submission_batch_index, - 'l1_submission_batch_root', - l1_submission_batch_root, - 'l1_submission_tx_hash', - l1_submission_tx_hash - ) AS l1_submission_details, - cumulative_gas_used, - max_fee_per_gas, - max_priority_fee_per_gas, - input_data, - tx_status AS status, - r, - s, - v, - tx_type, - to_varchar( - TO_NUMBER(REPLACE(DATA :value :: STRING, '0x'), REPEAT('X', LENGTH(REPLACE(DATA :value :: STRING, '0x')))) - ) AS eth_value_precise_raw, - IFF(LENGTH(eth_value_precise_raw) > 18, LEFT(eth_value_precise_raw, LENGTH(eth_value_precise_raw) - 18) || '.' || RIGHT(eth_value_precise_raw, 18), '0.' || LPAD(eth_value_precise_raw, 18, '0')) AS rough_conversion, - IFF( - POSITION( - '.000000000000000000' IN rough_conversion - ) > 0, - LEFT(rough_conversion, LENGTH(rough_conversion) - 19), - REGEXP_REPLACE( - rough_conversion, - '0*$', - '' - ) - ) AS eth_value_precise - FROM - {{ ref('silver__transactions') }} - LEFT JOIN {{ ref('silver__state_hashes') }} USING (block_number) - LEFT JOIN {{ ref('silver__submission_hashes') }} USING (block_number) - ) + {{ ref('silver__transactions') }} A + LEFT JOIN {{ ref('silver__state_hashes') }} + b USING (block_number) + LEFT JOIN {{ ref('silver__submission_hashes') }} C USING (block_number) diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml index 3a1dd76d..413c62c3 100644 --- a/models/gold/core/core__fact_transactions.yml +++ b/models/gold/core/core__fact_transactions.yml @@ -21,10 +21,16 @@ models: - name: TO_ADDRESS description: '{{ doc("opt_to_address") }}' - name: ETH_VALUE - description: '{{ doc("opt_value") }}' + description: '{{ doc("amount_deprecation") }}' - name: ETH_VALUE_PRECISE_RAW - description: '{{ doc("precise_amount_unadjusted") }}' + description: '{{ doc("amount_deprecation") }}' - name: ETH_VALUE_PRECISE + description: '{{ doc("amount_deprecation") }}' + - name: VALUE + description: '{{ doc("opt_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: VALUE_PRECISE description: '{{ doc("precise_amount_adjusted") }}' - name: TX_FEE description: '{{ doc("opt_tx_fee") }}' @@ -75,4 +81,10 @@ models: - name: V description: The v value of the transaction signature. - name: TX_TYPE - description: The type of the transaction, 2 for EIP-1559 transactions and 0 for legacy transactions. \ No newline at end of file + description: The type of the transaction, 2 for EIP-1559 transactions and 0 for legacy transactions. + - name: FACT_TRANSACTIONS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/defi/defi__dim_dex_liquidity_pools.sql b/models/gold/defi/defi__dim_dex_liquidity_pools.sql index 05ca3c26..c284b594 100644 --- a/models/gold/defi/defi__dim_dex_liquidity_pools.sql +++ b/models/gold/defi/defi__dim_dex_liquidity_pools.sql @@ -22,6 +22,20 @@ SELECT pool_name, tokens, symbols, - decimals + decimals, + COALESCE ( + complete_dex_liquidity_pools_id, + {{ dbt_utils.generate_surrogate_key( + ['block_number','platform','version'] + ) }} + ) AS dim_dex_liquidity_pools_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver_dex__complete_dex_liquidity_pools') }} \ No newline at end of file diff --git a/models/gold/defi/defi__dim_dex_liquidity_pools.yml b/models/gold/defi/defi__dim_dex_liquidity_pools.yml index 9c828f15..27b773f8 100644 --- a/models/gold/defi/defi__dim_dex_liquidity_pools.yml +++ b/models/gold/defi/defi__dim_dex_liquidity_pools.yml @@ -23,4 +23,10 @@ models: - name: SYMBOLS description: '{{ doc("eth_dex_lp_symbols") }}' - name: DECIMALS - description: '{{ doc("eth_dex_lp_decimals") }}' \ No newline at end of file + description: '{{ doc("eth_dex_lp_decimals") }}' + - name: DIM_DEX_LIQUIDITY_POOLS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql index 703f8160..8d548422 100644 --- a/models/gold/defi/defi__ez_dex_swaps.sql +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -36,6 +36,20 @@ SELECT token_out, symbol_in, symbol_out, - _log_id + _log_id, + COALESCE ( + complete_dex_swaps_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index'] + ) }} + ) AS ez_dex_swaps_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver_dex__complete_dex_swaps') }} \ No newline at end of file diff --git a/models/gold/defi/defi__ez_dex_swaps.yml b/models/gold/defi/defi__ez_dex_swaps.yml index af12b173..8fdb189e 100644 --- a/models/gold/defi/defi__ez_dex_swaps.yml +++ b/models/gold/defi/defi__ez_dex_swaps.yml @@ -39,7 +39,7 @@ models: - name: EVENT_INDEX description: '{{ doc("opt_event_index") }}' - name: _LOG_ID - description: '{{ doc("opt_log_id_events") }}' + description: '{{ doc("internal_column") }}' - name: ORIGIN_FUNCTION_SIGNATURE description: '{{ doc("opt_tx_origin_sig") }}' - name: ORIGIN_FROM_ADDRESS @@ -50,3 +50,9 @@ models: description: '{{ doc("eth_dex_swaps_amount_in_unadj") }}' - name: AMOUNT_OUT_UNADJ description: '{{ doc("eth_dex_swaps_amount_out_unadj") }}' + - name: EZ_DEX_SWAPS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/governance/gov__fact_delegations.sql b/models/gold/governance/gov__fact_delegations.sql index 61114850..0b22cdbe 100644 --- a/models/gold/governance/gov__fact_delegations.sql +++ b/models/gold/governance/gov__fact_delegations.sql @@ -17,6 +17,20 @@ SELECT new_balance, previous_balance, raw_new_balance, - raw_previous_balance + raw_previous_balance, + COALESCE ( + delegations_id, + {{ dbt_utils.generate_surrogate_key( + ['_log_id'] + ) }} + ) AS fact_delegations_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__delegations') }} \ No newline at end of file diff --git a/models/gold/governance/gov__fact_delegations.yml b/models/gold/governance/gov__fact_delegations.yml index 57066969..6eb897b6 100644 --- a/models/gold/governance/gov__fact_delegations.yml +++ b/models/gold/governance/gov__fact_delegations.yml @@ -43,3 +43,9 @@ models: description: The amount of OP the user had delegated before the transaction. tests: - dbt_expectations.expect_column_to_exist + - name: FACT_DELEGATIONS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/nft/nft__ez_nft_sales.sql b/models/gold/nft/nft__ez_nft_sales.sql index ab863121..565e4820 100644 --- a/models/gold/nft/nft__ez_nft_sales.sql +++ b/models/gold/nft/nft__ez_nft_sales.sql @@ -15,6 +15,7 @@ SELECT block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -37,6 +38,20 @@ SELECT creator_fee_usd, origin_from_address, origin_to_address, - origin_function_signature + origin_function_signature, + COALESCE ( + complete_nft_sales_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index', 'nft_address','tokenId','platform_exchange_version'] + ) }} + ) AS ez_nft_sales_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__complete_nft_sales') }} diff --git a/models/gold/nft/nft__ez_nft_sales.yml b/models/gold/nft/nft__ez_nft_sales.yml index 04c75935..0a2bf5d4 100644 --- a/models/gold/nft/nft__ez_nft_sales.yml +++ b/models/gold/nft/nft__ez_nft_sales.yml @@ -10,6 +10,8 @@ models: description: '{{ doc("nft_blocktime") }}' - name: TX_HASH description: '{{ doc("nft_tx_hash") }}' + - name: EVENT_INDEX + description: '{{ doc("nft_event_index") }}' - name: EVENT_TYPE description: '{{ doc("nft_event_type") }}' - name: PLATFORM_ADDRESS @@ -54,3 +56,9 @@ models: description: '{{ doc("nft_origin_to") }}' - name: ORIGIN_FUNCTION_SIGNATURE description: '{{ doc("nft_origin_sig") }}' + - name: EZ_NFT_SALES_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/nft/nft__ez_nft_transfers.sql b/models/gold/nft/nft__ez_nft_transfers.sql index cd8287db..486b18a4 100644 --- a/models/gold/nft/nft__ez_nft_transfers.sql +++ b/models/gold/nft/nft__ez_nft_transfers.sql @@ -10,12 +10,27 @@ SELECT block_number, tx_hash, event_index, + intra_event_index, event_type, contract_address AS nft_address, project_name, from_address AS nft_from_address, to_address AS nft_to_address, tokenId, - erc1155_value + erc1155_value, + COALESCE ( + nft_transfers_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index','intra_event_index'] + ) }} + ) AS ez_nft_transfers_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__nft_transfers') }} diff --git a/models/gold/nft/nft__ez_nft_transfers.yml b/models/gold/nft/nft__ez_nft_transfers.yml index 4d06a160..dfc0c45f 100644 --- a/models/gold/nft/nft__ez_nft_transfers.yml +++ b/models/gold/nft/nft__ez_nft_transfers.yml @@ -12,6 +12,8 @@ models: description: '{{ doc("nft_tx_hash") }}' - name: EVENT_INDEX description: '{{ doc("nft_event_index") }}' + - name: INTRA_EVENT_INDEX + description: '{{ doc("nft_intra_event_index") }}' - name: EVENT_TYPE description: '{{ doc("nft_event_type") }}' - name: NFT_ADDRESS @@ -26,3 +28,9 @@ models: description: '{{ doc("nft_tokenid") }}' - name: ERC1155_VALUE description: '{{ doc("nft_erc1155_value") }}' + - name: EZ_NFT_TRANSFERS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/price/price__dim_asset_metadata.sql b/models/gold/price/price__dim_asset_metadata.sql index 8b6c9043..794c6154 100644 --- a/models/gold/price/price__dim_asset_metadata.sql +++ b/models/gold/price/price__dim_asset_metadata.sql @@ -10,6 +10,20 @@ SELECT symbol, NAME, decimals, - provider + provider, + COALESCE ( + asset_metadata_all_providers_id, + {{ dbt_utils.generate_surrogate_key( + ['token_address','symbol','id','provider'] + ) }} + ) AS dim_asset_metadata_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__asset_metadata_all_providers') }} \ No newline at end of file diff --git a/models/gold/price/price__dim_asset_metadata.yml b/models/gold/price/price__dim_asset_metadata.yml index 866753dc..fe7022b2 100644 --- a/models/gold/price/price__dim_asset_metadata.yml +++ b/models/gold/price/price__dim_asset_metadata.yml @@ -23,3 +23,9 @@ models: description: The specific address representing the asset in a specific platform. - name: DECIMALS description: The number of decimal places the token needs adjusted where token values exist. + - name: DIM_ASSET_METADATA_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/price/price__ez_asset_metadata.sql b/models/gold/price/price__ez_asset_metadata.sql index aa4a3dfc..b145c352 100644 --- a/models/gold/price/price__ez_asset_metadata.sql +++ b/models/gold/price/price__ez_asset_metadata.sql @@ -9,6 +9,20 @@ SELECT id, symbol, NAME, - decimals + decimals, + COALESCE ( + asset_metadata_priority_id, + {{ dbt_utils.generate_surrogate_key( + ['token_address'] + ) }} + ) AS ez_asset_metadata_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__asset_metadata_priority') }} \ No newline at end of file diff --git a/models/gold/price/price__ez_asset_metadata.yml b/models/gold/price/price__ez_asset_metadata.yml index eda50f74..a7a2c3ef 100644 --- a/models/gold/price/price__ez_asset_metadata.yml +++ b/models/gold/price/price__ez_asset_metadata.yml @@ -18,3 +18,9 @@ models: description: The specific address representing the asset in a specific platform. - name: DECIMALS description: The number of decimal places the token needs adjusted where token values exist. + - name: EZ_ASSET_METADATA_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/price/price__ez_hourly_token_prices.sql b/models/gold/price/price__ez_hourly_token_prices.sql index a584b9e0..53d6494a 100644 --- a/models/gold/price/price__ez_hourly_token_prices.sql +++ b/models/gold/price/price__ez_hourly_token_prices.sql @@ -10,6 +10,20 @@ SELECT symbol, decimals, price, - is_imputed + is_imputed, + COALESCE ( + hourly_prices_priority_id, + {{ dbt_utils.generate_surrogate_key( + ['token_address', 'hour'] + ) }} + ) AS ez_hourly_token_prices_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__hourly_prices_priority') }} diff --git a/models/gold/price/price__ez_hourly_token_prices.yml b/models/gold/price/price__ez_hourly_token_prices.yml index 3e155475..e705476e 100644 --- a/models/gold/price/price__ez_hourly_token_prices.yml +++ b/models/gold/price/price__ez_hourly_token_prices.yml @@ -20,4 +20,10 @@ models: - name: PRICE description: Closing price of the recorded hour in USD - name: IS_IMPUTED - description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) \ No newline at end of file + description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) + - name: EZ_HOURLY_TOKEN_PRICES_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/price/price__fact_hourly_token_prices.sql b/models/gold/price/price__fact_hourly_token_prices.sql index 1894e4a1..8bb363e5 100644 --- a/models/gold/price/price__fact_hourly_token_prices.sql +++ b/models/gold/price/price__fact_hourly_token_prices.sql @@ -9,6 +9,20 @@ SELECT token_address, price, is_imputed, - provider + provider, + COALESCE ( + hourly_prices_all_providers_id, + {{ dbt_utils.generate_surrogate_key( + ['token_address', 'hour', 'provider'] + ) }} + ) AS fact_hourly_token_prices_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__hourly_prices_all_providers') }} diff --git a/models/gold/price/price__fact_hourly_token_prices.yml b/models/gold/price/price__fact_hourly_token_prices.yml index 9337e7b0..3b12c101 100644 --- a/models/gold/price/price__fact_hourly_token_prices.yml +++ b/models/gold/price/price__fact_hourly_token_prices.yml @@ -13,4 +13,10 @@ models: - name: PRICE description: Closing price of the recorded hour in USD - name: IS_IMPUTED - description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) \ No newline at end of file + description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) + - name: FACT_HOURLY_TOKEN_PRICES_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.sql b/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.sql index e94868e7..24e00ccc 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.sql @@ -87,7 +87,40 @@ SELECT ) AS token_symbol, base.token_address AS token_address, claim_epoch, - max_epoch + max_epoch, + COALESCE ( + claimed_rewards_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS ez_claimed_rewards_id, + greatest( + COALESCE( + base.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + prices.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + C.inserted_timestamp, + '2000-01-01' + ) + ) AS inserted_timestamp, + greatest( + COALESCE( + base.modified_timestamp, + '2000-01-01' + ), + COALESCE( + prices.modified_timestamp, + '2000-01-01' + ), + COALESCE( + C.modified_timestamp, + '2000-01-01' + )) AS modified_timestamp FROM {{ ref('silver__velodrome_claimed_rewards') }} base diff --git a/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.yml b/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.yml index 5765bb20..81cc9aa3 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.yml +++ b/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.yml @@ -40,4 +40,9 @@ models: description: 'The claim epoch, only applicable to veNFT distributions.' - name: MAX_EPOCH description: 'The max epoch, only applicable to veNFT distributions.' - \ No newline at end of file + - name: EZ_CLAIMED_REWARDS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/protocols/velodrome/velodrome__ez_gauges.sql b/models/gold/protocols/velodrome/velodrome__ez_gauges.sql index d3016f3b..0dbf4f43 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_gauges.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_gauges.sql @@ -28,7 +28,33 @@ SELECT token0_symbol, token1_symbol, token0_address, - token1_address + token1_address, + COALESCE ( + gauges_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS ez_guages_id, + GREATEST( + COALESCE( + A.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + b.inserted_timestamp, + '2000-01-01' + ) + ) AS inserted_timestamp, + GREATEST( + COALESCE( + A.modified_timestamp, + '2000-01-01' + ), + COALESCE( + b.modified_timestamp, + '2000-01-01' + ) + ) AS modified_timestamp FROM {{ ref('silver__velodrome_gauges') }} A LEFT JOIN {{ ref('silver__velodrome_pools') }} diff --git a/models/gold/protocols/velodrome/velodrome__ez_lp_actions.sql b/models/gold/protocols/velodrome/velodrome__ez_lp_actions.sql index aa622e68..dddf4a19 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_lp_actions.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_lp_actions.sql @@ -59,7 +59,49 @@ SELECT token1_address, lp_token_action, lp_token_amount, - token0_amount_usd + token1_amount_usd AS lp_token_amount_usd + token0_amount_usd + token1_amount_usd AS lp_token_amount_usd, + COALESCE ( + lp_actions_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS ez_lp_actions_id, + GREATEST( + COALESCE( + base.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + pools.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + p0.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + p1.inserted_timestamp, + '2000-01-01' + ) + ) AS inserted_timestamp, + GREATEST( + COALESCE( + base.modified_timestamp, + '2000-01-01' + ), + COALESCE( + pools.modified_timestamp, + '2000-01-01' + ), + COALESCE( + p0.modified_timestamp, + '2000-01-01' + ), + COALESCE( + p1.modified_timestamp, + '2000-01-01' + ) + ) AS modified_timestamp FROM {{ ref('silver__velodrome_LP_actions') }} base diff --git a/models/gold/protocols/velodrome/velodrome__ez_lp_actions.yml b/models/gold/protocols/velodrome/velodrome__ez_lp_actions.yml index ea6fa860..c3fe678c 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_lp_actions.yml +++ b/models/gold/protocols/velodrome/velodrome__ez_lp_actions.yml @@ -54,3 +54,9 @@ models: - not_null - name: LP_TOKEN_AMOUNT_USD description: 'The amount of LP tokens minted or burned in USD.' + - name: EZ_LP_ACTIONS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/protocols/velodrome/velodrome__ez_pool_details.sql b/models/gold/protocols/velodrome/velodrome__ez_pool_details.sql index 9415a6e4..9efe9b9e 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_pool_details.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_pool_details.sql @@ -25,6 +25,20 @@ SELECT created_timestamp, created_block, created_hash, - _inserted_timestamp + _inserted_timestamp, + COALESCE ( + pools_id, + {{ dbt_utils.generate_surrogate_key( + ['pool_address'] + ) }} + ) AS ez_pool_details_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__velodrome_pools') }} diff --git a/models/gold/protocols/velodrome/velodrome__ez_pool_details.yml b/models/gold/protocols/velodrome/velodrome__ez_pool_details.yml index 2dc17cba..b45bb932 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_pool_details.yml +++ b/models/gold/protocols/velodrome/velodrome__ez_pool_details.yml @@ -28,3 +28,9 @@ models: description: 'The block at which the pool was created.' - name: CREATED_HASH description: 'The tx hash in which the pool was created.' + - name: EZ_POOL_DETAILS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/protocols/velodrome/velodrome__ez_staking_actions.sql b/models/gold/protocols/velodrome/velodrome__ez_staking_actions.sql index 55fc804f..69f56e7c 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_staking_actions.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_staking_actions.sql @@ -31,7 +31,33 @@ SELECT token0_symbol, token1_symbol, token0_address, - token1_address + token1_address, + COALESCE ( + locks_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS ez_staking_actions_id, + GREATEST( + COALESCE( + base.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + pools.inserted_timestamp, + '2000-01-01' + ) + ) AS inserted_timestamp, + GREATEST( + COALESCE( + base.modified_timestamp, + '2000-01-01' + ), + COALESCE( + pools.modified_timestamp, + '2000-01-01' + ) + ) AS modified_timestamp FROM {{ ref('silver__velodrome_staking_actions') }} base diff --git a/models/gold/protocols/velodrome/velodrome__ez_staking_actions.yml b/models/gold/protocols/velodrome/velodrome__ez_staking_actions.yml index 28a9d54f..eb25afbc 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_staking_actions.yml +++ b/models/gold/protocols/velodrome/velodrome__ez_staking_actions.yml @@ -86,4 +86,9 @@ models: - not_null - dbt_expectations.expect_column_values_to_match_regex: regex: 0[xX][0-9a-fA-F]+ - \ No newline at end of file + - name: EZ_STAKING_ACTIONS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/protocols/velodrome/velodrome__ez_swaps.sql b/models/gold/protocols/velodrome/velodrome__ez_swaps.sql index 67adc204..b47f03dc 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_swaps.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_swaps.sql @@ -171,7 +171,49 @@ SELECT AND pool_address <> '0xce9accfbb25eddce91845c3a7c3d1613d1d7081f' THEN token_address_in ELSE fee_currency END AS lp_fee_token_address, - _log_id + _log_id, + COALESCE ( + swaps_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS fact_blocks_id, + GREATEST( + COALESCE( + base.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + p0.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + p1.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + p2.inserted_timestamp, + '2000-01-01' + ) + ) AS inserted_timestamp, + GREATEST( + COALESCE( + base.modified_timestamp, + '2000-01-01' + ), + COALESCE( + p0.modified_timestamp, + '2000-01-01' + ), + COALESCE( + p1.modified_timestamp, + '2000-01-01' + ), + COALESCE( + p2.modified_timestamp, + '2000-01-01' + ) + ) AS modified_timestamp FROM {{ ref('silver__velodrome_swaps') }} base diff --git a/models/gold/protocols/velodrome/velodrome__ez_swaps.yml b/models/gold/protocols/velodrome/velodrome__ez_swaps.yml index 2411e1b2..cd8be181 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_swaps.yml +++ b/models/gold/protocols/velodrome/velodrome__ez_swaps.yml @@ -133,7 +133,12 @@ models: - dbt_expectations.expect_column_values_to_match_regex: regex: 0[xX][0-9a-fA-F]+ - name: _LOG_ID - description: '{{ doc("opt_log_id_events") }}' + description: '{{ doc("internal_column") }}' tests: - not_null - \ No newline at end of file + - name: EZ_SWAPS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/protocols/velodrome/velodrome__ez_velo_locks.sql b/models/gold/protocols/velodrome/velodrome__ez_velo_locks.sql index 3df83563..00ebddd4 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_velo_locks.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_velo_locks.sql @@ -30,9 +30,35 @@ SELECT velo_amount * price, 2 ) AS velo_amount_usd, - deposit_type + deposit_type, + COALESCE ( + locks_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS ez_velo_locks_id, + GREATEST( + COALESCE( + l.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + prices.inserted_timestamp, + '2000-01-01' + ) + ) AS inserted_timestamp, + GREATEST( + COALESCE( + l.modified_timestamp, + '2000-01-01' + ), + COALESCE( + prices.modified_timestamp, + '2000-01-01' + ) + ) AS modified_timestamp FROM - {{ ref('silver__velodrome_locks') }} + {{ ref('silver__velodrome_locks') }} l LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} prices ON HOUR = DATE_TRUNC( diff --git a/models/gold/protocols/velodrome/velodrome__ez_velo_locks.yml b/models/gold/protocols/velodrome/velodrome__ez_velo_locks.yml index b6e99972..6991bb24 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_velo_locks.yml +++ b/models/gold/protocols/velodrome/velodrome__ez_velo_locks.yml @@ -33,4 +33,10 @@ models: - name: VELO_AMOUNT_USD description: 'The amount of VELO tokens locked, in USD.' - name: DEPOSIT_TYPE - description: 'The type of deposit.' \ No newline at end of file + description: 'The type of deposit.' + - name: EZ_VELO_LOCKS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/protocols/velodrome/velodrome__ez_votes.sql b/models/gold/protocols/velodrome/velodrome__ez_votes.sql index 43919c62..b95cb5f2 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_votes.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_votes.sql @@ -2,14 +2,8 @@ materialized = 'view', persist_docs ={ "relation": true, "columns": true }, - meta={ - 'database_tags':{ - 'table': { - 'PROTOCOL': 'VELODROME', - 'PURPOSE': 'DEFI, DEX' - } - } - } + meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'VELODROME', + 'PURPOSE': 'DEFI, DEX' }}} ) }} SELECT @@ -42,7 +36,41 @@ SELECT from_address, token_id, vote_amount, - vote_action + vote_action, + COALESCE ( + votes_id, + {{ dbt_utils.generate_surrogate_key( + ['base.tx_hash', 'base.event_index'] + ) }} + ) AS ez_votes_id, + GREATEST( + COALESCE( + base.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + g1.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + g0.inserted_timestamp, + '2000-01-01' + ) + ) AS inserted_timestamp, + GREATEST( + COALESCE( + base.modified_timestamp, + '2000-01-01' + ), + COALESCE( + g1.modified_timestamp, + '2000-01-01' + ), + COALESCE( + g0.inserted_timestamp, + '2000-01-01' + ) + ) AS modified_timestamp FROM {{ ref('silver__velodrome_votes') }} base diff --git a/models/gold/protocols/velodrome/velodrome__ez_votes.yml b/models/gold/protocols/velodrome/velodrome__ez_votes.yml index b5e6154f..2fb2b79e 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_votes.yml +++ b/models/gold/protocols/velodrome/velodrome__ez_votes.yml @@ -55,4 +55,10 @@ models: - name: VOTE_AMOUNT description: 'The amount of tokens voted.' - name: VOTE_ACTION - description: 'The type of action, either vote or unvote.' \ No newline at end of file + description: 'The type of action, either vote or unvote.' + - name: EZ_VOTES_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index 1aa4ad56..dc550871 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -4,6 +4,7 @@ unique_key = "block_number", cluster_by = "block_timestamp::date", tags = ['core','non_realtime'], + merge_exclude_columns = ["inserted_timestamp"], full_refresh = false ) }} @@ -47,7 +48,13 @@ SELECT ) :: INT AS total_difficulty, DATA :transactionsRoot :: STRING AS transactions_root, DATA :uncles AS uncles, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {% if is_incremental() %} diff --git a/models/silver/core/silver__contracts.sql b/models/silver/core/silver__contracts.sql index 7282fda6..45be2cde 100644 --- a/models/silver/core/silver__contracts.sql +++ b/models/silver/core/silver__contracts.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = 'contract_address', + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -81,7 +82,13 @@ token_names AS ( token_name, token_decimals :: INTEGER AS token_decimals, token_symbol, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['c1.contract_address'] + ) }} AS contracts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM contracts c1 LEFT JOIN token_names diff --git a/models/silver/core/silver__created_contracts.sql b/models/silver/core/silver__created_contracts.sql index ed3e3f30..7457d057 100644 --- a/models/silver/core/silver__created_contracts.sql +++ b/models/silver/core/silver__created_contracts.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "incremental", unique_key = "created_contract_address", + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -11,7 +12,13 @@ SELECT to_address AS created_contract_address, from_address AS creator_address, input AS created_contract_input, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['to_address'] + ) }} AS created_contracts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('silver__traces') }} WHERE diff --git a/models/silver/core/silver__decoded_logs.sql b/models/silver/core/silver__decoded_logs.sql index 997578f6..d85fdddf 100644 --- a/models/silver/core/silver__decoded_logs.sql +++ b/models/silver/core/silver__decoded_logs.sql @@ -6,6 +6,7 @@ incremental_predicates = ["dynamic_range", "block_number"], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", full_refresh = false, + merge_exclude_columns = ["inserted_timestamp"], tags = ['decoded_logs','reorg'] ) }} @@ -191,7 +192,13 @@ SELECT DATA, event_removed, tx_status, - is_pending + is_pending, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS decoded_logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM new_records @@ -216,7 +223,13 @@ SELECT DATA, event_removed, tx_status, - is_pending + is_pending, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS decoded_logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM missing_data -{% endif %} \ No newline at end of file +{% endif %} diff --git a/models/silver/core/silver__logs.sql b/models/silver/core/silver__logs.sql index 1b2849ea..8de38ef4 100644 --- a/models/silver/core/silver__logs.sql +++ b/models/silver/core/silver__logs.sql @@ -176,7 +176,13 @@ FROM {% endif %} ) SELECT - * + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, event_index ORDER BY diff --git a/models/silver/core/silver__native_transfers.sql b/models/silver/core/silver__native_transfers.sql new file mode 100644 index 00000000..4ae5a47f --- /dev/null +++ b/models/silver/core/silver__native_transfers.sql @@ -0,0 +1,109 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'block_number', + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core','non_realtime','reorg'] +) }} + +WITH eth_base AS ( + + SELECT + tx_hash, + block_number, + block_timestamp, + identifier, + from_address, + to_address, + eth_value, + _call_id, + _inserted_timestamp, + eth_value_precise_raw, + eth_value_precise, + tx_position, + trace_index + FROM + {{ ref('silver__traces') }} + WHERE + eth_value > 0 + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' + AND TYPE NOT IN ( + 'DELEGATECALL', + 'STATICCALL' + ) + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '72 hours' + FROM + {{ this }} +) +{% endif %} +), +tx_table AS ( + SELECT + block_number, + tx_hash, + from_address AS origin_from_address, + to_address AS origin_to_address, + origin_function_signature + FROM + {{ ref('silver__transactions') }} + WHERE + tx_hash IN ( + SELECT + DISTINCT tx_hash + FROM + eth_base + ) + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '72 hours' + FROM + {{ this }} +) +{% endif %} +) +SELECT + tx_hash AS tx_hash, + block_number AS block_number, + block_timestamp AS block_timestamp, + identifier AS identifier, + origin_from_address, + origin_to_address, + origin_function_signature, + from_address, + to_address, + eth_value AS amount, + eth_value_precise_raw AS amount_precise_raw, + eth_value_precise AS amount_precise, + ROUND( + eth_value * price, + 2 + ) AS amount_usd, + _call_id, + _inserted_timestamp, + tx_position, + trace_index, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} AS native_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + eth_base A + LEFT JOIN {{ ref('silver__hourly_prices_priority_eth') }} + ON DATE_TRUNC( + 'hour', + A.block_timestamp + ) = HOUR + JOIN tx_table USING ( + tx_hash, + block_number + ) diff --git a/models/silver/core/silver__traces.sql b/models/silver/core/silver__traces.sql index 06e6dbd5..3392d9d6 100644 --- a/models/silver/core/silver__traces.sql +++ b/models/silver/core/silver__traces.sql @@ -384,7 +384,13 @@ FROM {% endif %} ) SELECT - * + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} AS traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index ORDER BY diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index 9d1d82c8..1763322a 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -107,7 +107,12 @@ base_tx AS ( A.data :v :: STRING AS v, utils.udf_hex_to_int( A.data :value :: STRING - ) :: FLOAT AS VALUE, + ) AS value_precise_raw, + utils.udf_decimal_adjust( + value_precise_raw, + 18 + ) AS value_precise, + value_precise :: FLOAT AS VALUE, A._INSERTED_TIMESTAMP, A.data FROM @@ -133,6 +138,8 @@ new_records AS ( t.position, t.type, t.v, + t.value_precise_raw, + t.value_precise, t.value, block_timestamp, CASE @@ -145,7 +152,16 @@ new_records AS ( r.l1_fee_scalar, r.l1_gas_used, r.l1_gas_price, - utils.udf_decimal_adjust((r.gas_used * t.gas_price) + ((r.l1_gas_price / pow(10, 9)) * r.l1_gas_used * r.l1_fee_scalar), 9) AS tx_fee_precise, + utils.udf_decimal_adjust( + ( + r.gas_used * utils.udf_hex_to_int( + t.data :gasPrice :: STRING + ) :: bigint + ) + FLOOR( + r.l1_gas_price * r.l1_gas_used * r.l1_fee_scalar + ), + 18 + ) AS tx_fee_precise, COALESCE( tx_fee_precise :: FLOAT, 0 @@ -198,6 +214,8 @@ missing_data AS ( t.position, t.type, t.v, + t.value_precise_raw, + t.value_precise, t.value, b.block_timestamp, FALSE AS is_pending, @@ -210,7 +228,16 @@ missing_data AS ( r.l1_fee_scalar, r.l1_gas_used, r.l1_gas_price, - utils.udf_decimal_adjust((r.gas_used * t.gas_price) + ((r.l1_gas_price / pow(10, 9)) * r.l1_gas_used * r.l1_fee_scalar), 9) AS tx_fee_precise_heal, + utils.udf_decimal_adjust( + ( + r.gas_used * utils.udf_hex_to_int( + t.data :gasPrice :: STRING + ) :: bigint + ) + FLOOR( + r.l1_gas_price * r.l1_gas_used * r.l1_fee_scalar + ), + 18 + ) AS tx_fee_precise_heal, COALESCE( tx_fee_precise_heal :: FLOAT, 0 @@ -266,6 +293,8 @@ FINAL AS ( POSITION, TYPE, v, + value_precise_raw, + value_precise, VALUE, block_timestamp, is_pending, @@ -307,6 +336,8 @@ SELECT POSITION, TYPE, v, + value_precise_raw, + value_precise, VALUE, block_timestamp, is_pending, @@ -347,10 +378,9 @@ SELECT POSITION, TYPE, v, - VALUE / pow( - 10, - 18 - ) AS VALUE, + VALUE, + value_precise_raw, + value_precise, block_timestamp, CASE WHEN CONCAT( @@ -370,7 +400,11 @@ SELECT tx_status, cumulative_gas_used, effective_gas_price, - l1_fee AS l1_fee_precise_raw, + utils.udf_decimal_adjust( + l1_fee, + 18 + ) AS l1_fee_precise, + l1_fee_precise :: FLOAT AS l1_fee, l1_fee_scalar, l1_gas_used, l1_gas_price / pow( @@ -381,7 +415,13 @@ SELECT tx_fee_precise, tx_type, _inserted_timestamp, - DATA + DATA, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL WHERE diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index b2e8c2d1..f4ee3327 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -239,7 +239,13 @@ heal_model AS ( has_decimal, has_price, _log_id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM token_transfers @@ -269,7 +275,13 @@ SELECT has_decimal, has_price, _log_id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM heal_model {% endif %} diff --git a/models/silver/core/silver__tx_count.sql b/models/silver/core/silver__tx_count.sql index e90e9194..0635342a 100644 --- a/models/silver/core/silver__tx_count.sql +++ b/models/silver/core/silver__tx_count.sql @@ -1,15 +1,18 @@ {{ config( materialized = 'incremental', unique_key = "block_number", + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} -SELECT - block_number, - MIN(_inserted_timestamp) AS _inserted_timestamp, - COUNT(*) AS tx_count -FROM - {{ ref('silver__transactions') }} +WITH base AS ( + + SELECT + block_number, + MIN(_inserted_timestamp) AS _inserted_timestamp, + COUNT(*) AS tx_count + FROM + {{ ref('silver__transactions') }} {% if is_incremental() %} WHERE @@ -21,4 +24,15 @@ WHERE ) {% endif %} GROUP BY - block_number \ No newline at end of file + block_number +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS tx_count_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + base diff --git a/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql b/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql index 1ef81191..e45aeaa3 100644 --- a/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql +++ b/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql @@ -4,7 +4,6 @@ unique_key = ['block_number','platform','version'], cluster_by = ['block_timestamp::DATE'], tags = ['curated','reorg'] - ) }} WITH contracts AS ( @@ -649,6 +648,12 @@ SELECT symbols, decimals, _id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number','platform','version'] + ) }} AS complete_dex_liquidity_pools_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL diff --git a/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql b/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql index b0c62c48..2080231f 100644 --- a/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql +++ b/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql @@ -166,13 +166,25 @@ hashflow_swaps AS ( token_out, CONCAT( LEAST( - COALESCE(symbol_in, CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42))), - COALESCE(symbol_out, CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42))) + COALESCE( + symbol_in, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + symbol_out, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) ), '-', GREATEST( - COALESCE(symbol_in, CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42))), - COALESCE(symbol_out, CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42))) + COALESCE( + symbol_in, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + symbol_out, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) ) ) AS pool_name, _log_id, @@ -228,13 +240,25 @@ hashflow_v3_swaps AS ( token_out, CONCAT( LEAST( - COALESCE(symbol_in, CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42))), - COALESCE(symbol_out, CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42))) + COALESCE( + symbol_in, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + symbol_out, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) ), '-', GREATEST( - COALESCE(symbol_in, CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42))), - COALESCE(symbol_out, CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42))) + COALESCE( + symbol_in, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + symbol_out, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) ) ) AS pool_name, _log_id, @@ -797,13 +821,25 @@ woofi_swaps AS ( token_out, CONCAT( LEAST( - COALESCE(symbol_in, CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42))), - COALESCE(symbol_out, CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42))) + COALESCE( + symbol_in, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + symbol_out, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) ), '-', GREATEST( - COALESCE(symbol_in, CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42))), - COALESCE(symbol_out, CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42))) + COALESCE( + symbol_in, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + symbol_out, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) ) ) AS pool_name, _log_id, @@ -1089,7 +1125,7 @@ all_dex_standard AS ( _inserted_timestamp FROM hashflow_swaps - UNION ALL + UNION ALL SELECT block_number, block_timestamp, @@ -1435,7 +1471,13 @@ SELECT symbol_in, symbol_out, f._log_id, - f._inserted_timestamp + f._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['f.tx_hash','f.event_index'] + ) }} AS complete_dex_swaps_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL f LEFT JOIN {{ ref('silver_dex__complete_dex_liquidity_pools') }} diff --git a/models/silver/ethereum/silver__state_hashes.sql b/models/silver/ethereum/silver__state_hashes.sql index af530e5f..738b045d 100644 --- a/models/silver/ethereum/silver__state_hashes.sql +++ b/models/silver/ethereum/silver__state_hashes.sql @@ -40,7 +40,16 @@ blocks AS ( SELECT SEQ4() AS block_number FROM - TABLE(GENERATOR(rowcount => (SELECT max(block_number) as max_block FROM {{ref ('silver__blocks')}}) )) + TABLE( + GENERATOR( + rowcount => ( + SELECT + MAX(block_number) AS max_block + FROM + {{ ref ('silver__blocks') }} + ) + ) + ) ) SELECT block_number, @@ -55,7 +64,13 @@ SELECT state_prev_total_elements, state_min_block, state_max_block, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS state_hashes_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM blocks INNER JOIN base diff --git a/models/silver/ethereum/silver__submission_hashes.sql b/models/silver/ethereum/silver__submission_hashes.sql index 2015c08e..88b0b003 100644 --- a/models/silver/ethereum/silver__submission_hashes.sql +++ b/models/silver/ethereum/silver__submission_hashes.sql @@ -51,7 +51,13 @@ SELECT l1_submission_prev_total_elements, sub_min_block, sub_max_block, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS submission_hashes_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM blocks INNER JOIN base diff --git a/models/silver/governance/silver__delegations.sql b/models/silver/governance/silver__delegations.sql index 62672fce..dc67c3d6 100644 --- a/models/silver/governance/silver__delegations.sql +++ b/models/silver/governance/silver__delegations.sql @@ -6,67 +6,129 @@ tags = ['non_realtime'] ) }} -SELECT - r.block_number, +SELECT + r.block_number, l.block_timestamp, r.tx_hash, - l.tx_status AS status, + l.tx_status AS status, CASE WHEN topics [0] :: STRING = '0x3134e8a2e6d97e929a7e54011ea5485d7d196dd5f0ba4d4ef95803e8e3fc257f' THEN 'DelegateChanged' WHEN topics [0] :: STRING = '0xdec2bacdd2f05b59de34da9b523dff8be42e5e38e818c82fdb0bae774387a724' THEN 'DelegateVotesChanged' END AS event_name, from_address AS delegator, - CASE - WHEN CONCAT('0x', SUBSTR(logs[0]:topics [2] :: STRING, 27, 40)) = '0x0000000000000000000000000000000000000000' - AND CONCAT('0x', SUBSTR(logs[0]:topics [3] :: STRING, 27, 40)) <> delegator THEN 'First Time Delegator' - WHEN CONCAT('0x', SUBSTR(logs[0]:topics [2] :: STRING, 27, 40)) = '0x0000000000000000000000000000000000000000' - AND delegator = CONCAT('0x', SUBSTR(logs[0]:topics [3] :: STRING, 27, 40)) THEN 'First Time Delegator - Self Delegation' - WHEN delegator = CONCAT('0x', SUBSTR(logs[0]:topics [3] :: STRING, 27, 40)) THEN 'Self-Delegation' - ELSE 'Re-Delegation' - END AS delegation_type, - CASE - WHEN delegation_type = 'Re-Delegation' AND event_name = 'DelegateVotesChanged' THEN CONCAT('0x', SUBSTR(l.topics [1] :: STRING, 27, 40)) - ELSE CONCAT('0x', SUBSTR(logs[0]:topics [3] :: STRING, 27, 40)) - END AS to_delegate, - CASE - WHEN delegation_type = 'Re-Delegation' THEN CONCAT('0x', SUBSTR(logs[0]:topics [2] :: STRING, 27, 40)) - WHEN delegation_type = 'First Time Delegator' AND event_name = 'DelegateChanged' THEN CONCAT('0x', SUBSTR(logs[0]:topics [2] :: STRING, 27, 40)) - ELSE NULL - END AS from_delegate, + CASE + WHEN CONCAT( + '0x', + SUBSTR( + logs [0] :topics [2] :: STRING, + 27, + 40 + ) + ) = '0x0000000000000000000000000000000000000000' + AND CONCAT( + '0x', + SUBSTR( + logs [0] :topics [3] :: STRING, + 27, + 40 + ) + ) <> delegator THEN 'First Time Delegator' + WHEN CONCAT( + '0x', + SUBSTR( + logs [0] :topics [2] :: STRING, + 27, + 40 + ) + ) = '0x0000000000000000000000000000000000000000' + AND delegator = CONCAT( + '0x', + SUBSTR( + logs [0] :topics [3] :: STRING, + 27, + 40 + ) + ) THEN 'First Time Delegator - Self Delegation' + WHEN delegator = CONCAT( + '0x', + SUBSTR( + logs [0] :topics [3] :: STRING, + 27, + 40 + ) + ) THEN 'Self-Delegation' + ELSE 'Re-Delegation' + END AS delegation_type, + CASE + WHEN delegation_type = 'Re-Delegation' + AND event_name = 'DelegateVotesChanged' THEN CONCAT('0x', SUBSTR(l.topics [1] :: STRING, 27, 40)) + ELSE CONCAT( + '0x', + SUBSTR( + logs [0] :topics [3] :: STRING, + 27, + 40 + ) + ) + END AS to_delegate, + CASE + WHEN delegation_type = 'Re-Delegation' THEN CONCAT( + '0x', + SUBSTR( + logs [0] :topics [2] :: STRING, + 27, + 40 + ) + ) + WHEN delegation_type = 'First Time Delegator' + AND event_name = 'DelegateChanged' THEN CONCAT( + '0x', + SUBSTR( + logs [0] :topics [2] :: STRING, + 27, + 40 + ) + ) + ELSE NULL + END AS from_delegate, COALESCE( - TRY_TO_NUMBER( - utils.udf_hex_to_int( - regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') [0] :: STRING - ) + TRY_TO_NUMBER( + utils.udf_hex_to_int( + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') [0] :: STRING) ), 0 ) AS raw_previous_balance, COALESCE( TRY_TO_NUMBER( utils.udf_hex_to_int( - regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') [1] :: STRING + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') [1] :: STRING) + ), + 0 + ) AS raw_new_balance, + COALESCE(raw_new_balance / pow(10, 18), 0) AS new_balance, + COALESCE(raw_previous_balance / pow(10, 18), 0) AS previous_balance, + r._inserted_timestamp, + l._log_id, + {{ dbt_utils.generate_surrogate_key( + ['l._log_id'] + ) }} AS delegations_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + {{ ref('silver__receipts') }} + r + LEFT OUTER JOIN {{ ref('silver__logs') }} + l + ON r.tx_hash = l.tx_hash + WHERE + origin_function_signature = '0x5c19a95c' + AND to_address = '0x4200000000000000000000000000000000000042' + AND topics [0] :: STRING IN ( + '0x3134e8a2e6d97e929a7e54011ea5485d7d196dd5f0ba4d4ef95803e8e3fc257f', + '0xdec2bacdd2f05b59de34da9b523dff8be42e5e38e818c82fdb0bae774387a724' ) - ), - 0 - ) AS raw_new_balance, - COALESCE(raw_new_balance / pow(10, 18), 0) AS new_balance, - COALESCE(raw_previous_balance / pow(10, 18), 0) AS previous_balance, - r._inserted_timestamp, - l._log_id -FROM - {{ ref('silver__receipts') }} r -LEFT OUTER JOIN - {{ ref('silver__logs')}} l -ON r.tx_hash = l.tx_hash - -WHERE - origin_function_signature = '0x5c19a95c' - AND to_address = '0x4200000000000000000000000000000000000042' - AND topics[0] :: STRING IN ( - '0x3134e8a2e6d97e929a7e54011ea5485d7d196dd5f0ba4d4ef95803e8e3fc257f', - '0xdec2bacdd2f05b59de34da9b523dff8be42e5e38e818c82fdb0bae774387a724' - ) - AND to_delegate IS NOT NULL + AND to_delegate IS NOT NULL {% if is_incremental() %} AND r._inserted_timestamp >= ( @@ -85,4 +147,4 @@ AND l._inserted_timestamp >= ( FROM {{ this }} ) -{% endif %} \ No newline at end of file +{% endif %} diff --git a/models/silver/labels/silver__labels.sql b/models/silver/labels/silver__labels.sql index 6955fde7..891d25c9 100644 --- a/models/silver/labels/silver__labels.sql +++ b/models/silver/labels/silver__labels.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = 'address', + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -13,7 +14,13 @@ SELECT label_type, label_subtype, address_name, - project_name + project_name, + {{ dbt_utils.generate_surrogate_key( + ['address'] + ) }} AS labels_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__labels') }} WHERE diff --git a/models/silver/nft/silver__complete_nft_sales.sql b/models/silver/nft/silver__complete_nft_sales.sql index bc7e03fd..6d84ae98 100644 --- a/models/silver/nft/silver__complete_nft_sales.sql +++ b/models/silver/nft/silver__complete_nft_sales.sql @@ -4,7 +4,6 @@ unique_key = ['block_number','platform_name','platform_exchange_version'], cluster_by = ['block_timestamp::DATE'], tags = ['curated','reorg'] - ) }} WITH nft_base_models AS ( @@ -13,6 +12,7 @@ WITH nft_base_models AS ( block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -54,6 +54,7 @@ SELECT block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -95,6 +96,7 @@ SELECT block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -136,6 +138,7 @@ SELECT block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -177,6 +180,7 @@ SELECT block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -272,6 +276,7 @@ final_base AS ( block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, CASE @@ -390,6 +395,7 @@ label_fill_sales AS ( block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -435,6 +441,30 @@ label_fill_sales AS ( WHERE t.project_name IS NULL AND C.token_name IS NOT NULL +), +blocks_fill AS ( + SELECT + * exclude ( + complete_nft_sales_id, + inserted_timestamp, + modified_timestamp, + _invocation_id + ) + FROM + {{ this }} + WHERE + block_number IN ( + SELECT + block_number + FROM + label_fill_sales + ) + AND nft_log_id NOT IN ( + SELECT + nft_log_id + FROM + label_fill_sales + ) ) {% endif %}, final_joins AS ( @@ -444,17 +474,23 @@ final_joins AS ( final_base {% if is_incremental() %} -UNION +UNION ALL SELECT * FROM label_fill_sales +UNION ALL +SELECT + * +FROM + blocks_fill {% endif %} ) SELECT block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -488,7 +524,13 @@ SELECT nft_log_id, input_data, _log_id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index', 'nft_address','tokenId','platform_exchange_version'] + ) }} AS complete_nft_sales_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM final_joins qualify(ROW_NUMBER() over(PARTITION BY nft_log_id ORDER BY diff --git a/models/silver/nft/silver__complete_nft_sales.yml b/models/silver/nft/silver__complete_nft_sales.yml index 6eb5eb31..4341d5d6 100644 --- a/models/silver/nft/silver__complete_nft_sales.yml +++ b/models/silver/nft/silver__complete_nft_sales.yml @@ -28,6 +28,9 @@ models: - not_null - dbt_expectations.expect_column_values_to_match_regex: regex: 0[xX][0-9a-fA-F]+ + - name: EVENT_INDEX + tests: + - not_null - name: PLATFORM_ADDRESS tests: - not_null diff --git a/models/silver/nft/silver__nft_transfers.sql b/models/silver/nft/silver__nft_transfers.sql index a895d173..0357c3c2 100644 --- a/models/silver/nft/silver__nft_transfers.sql +++ b/models/silver/nft/silver__nft_transfers.sql @@ -43,7 +43,7 @@ AND TO_TIMESTAMP_NTZ(_inserted_timestamp) >= ( SELECT MAX( _inserted_timestamp - ) + ) - INTERVAL '24 hours' FROM {{ this }} ) @@ -234,6 +234,7 @@ all_transfers AS ( erc1155_value, _inserted_timestamp, event_index, + 1 AS intra_event_index, 'erc721_Transfer' AS token_transfer_type, CONCAT( _log_id, @@ -256,6 +257,7 @@ all_transfers AS ( erc1155_value, _inserted_timestamp, event_index, + 1 AS intra_event_index, 'erc1155_TransferSingle' AS token_transfer_type, CONCAT( _log_id, @@ -280,6 +282,7 @@ all_transfers AS ( erc1155_value, _inserted_timestamp, event_index, + intra_event_index, 'erc1155_TransferBatch' AS token_transfer_type, CONCAT( _log_id, @@ -301,6 +304,7 @@ transfer_base AS ( block_timestamp, tx_hash, event_index, + intra_event_index, contract_address, C.token_name AS project_name, from_address, @@ -328,6 +332,7 @@ fill_transfers AS ( t.block_timestamp, t.tx_hash, t.event_index, + t.intra_event_index, t.contract_address, C.token_name AS project_name, t.from_address, @@ -348,6 +353,30 @@ fill_transfers AS ( WHERE t.project_name IS NULL AND C.token_name IS NOT NULL +), +blocks_fill AS ( + SELECT + * exclude ( + nft_transfers_id, + inserted_timestamp, + modified_timestamp, + _invocation_id + ) + FROM + {{ this }} + WHERE + block_number IN ( + SELECT + block_number + FROM + fill_transfers + ) + AND _log_id NOT IN ( + SELECT + _log_id + FROM + fill_transfers + ) ) {% endif %}, final_base AS ( @@ -356,6 +385,7 @@ final_base AS ( block_timestamp, tx_hash, event_index, + intra_event_index, contract_address, project_name, from_address, @@ -370,12 +400,13 @@ final_base AS ( transfer_base {% if is_incremental() %} -UNION +UNION ALL SELECT block_number, block_timestamp, tx_hash, event_index, + intra_event_index, contract_address, project_name, from_address, @@ -388,6 +419,25 @@ SELECT _inserted_timestamp FROM fill_transfers +UNION ALL +SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + intra_event_index, + contract_address, + project_name, + from_address, + to_address, + tokenId, + erc1155_value, + event_type, + token_transfer_type, + _log_id, + _inserted_timestamp +FROM + blocks_fill {% endif %} ) SELECT @@ -395,6 +445,7 @@ SELECT block_timestamp, tx_hash, event_index, + intra_event_index, contract_address, project_name, from_address, @@ -404,7 +455,13 @@ SELECT event_type, token_transfer_type, _log_id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index','intra_event_index'] + ) }} AS nft_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM final_base qualify ROW_NUMBER() over ( PARTITION BY _log_id diff --git a/models/silver/nft/silver__nft_transfers.yml b/models/silver/nft/silver__nft_transfers.yml index 6cdc8faa..6ba35b35 100644 --- a/models/silver/nft/silver__nft_transfers.yml +++ b/models/silver/nft/silver__nft_transfers.yml @@ -28,6 +28,12 @@ models: - not_null - dbt_expectations.expect_column_values_to_match_regex: regex: 0[xX][0-9a-fA-F]+ + - name: EVENT_INDEX + tests: + - not_null + - name: INTRA_EVENT_INDEX + tests: + - not_null - name: CONTRACT_ADDRESS tests: - not_null diff --git a/models/silver/prices/silver__asset_metadata_all_providers.sql b/models/silver/prices/silver__asset_metadata_all_providers.sql index 1bb61ef8..8ccb4223 100644 --- a/models/silver/prices/silver__asset_metadata_all_providers.sql +++ b/models/silver/prices/silver__asset_metadata_all_providers.sql @@ -1,5 +1,6 @@ {{ config( materialized = 'incremental', + merge_exclude_columns = ["inserted_timestamp"], unique_key = ['token_address','symbol','id','provider'], tags = ['non_realtime'] ) }} @@ -14,7 +15,13 @@ SELECT token_name AS NAME, token_decimals AS decimals, provider, - p._inserted_timestamp + p._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['token_address','symbol','id','provider'] + ) }} AS asset_metadata_all_providers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__asset_metadata_all_providers') }} p diff --git a/models/silver/prices/silver__asset_metadata_priority.sql b/models/silver/prices/silver__asset_metadata_priority.sql index 0c9a8c87..1e301f66 100644 --- a/models/silver/prices/silver__asset_metadata_priority.sql +++ b/models/silver/prices/silver__asset_metadata_priority.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = 'token_address', + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -11,14 +12,20 @@ SELECT C.token_symbol, p.symbol ) AS symbol, - C.token_name AS name, + C.token_name AS NAME, C.token_decimals AS decimals, p.provider, CASE WHEN p.provider = 'coingecko' THEN 1 WHEN p.provider = 'coinmarketcap' THEN 2 END AS priority, - p._inserted_timestamp + p._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['p.token_address'] + ) }} AS asset_metadata_priority_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__asset_metadata_priority') }} p diff --git a/models/silver/prices/silver__hourly_prices_all_providers.sql b/models/silver/prices/silver__hourly_prices_all_providers.sql index 892f3a90..a9a4b05f 100644 --- a/models/silver/prices/silver__hourly_prices_all_providers.sql +++ b/models/silver/prices/silver__hourly_prices_all_providers.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = ['token_address', 'hour', 'provider'], + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -10,7 +11,13 @@ SELECT provider, price, is_imputed, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['token_address', 'hour', 'provider'] + ) }} AS hourly_prices_all_providers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__hourly_prices_all_providers') }} WHERE diff --git a/models/silver/prices/silver__hourly_prices_priority.sql b/models/silver/prices/silver__hourly_prices_priority.sql index bb314174..09053f0b 100644 --- a/models/silver/prices/silver__hourly_prices_priority.sql +++ b/models/silver/prices/silver__hourly_prices_priority.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = ['token_address', 'hour'], + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -14,7 +15,13 @@ SELECT C.token_symbol, m.symbol ) AS symbol, - C.token_decimals AS decimals + C.token_decimals AS decimals, + {{ dbt_utils.generate_surrogate_key( + ['p.token_address', 'p.hour'] + ) }} AS hourly_prices_priority_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__hourly_prices_priority') }} p @@ -35,4 +42,4 @@ AND p._inserted_timestamp >= ( FROM {{ this }} ) -{% endif %} \ No newline at end of file +{% endif %} diff --git a/models/silver/prices/silver__hourly_prices_priority_eth.sql b/models/silver/prices/silver__hourly_prices_priority_eth.sql index 9ed40ef1..9bae884f 100644 --- a/models/silver/prices/silver__hourly_prices_priority_eth.sql +++ b/models/silver/prices/silver__hourly_prices_priority_eth.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = ['token_address', 'hour'], + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -9,7 +10,13 @@ SELECT token_address, price, is_imputed, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['token_address', 'hour'] + ) }} AS hourly_prices_priority_eth_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__hourly_prices_priority_eth') }} WHERE diff --git a/models/silver/protocols/velodrome/silver__velodrome_LP_actions.sql b/models/silver/protocols/velodrome/silver__velodrome_LP_actions.sql index 6835c4d4..cbdad348 100644 --- a/models/silver/protocols/velodrome/silver__velodrome_LP_actions.sql +++ b/models/silver/protocols/velodrome/silver__velodrome_LP_actions.sql @@ -123,7 +123,13 @@ SELECT function_type, _log_id, _inserted_timestamp, - b.lp_token_amount AS lp_token_amount + b.lp_token_amount AS lp_token_amount, + {{ dbt_utils.generate_surrogate_key( + ['a.tx_hash', 'event_index'] + ) }} AS lp_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM lp_actions A LEFT JOIN lp_tokens_actions b diff --git a/models/silver/protocols/velodrome/silver__velodrome_claimed_rewards.sql b/models/silver/protocols/velodrome/silver__velodrome_claimed_rewards.sql index 417896f7..8a32d319 100644 --- a/models/silver/protocols/velodrome/silver__velodrome_claimed_rewards.sql +++ b/models/silver/protocols/velodrome/silver__velodrome_claimed_rewards.sql @@ -4,7 +4,6 @@ unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], tags = ['curated','reorg'] - ) }} WITH velo_distributions AS ( @@ -154,7 +153,13 @@ SELECT claim_epoch, max_epoch, _log_id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS claimed_rewards_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id ORDER BY diff --git a/models/silver/protocols/velodrome/silver__velodrome_gauges.sql b/models/silver/protocols/velodrome/silver__velodrome_gauges.sql index 2566e632..2bea63b3 100644 --- a/models/silver/protocols/velodrome/silver__velodrome_gauges.sql +++ b/models/silver/protocols/velodrome/silver__velodrome_gauges.sql @@ -95,7 +95,13 @@ SELECT internal_bribe_address, pool_address, creator_address, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS gauges_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM all_gauges qualify(ROW_NUMBER() over(PARTITION BY gauge_address ORDER BY diff --git a/models/silver/protocols/velodrome/silver__velodrome_locks.sql b/models/silver/protocols/velodrome/silver__velodrome_locks.sql index 93e3960b..e2e4c2bb 100644 --- a/models/silver/protocols/velodrome/silver__velodrome_locks.sql +++ b/models/silver/protocols/velodrome/silver__velodrome_locks.sql @@ -87,7 +87,13 @@ SELECT velo_value AS velo_amount, deposit_type, _log_id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS locks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM new_locks qualify(ROW_NUMBER() over(PARTITION BY _log_id ORDER BY diff --git a/models/silver/protocols/velodrome/silver__velodrome_pool_details.sql b/models/silver/protocols/velodrome/silver__velodrome_pool_details.sql index f2de30a9..8eb78ff6 100644 --- a/models/silver/protocols/velodrome/silver__velodrome_pool_details.sql +++ b/models/silver/protocols/velodrome/silver__velodrome_pool_details.sql @@ -22,7 +22,13 @@ SELECT token0_decimals, token1_decimals, l._log_id, - p._inserted_timestamp + p._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['pool_address'] + ) }} AS pool_details_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('silver__velodrome_pools') }} p diff --git a/models/silver/protocols/velodrome/silver__velodrome_pools.sql b/models/silver/protocols/velodrome/silver__velodrome_pools.sql index 87bb4b82..56f4b37f 100644 --- a/models/silver/protocols/velodrome/silver__velodrome_pools.sql +++ b/models/silver/protocols/velodrome/silver__velodrome_pools.sql @@ -29,6 +29,7 @@ WITH pool_creation AS ( WHERE topics [0] :: STRING = '0xc4805696c66d7cf352fc1d6bb633ad5ee82f6cb577c453024b6e0eb8306c6fc9' -- pair created AND contract_address = '0x25cbddb98b35ab1ff77413456b31ec81a6b6b746' --velo deployer + {% if is_incremental() %} AND pool_address NOT IN ( SELECT @@ -38,110 +39,109 @@ AND pool_address NOT IN ( ) {% endif %} ), - function_sigs AS ( - -SELECT - '0x06fdde03' AS function_sig, - 'name' AS function_name -UNION ALL -SELECT - '0x95d89b41' AS function_sig, - 'symbol' AS function_name -UNION ALL -SELECT - '0x313ce567' AS function_sig, - 'decimals' AS function_name + SELECT + '0x06fdde03' AS function_sig, + 'name' AS function_name + UNION ALL + SELECT + '0x95d89b41' AS function_sig, + 'symbol' AS function_name + UNION ALL + SELECT + '0x313ce567' AS function_sig, + 'decimals' AS function_name ), - all_inputs AS ( - -SELECT - pool_address AS contract_address, - block_number, - function_sig, - (ROW_NUMBER() OVER (PARTITION BY pool_address - ORDER BY block_number)) - 1 AS function_input, - 'pool' AS address_label -FROM pool_creation -JOIN function_sigs ON 1=1 -UNION ALL -SELECT - token0_address AS contract_address, - block_number, - function_sig, - (ROW_NUMBER() OVER (PARTITION BY token0_address - ORDER BY block_number)) - 1 AS function_input, - 'token0' AS address_label -FROM pool_creation -JOIN function_sigs ON 1=1 -UNION ALL -SELECT - token1_address AS contract_address, - block_number, - function_sig, - (ROW_NUMBER() OVER (PARTITION BY token1_address - ORDER BY block_number)) - 1 AS function_input, - 'token1' AS address_label -FROM pool_creation -JOIN function_sigs ON 1=1 + SELECT + pool_address AS contract_address, + block_number, + function_sig, + (ROW_NUMBER() over (PARTITION BY pool_address + ORDER BY + block_number)) - 1 AS function_input, + 'pool' AS address_label + FROM + pool_creation + JOIN function_sigs + ON 1 = 1 + UNION ALL + SELECT + token0_address AS contract_address, + block_number, + function_sig, + (ROW_NUMBER() over (PARTITION BY token0_address + ORDER BY + block_number)) - 1 AS function_input, + 'token0' AS address_label + FROM + pool_creation + JOIN function_sigs + ON 1 = 1 + UNION ALL + SELECT + token1_address AS contract_address, + block_number, + function_sig, + (ROW_NUMBER() over (PARTITION BY token1_address + ORDER BY + block_number)) - 1 AS function_input, + 'token1' AS address_label + FROM + pool_creation + JOIN function_sigs + ON 1 = 1 ), - ready_reads_all AS ( -SELECT - contract_address, - block_number, - function_sig, - function_input, - CONCAT( - '[\'', + SELECT contract_address, - '\',', block_number, - ',\'', function_sig, - '\',\'', function_input, - '\']' + CONCAT( + '[\'', + contract_address, + '\',', + block_number, + ',\'', + function_sig, + '\',\'', + function_input, + '\']' ) AS read_input -FROM all_inputs + FROM + all_inputs ), - batch_reads_all AS ( - -SELECT - CONCAT('[', LISTAGG(read_input, ','), ']') AS batch_read -FROM - ready_reads_all + SELECT + CONCAT('[', LISTAGG(read_input, ','), ']') AS batch_read + FROM + ready_reads_all ), - all_reads AS ( - -SELECT - ethereum.streamline.udf_json_rpc_read_calls( - node_url, - headers, - PARSE_JSON(batch_read) - ) AS read_output, - SYSDATE() AS _inserted_timestamp -FROM - batch_reads_all -JOIN streamline.crosschain.node_mapping ON 1=1 - AND chain = 'optimism' -WHERE - EXISTS ( - SELECT - 1 - FROM - ready_reads_all - LIMIT - 1 - ) -), - -reads_adjusted AS ( - -SELECT + SELECT + ethereum.streamline.udf_json_rpc_read_calls( + node_url, + headers, + PARSE_JSON(batch_read) + ) AS read_output, + SYSDATE() AS _inserted_timestamp + FROM + batch_reads_all + JOIN streamline.crosschain.node_mapping + ON 1 = 1 + AND chain = 'optimism' + WHERE + EXISTS ( + SELECT + 1 + FROM + ready_reads_all + LIMIT + 1 + ) +), reads_adjusted AS ( + SELECT VALUE :id :: STRING AS read_id, VALUE :result :: STRING AS read_result, SPLIT( @@ -153,94 +153,166 @@ SELECT read_id_object [2] :: STRING AS function_sig, read_id_object [3] :: STRING AS function_input, _inserted_timestamp -FROM - all_reads, - LATERAL FLATTEN( - input => read_output [0] :data - ) + FROM + all_reads, + LATERAL FLATTEN( + input => read_output [0] :data + ) ), - details AS ( - -SELECT - contract_address, - function_sig, - function_name, - read_result, - regexp_substr_all(SUBSTR(read_result, 3, len(read_result)), '.{64}') AS segmented_output, - _inserted_timestamp -FROM reads_adjusted -LEFT JOIN function_sigs USING(function_sig) - ), - + SELECT + contract_address, + function_sig, + function_name, + read_result, + regexp_substr_all(SUBSTR(read_result, 3, len(read_result)), '.{64}') AS segmented_output, + _inserted_timestamp + FROM + reads_adjusted + LEFT JOIN function_sigs USING(function_sig) +), pools AS ( - -SELECT - d.contract_address AS pool_address, - MIN(CASE WHEN function_name = 'symbol' THEN utils.udf_hex_to_string(segmented_output [2] :: STRING) END) AS pool_symbol, - MIN(CASE WHEN function_name = 'name' THEN utils.udf_hex_to_string(segmented_output [2] :: STRING) END) AS pool_name, - MIN(CASE - WHEN read_result::STRING = '0x' THEN NULL - ELSE utils.udf_hex_to_int(read_result::STRING) - END)::INTEGER AS pool_decimals, - MAX(_inserted_timestamp) AS _inserted_timestamp -FROM details d -LEFT JOIN all_inputs i ON d.contract_address = i.contract_address -WHERE address_label = 'pool' -GROUP BY 1 + SELECT + d.contract_address AS pool_address, + MIN( + CASE + WHEN function_name = 'symbol' THEN utils.udf_hex_to_string( + segmented_output [2] :: STRING + ) + END + ) AS pool_symbol, + MIN( + CASE + WHEN function_name = 'name' THEN utils.udf_hex_to_string( + segmented_output [2] :: STRING + ) + END + ) AS pool_name, + MIN( + CASE + WHEN read_result :: STRING = '0x' THEN NULL + ELSE utils.udf_hex_to_int( + read_result :: STRING + ) + END + ) :: INTEGER AS pool_decimals, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + details d + LEFT JOIN all_inputs i + ON d.contract_address = i.contract_address + WHERE + address_label = 'pool' + GROUP BY + 1 ), - token0 AS ( - -SELECT - d.contract_address AS token0_address, - MIN(CASE WHEN function_name = 'symbol' THEN utils.udf_hex_to_string(segmented_output [2] :: STRING) END) AS token0_symbol, - MIN(CASE WHEN function_name = 'name' THEN utils.udf_hex_to_string(segmented_output [2] :: STRING) END) AS token0_name, - MIN(CASE - WHEN function_name = 'decimals' AND read_result::STRING <> '0x' THEN utils.udf_hex_to_int(segmented_output [0] :: STRING) - ELSE NULL - END)::INTEGER AS token0_decimals, - MAX(_inserted_timestamp) AS _inserted_timestamp -FROM details d -LEFT JOIN all_inputs i ON d.contract_address = i.contract_address -WHERE address_label = 'token0' -GROUP BY 1 + SELECT + d.contract_address AS token0_address, + MIN( + CASE + WHEN function_name = 'symbol' THEN utils.udf_hex_to_string( + segmented_output [2] :: STRING + ) + END + ) AS token0_symbol, + MIN( + CASE + WHEN function_name = 'name' THEN utils.udf_hex_to_string( + segmented_output [2] :: STRING + ) + END + ) AS token0_name, + MIN( + CASE + WHEN function_name = 'decimals' + AND read_result :: STRING <> '0x' THEN utils.udf_hex_to_int( + segmented_output [0] :: STRING + ) + ELSE NULL + END + ) :: INTEGER AS token0_decimals, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + details d + LEFT JOIN all_inputs i + ON d.contract_address = i.contract_address + WHERE + address_label = 'token0' + GROUP BY + 1 ), - token1 AS ( - -SELECT - d.contract_address AS token1_address, - MIN(CASE WHEN function_name = 'symbol' THEN utils.udf_hex_to_string(segmented_output [2] :: STRING) END) AS token1_symbol, - MIN(CASE WHEN function_name = 'name' THEN utils.udf_hex_to_string(segmented_output [2] :: STRING) END) AS token1_name, - MIN(CASE - WHEN function_name = 'decimals' AND read_result::STRING <> '0x' THEN utils.udf_hex_to_int(segmented_output [0] :: STRING) - ELSE NULL - END)::INTEGER AS token1_decimals, - MAX(_inserted_timestamp) AS _inserted_timestamp -FROM details d -LEFT JOIN all_inputs i ON d.contract_address = i.contract_address -WHERE address_label = 'token1' -GROUP BY 1 + SELECT + d.contract_address AS token1_address, + MIN( + CASE + WHEN function_name = 'symbol' THEN utils.udf_hex_to_string( + segmented_output [2] :: STRING + ) + END + ) AS token1_symbol, + MIN( + CASE + WHEN function_name = 'name' THEN utils.udf_hex_to_string( + segmented_output [2] :: STRING + ) + END + ) AS token1_name, + MIN( + CASE + WHEN function_name = 'decimals' + AND read_result :: STRING <> '0x' THEN utils.udf_hex_to_int( + segmented_output [0] :: STRING + ) + ELSE NULL + END + ) :: INTEGER AS token1_decimals, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + details d + LEFT JOIN all_inputs i + ON d.contract_address = i.contract_address + WHERE + address_label = 'token1' + GROUP BY + 1 ) - SELECT pool_address, pool_name, pool_type, - REGEXP_REPLACE(pool_symbol,'[^a-zA-Z0-9/-]+') AS pool_symbol, + REGEXP_REPLACE( + pool_symbol, + '[^a-zA-Z0-9/-]+' + ) AS pool_symbol, pool_decimals, - REGEXP_REPLACE(token0_symbol,'[^a-zA-Z0-9]+') AS token0_symbol, - REGEXP_REPLACE(token1_symbol,'[^a-zA-Z0-9]+') AS token1_symbol, - c.token0_address, - c.token1_address, + REGEXP_REPLACE( + token0_symbol, + '[^a-zA-Z0-9]+' + ) AS token0_symbol, + REGEXP_REPLACE( + token1_symbol, + '[^a-zA-Z0-9]+' + ) AS token1_symbol, + C.token0_address, + C.token1_address, token0_decimals, token1_decimals, created_timestamp, block_number AS created_block, created_hash, - a._inserted_timestamp -FROM pools a -LEFT JOIN pool_creation c USING(pool_address) -LEFT JOIN token0 ON c.token0_address = token0.token0_address -LEFT JOIN token1 ON c.token1_address = token1.token1_address + A._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['pool_address'] + ) }} AS pools_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + pools A + LEFT JOIN pool_creation C USING(pool_address) + LEFT JOIN token0 + ON C.token0_address = token0.token0_address + LEFT JOIN token1 + ON C.token1_address = token1.token1_address diff --git a/models/silver/protocols/velodrome/silver__velodrome_staking_actions.sql b/models/silver/protocols/velodrome/silver__velodrome_staking_actions.sql index 58092235..ca704f62 100644 --- a/models/silver/protocols/velodrome/silver__velodrome_staking_actions.sql +++ b/models/silver/protocols/velodrome/silver__velodrome_staking_actions.sql @@ -120,7 +120,13 @@ SELECT A.gauge_address AS gauge_address, b.pool_address AS pool_address, _inserted_timestamp, - _log_id + _log_id, + {{ dbt_utils.generate_surrogate_key( + ['a.tx_hash', 'a.event_index'] + ) }} AS locks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM staking_actions A LEFT JOIN token_transfer b diff --git a/models/silver/protocols/velodrome/silver__velodrome_swaps.sql b/models/silver/protocols/velodrome/silver__velodrome_swaps.sql index 0151409a..12ebe178 100644 --- a/models/silver/protocols/velodrome/silver__velodrome_swaps.sql +++ b/models/silver/protocols/velodrome/silver__velodrome_swaps.sql @@ -62,9 +62,9 @@ WITH base AS ( _inserted_timestamp, event_index, CASE - WHEN topics[0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' THEN 'Transfer' - WHEN topics[0] :: STRING = '0x112c256902bf554b6ed882d2936687aaeb4225e8cd5b51303c90ca6cf43a8602' THEN 'Fees' - WHEN topics[0] :: STRING = '0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822' THEN 'Swap' + WHEN topics [0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' THEN 'Transfer' + WHEN topics [0] :: STRING = '0x112c256902bf554b6ed882d2936687aaeb4225e8cd5b51303c90ca6cf43a8602' THEN 'Fees' + WHEN topics [0] :: STRING = '0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822' THEN 'Swap' END AS event_name, 'velodrome' AS platform, topics [0] :: STRING AS function_type @@ -186,11 +186,16 @@ SELECT fees_adj, 0 ) AS lp_fee_unadj, - fee_currency + fee_currency, + {{ dbt_utils.generate_surrogate_key( + ['b.tx_hash', 'event_index'] + ) }} AS swaps_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM swaps b LEFT JOIN lp_fees l ON b.contract_address = l.contract_address AND b.tx_hash = l.tx_hash AND b.agg_id = l.agg_id - \ No newline at end of file diff --git a/models/silver/protocols/velodrome/silver__velodrome_votes.sql b/models/silver/protocols/velodrome/silver__velodrome_votes.sql index eb7706ad..617273be 100644 --- a/models/silver/protocols/velodrome/silver__velodrome_votes.sql +++ b/models/silver/protocols/velodrome/silver__velodrome_votes.sql @@ -85,7 +85,8 @@ FINAL AS ( COALESCE( g1.pool_address, g0.pool_address - ) AS pool_address + ) AS pool_address, + votes_base.event_index AS event_index FROM votes_base LEFT JOIN gauges g1 @@ -105,6 +106,7 @@ SELECT block_number, block_timestamp, tx_hash, + event_index, origin_function_signature, origin_from_address, origin_to_address, @@ -115,7 +117,13 @@ SELECT vote_amount, vote_action, _log_id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS votes_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL WHERE