From 010b62b672f20dfdb20c8b236eb1d4051e80b972 Mon Sep 17 00:00:00 2001 From: drethereum <71602799+drethereum@users.noreply.github.com> Date: Wed, 1 May 2024 12:44:49 -0600 Subject: [PATCH] AN-4705/prices-v2 (#269) * prices v2 changes * transfers * bronze ref * source column * remove id * yml * revert id * modified and id column * native and provider tables * lower * new names * overview docs * docs * id names * deprecation notice * yml tests * docs * ez refs * alias * docs * update for new names and docs * deprecation note * addr desc --- .../bronze__asset_metadata_all_providers.sql | 0 .../bronze__asset_metadata_priority.sql | 0 .../bronze__hourly_prices_all_providers.sql | 0 .../bronze__hourly_prices_priority.sql | 0 .../bronze__hourly_prices_priority_eth.sql | 0 ...bronze__complete_native_asset_metadata.sql | 26 ++++ .../prices/bronze__complete_native_prices.sql | 29 ++++ ...onze__complete_provider_asset_metadata.sql | 29 ++++ .../bronze__complete_provider_prices.sql | 24 +++ .../bronze__complete_token_asset_metadata.sql | 28 ++++ .../prices/bronze__complete_token_prices.sql | 31 ++++ .../doc_descriptions/general/__overview__.md | 8 +- models/doc_descriptions/prices/prices.md | 143 ++++++++++++++++++ .../gold/price/price__dim_asset_metadata.sql | 36 +++-- .../gold/price/price__dim_asset_metadata.yml | 27 ++-- .../gold/price/price__ez_asset_metadata.sql | 41 +++-- .../gold/price/price__ez_asset_metadata.yml | 27 ++-- .../price/price__ez_hourly_token_prices.yml | 27 +--- models/gold/price/price__ez_prices_hourly.sql | 39 +++++ models/gold/price/price__ez_prices_hourly.yml | 30 ++++ .../price/price__fact_hourly_token_prices.yml | 20 +-- .../price/price__fact_prices_ohlc_hourly.sql | 19 +++ .../price/price__fact_prices_ohlc_hourly.yml | 24 +++ .../velodrome__ez_claimed_rewards.sql | 2 +- .../velodrome/velodrome__ez_lp_actions.sql | 4 +- .../velodrome/velodrome__ez_swaps.sql | 6 +- .../velodrome/velodrome__ez_velo_locks.sql | 2 +- .../stats/stats__ez_core_metrics_hourly.sql | 5 +- .../silver/core/silver__native_transfers.sql | 4 +- models/silver/core/silver__transfers.sql | 8 +- ...ilver_bridge__complete_bridge_activity.sql | 2 +- .../dex/silver_dex__complete_dex_swaps.sql | 2 +- .../silver__complete_lending_borrows.sql | 4 +- .../silver__complete_lending_deposits.sql | 4 +- .../silver__complete_lending_flashloans.sql | 4 +- .../silver__complete_lending_liquidations.sql | 2 +- .../silver__complete_lending_repayments.sql | 4 +- .../silver__complete_lending_withdraws.sql | 4 +- .../nft/sales/silver__complete_nft_sales.sql | 2 +- .../silver__asset_metadata_all_providers.sql | 0 .../silver__asset_metadata_all_providers.yml | 0 .../silver__asset_metadata_priority.sql | 0 .../silver__asset_metadata_priority.yml | 0 .../silver__hourly_prices_all_providers.sql | 0 .../silver__hourly_prices_all_providers.yml | 0 .../silver__hourly_prices_priority.sql | 0 .../silver__hourly_prices_priority.yml | 0 .../silver__hourly_prices_priority_eth.sql | 0 .../silver__hourly_prices_priority_eth.yml | 0 ...silver__complete_native_asset_metadata.sql | 37 +++++ ...silver__complete_native_asset_metadata.yml | 24 +++ .../prices/silver__complete_native_prices.sql | 40 +++++ .../prices/silver__complete_native_prices.yml | 37 +++++ ...lver__complete_provider_asset_metadata.sql | 37 +++++ ...lver__complete_provider_asset_metadata.yml | 26 ++++ .../silver__complete_provider_prices.sql | 45 ++++++ .../silver__complete_provider_prices.yml | 25 +++ .../silver__complete_token_asset_metadata.sql | 42 +++++ .../silver__complete_token_asset_metadata.yml | 28 ++++ .../prices/silver__complete_token_prices.sql | 46 ++++++ .../prices/silver__complete_token_prices.yml | 41 +++++ models/sources.yml | 6 + 62 files changed, 963 insertions(+), 138 deletions(-) rename models/bronze/prices/{ => _legacy}/bronze__asset_metadata_all_providers.sql (100%) rename models/bronze/prices/{ => _legacy}/bronze__asset_metadata_priority.sql (100%) rename models/bronze/prices/{ => _legacy}/bronze__hourly_prices_all_providers.sql (100%) rename models/bronze/prices/{ => _legacy}/bronze__hourly_prices_priority.sql (100%) rename models/bronze/prices/{ => _legacy}/bronze__hourly_prices_priority_eth.sql (100%) create mode 100644 models/bronze/prices/bronze__complete_native_asset_metadata.sql create mode 100644 models/bronze/prices/bronze__complete_native_prices.sql create mode 100644 models/bronze/prices/bronze__complete_provider_asset_metadata.sql create mode 100644 models/bronze/prices/bronze__complete_provider_prices.sql create mode 100644 models/bronze/prices/bronze__complete_token_asset_metadata.sql create mode 100644 models/bronze/prices/bronze__complete_token_prices.sql create mode 100644 models/doc_descriptions/prices/prices.md create mode 100644 models/gold/price/price__ez_prices_hourly.sql create mode 100644 models/gold/price/price__ez_prices_hourly.yml create mode 100644 models/gold/price/price__fact_prices_ohlc_hourly.sql create mode 100644 models/gold/price/price__fact_prices_ohlc_hourly.yml rename models/silver/prices/{ => _legacy}/silver__asset_metadata_all_providers.sql (100%) rename models/silver/prices/{ => _legacy}/silver__asset_metadata_all_providers.yml (100%) rename models/silver/prices/{ => _legacy}/silver__asset_metadata_priority.sql (100%) rename models/silver/prices/{ => _legacy}/silver__asset_metadata_priority.yml (100%) rename models/silver/prices/{ => _legacy}/silver__hourly_prices_all_providers.sql (100%) rename models/silver/prices/{ => _legacy}/silver__hourly_prices_all_providers.yml (100%) rename models/silver/prices/{ => _legacy}/silver__hourly_prices_priority.sql (100%) rename models/silver/prices/{ => _legacy}/silver__hourly_prices_priority.yml (100%) rename models/silver/prices/{ => _legacy}/silver__hourly_prices_priority_eth.sql (100%) rename models/silver/prices/{ => _legacy}/silver__hourly_prices_priority_eth.yml (100%) create mode 100644 models/silver/prices/silver__complete_native_asset_metadata.sql create mode 100644 models/silver/prices/silver__complete_native_asset_metadata.yml create mode 100644 models/silver/prices/silver__complete_native_prices.sql create mode 100644 models/silver/prices/silver__complete_native_prices.yml create mode 100644 models/silver/prices/silver__complete_provider_asset_metadata.sql create mode 100644 models/silver/prices/silver__complete_provider_asset_metadata.yml create mode 100644 models/silver/prices/silver__complete_provider_prices.sql create mode 100644 models/silver/prices/silver__complete_provider_prices.yml create mode 100644 models/silver/prices/silver__complete_token_asset_metadata.sql create mode 100644 models/silver/prices/silver__complete_token_asset_metadata.yml create mode 100644 models/silver/prices/silver__complete_token_prices.sql create mode 100644 models/silver/prices/silver__complete_token_prices.yml diff --git a/models/bronze/prices/bronze__asset_metadata_all_providers.sql b/models/bronze/prices/_legacy/bronze__asset_metadata_all_providers.sql similarity index 100% rename from models/bronze/prices/bronze__asset_metadata_all_providers.sql rename to models/bronze/prices/_legacy/bronze__asset_metadata_all_providers.sql diff --git a/models/bronze/prices/bronze__asset_metadata_priority.sql b/models/bronze/prices/_legacy/bronze__asset_metadata_priority.sql similarity index 100% rename from models/bronze/prices/bronze__asset_metadata_priority.sql rename to models/bronze/prices/_legacy/bronze__asset_metadata_priority.sql diff --git a/models/bronze/prices/bronze__hourly_prices_all_providers.sql b/models/bronze/prices/_legacy/bronze__hourly_prices_all_providers.sql similarity index 100% rename from models/bronze/prices/bronze__hourly_prices_all_providers.sql rename to models/bronze/prices/_legacy/bronze__hourly_prices_all_providers.sql diff --git a/models/bronze/prices/bronze__hourly_prices_priority.sql b/models/bronze/prices/_legacy/bronze__hourly_prices_priority.sql similarity index 100% rename from models/bronze/prices/bronze__hourly_prices_priority.sql rename to models/bronze/prices/_legacy/bronze__hourly_prices_priority.sql diff --git a/models/bronze/prices/bronze__hourly_prices_priority_eth.sql b/models/bronze/prices/_legacy/bronze__hourly_prices_priority_eth.sql similarity index 100% rename from models/bronze/prices/bronze__hourly_prices_priority_eth.sql rename to models/bronze/prices/_legacy/bronze__hourly_prices_priority_eth.sql diff --git a/models/bronze/prices/bronze__complete_native_asset_metadata.sql b/models/bronze/prices/bronze__complete_native_asset_metadata.sql new file mode 100644 index 00000000..39001f39 --- /dev/null +++ b/models/bronze/prices/bronze__complete_native_asset_metadata.sql @@ -0,0 +1,26 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_native_asset_metadata' + ) }} +WHERE + blockchain = 'ethereum' + AND symbol = 'ETH' diff --git a/models/bronze/prices/bronze__complete_native_prices.sql b/models/bronze/prices/bronze__complete_native_prices.sql new file mode 100644 index 00000000..84539f0f --- /dev/null +++ b/models/bronze/prices/bronze__complete_native_prices.sql @@ -0,0 +1,29 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_native_prices' + ) }} +WHERE + blockchain = 'ethereum' + AND symbol = 'ETH' diff --git a/models/bronze/prices/bronze__complete_provider_asset_metadata.sql b/models/bronze/prices/bronze__complete_provider_asset_metadata.sql new file mode 100644 index 00000000..27b9df19 --- /dev/null +++ b/models/bronze/prices/bronze__complete_provider_asset_metadata.sql @@ -0,0 +1,29 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + token_address, + NAME, + symbol, + platform, + platform_id, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_provider_asset_metadata' + ) }} +WHERE + platform IN ( + 'Optimism', + 'optimistic-ethereum' + ) +-- platforms specific to Optimism \ No newline at end of file diff --git a/models/bronze/prices/bronze__complete_provider_prices.sql b/models/bronze/prices/bronze__complete_provider_prices.sql new file mode 100644 index 00000000..7d856361 --- /dev/null +++ b/models/bronze/prices/bronze__complete_provider_prices.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + recorded_hour, + OPEN, + high, + low, + CLOSE, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_provider_prices' + ) }} +-- prices for all ids \ No newline at end of file diff --git a/models/bronze/prices/bronze__complete_token_asset_metadata.sql b/models/bronze/prices/bronze__complete_token_asset_metadata.sql new file mode 100644 index 00000000..89e20092 --- /dev/null +++ b/models/bronze/prices/bronze__complete_token_asset_metadata.sql @@ -0,0 +1,28 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + blockchain_name, + blockchain_id, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_token_asset_metadata' + ) }} +WHERE + blockchain = 'optimism' \ No newline at end of file diff --git a/models/bronze/prices/bronze__complete_token_prices.sql b/models/bronze/prices/bronze__complete_token_prices.sql new file mode 100644 index 00000000..468cd166 --- /dev/null +++ b/models/bronze/prices/bronze__complete_token_prices.sql @@ -0,0 +1,31 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + HOUR, + token_address, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + blockchain_name, + blockchain_id, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_token_prices' + ) }} +WHERE + blockchain = 'optimism' diff --git a/models/doc_descriptions/general/__overview__.md b/models/doc_descriptions/general/__overview__.md index bae11344..14bea7a3 100644 --- a/models/doc_descriptions/general/__overview__.md +++ b/models/doc_descriptions/general/__overview__.md @@ -38,13 +38,11 @@ There is more information on how to use dbt docs in the last section of this doc - [ez_token_transfers](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.core__ez_token_transfers) - [ez_decoded_event_logs](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.core__ez_decoded_event_logs) - ### Price Tables (optimism.price) -- [fact_hourly_token_prices](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.price__fact_hourly_token_prices) -- [ez_hourly_token_prices](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.price__ez_hourly_token_prices) - [dim_asset_metadata](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.price__dim_asset_metadata) +- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.price__fact_prices_ohlc_hourly) - [ez_asset_metadata](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.price__ez_asset_metadata) - +- [ez_prices_hourly](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.price__ez_prices_hourly) ### DeFi Tables (optimism.defi) - [ez_bridge_activity](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.defi__ez_bridge_activity) @@ -57,7 +55,7 @@ There is more information on how to use dbt docs in the last section of this doc - [ez_lending_repayments](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.defi__ez_lending_repayments) - [ez_lending_withdraws](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.defi__ez_lending_withdraws) -### Stats Tables (ethereum.stats) +### Stats Tables (optimism.stats) - [ez_core_metrics_hourly](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.stats__ez_core_metrics_hourly) **Velodrome** diff --git a/models/doc_descriptions/prices/prices.md b/models/doc_descriptions/prices/prices.md new file mode 100644 index 00000000..60d4e969 --- /dev/null +++ b/models/doc_descriptions/prices/prices.md @@ -0,0 +1,143 @@ +{% docs prices_dim_asset_metadata_table_doc %} + +A comprehensive dimensional table holding asset metadata and other relevant details pertaining to each id, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this dimensional model. If you are interested in using a curated data set instead, please utilize ez_asset_metadata. + +{% enddocs %} + +{% docs prices_ez_asset_metadata_table_doc %} + +A convenience table holding prioritized asset metadata and other relevant details pertaining to each token_address and native asset. This data set is highly curated and contains metadata for one unique asset per blockchain. + +{% enddocs %} + +{% docs prices_fact_prices_ohlc_hourly_table_doc %} + +A comprehensive fact table holding id and provider specific open, high, low, close hourly prices, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this fact based model. If you are interested in using a curated data set instead, please utilize ez_prices_hourly. + +{% enddocs %} + +{% docs prices_ez_prices_hourly_table_doc %} + +A convenience table for determining token prices by address and blockchain, and native asset prices by symbol and blockchain. This data set is highly curated and contains metadata for one price per hour per unique asset and blockchain. + +{% enddocs %} + +{% docs prices_provider %} + +The provider or source of the data. + +{% enddocs %} + +{% docs prices_asset_id %} + +The unique identifier representing the asset. + +{% enddocs %} + +{% docs prices_name %} + +The name of asset. + +{% enddocs %} + +{% docs prices_symbol %} + +The symbol of asset. + +{% enddocs %} + +{% docs prices_token_address %} + +The specific address representing the asset on a specific platform. This will be NULL if referring to a native asset. + +{% enddocs %} + +{% docs prices_token_address_evm %} + +The specific address representing the asset on a specific platform. This will be NULL if referring to a native asset. The case (upper / lower) may or may not be specified within the `dim_asset_metadata` table, as this column is raw and not transformed, coming directly from the provider APIs. However, in the `ez_` views, it will be lowercase by default for all EVMs. + +{% enddocs %} + +{% docs prices_blockchain %} + +The Blockchain, Network, or Platform for this asset. + +{% enddocs %} + +{% docs prices_blockchain_id %} + +The unique identifier of the Blockchain, Network, or Platform for this asset. + +{% enddocs %} + +{% docs prices_decimals %} + +The number of decimals for the asset. May be NULL. + +{% enddocs %} + +{% docs prices_is_native %} + +A flag indicating assets native to the respective blockchain. + +{% enddocs %} + +{% docs prices_is_deprecated %} + +A flag indicating if the asset is deprecated or no longer supported by the provider. + +{% enddocs %} + +{% docs prices_id_deprecation %} + +Deprecating soon! Please use the `asset_id` column instead. + +{% enddocs %} + +{% docs prices_decimals_deprecation %} + +Deprecating soon! Please use the decimals column in `ez_asset_metadata` or join in `dim_contracts` instead. + +{% enddocs %} + +{% docs prices_hour %} + +Hour that the price was recorded at. + +{% enddocs %} + +{% docs prices_price %} + +Closing price of the recorded hour in USD. + +{% enddocs %} + +{% docs prices_is_imputed %} + +A flag indicating if the price was imputed, or derived, from the last arriving record. This is generally used for tokens with low-liquidity or inconsistent reporting. + +{% enddocs %} + +{% docs prices_open %} + +Opening price of the recorded hour in USD. + +{% enddocs %} + +{% docs prices_high %} + +Highest price of the recorded hour in USD + +{% enddocs %} + +{% docs prices_low %} + +Lowest price of the recorded hour in USD + +{% enddocs %} + +{% docs prices_close %} + +Closing price of the recorded hour in USD + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/price/price__dim_asset_metadata.sql b/models/gold/price/price__dim_asset_metadata.sql index 794c6154..ba69cf68 100644 --- a/models/gold/price/price__dim_asset_metadata.sql +++ b/models/gold/price/price__dim_asset_metadata.sql @@ -6,24 +6,22 @@ SELECT token_address, - id, - symbol, - NAME, - decimals, + asset_id AS id, -- id column pending deprecation + asset_id, + A.symbol, + A.name, + C.decimals, -- decimals column pending deprecation + platform AS blockchain, + platform_id AS blockchain_id, provider, - COALESCE ( - asset_metadata_all_providers_id, - {{ dbt_utils.generate_surrogate_key( - ['token_address','symbol','id','provider'] - ) }} - ) AS dim_asset_metadata_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp + A.inserted_timestamp, + A.modified_timestamp, + A.complete_provider_asset_metadata_id AS dim_asset_metadata_id FROM - {{ ref('silver__asset_metadata_all_providers') }} \ No newline at end of file + {{ ref('silver__complete_provider_asset_metadata') }} A + LEFT JOIN {{ ref('core__dim_contracts') }} C --remove this join alongside decimal column deprecation + ON LOWER( + C.address + ) = LOWER( + A.token_address + ) \ No newline at end of file diff --git a/models/gold/price/price__dim_asset_metadata.yml b/models/gold/price/price__dim_asset_metadata.yml index fe7022b2..7352d961 100644 --- a/models/gold/price/price__dim_asset_metadata.yml +++ b/models/gold/price/price__dim_asset_metadata.yml @@ -1,28 +1,27 @@ version: 2 models: - name: price__dim_asset_metadata - description: A comprehensive dimensional table holding all provider asset metadata and other relevant details pertaining to each token_address. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - PROVIDER - - ID - - TOKEN_ADDRESS - - SYMBOL + description: Deprecating soon! Note, only the ID and DECIMALS columns are being removed. The dim_asset_metadata table itself is not being deprecated. '{{ doc("prices_dim_asset_metadata_table_doc") }}' columns: - name: PROVIDER - description: The provider or source of the data. + description: '{{ doc("prices_provider")}}' - name: ID - description: The unique identifier representing the asset. + description: '{{ doc("prices_id_deprecation") }}' + - name: ASSET_ID + description: '{{ doc("prices_asset_id") }}' - name: NAME - description: The name of asset. + description: '{{ doc("prices_name") }}' - name: SYMBOL - description: The symbol of asset. + description: '{{ doc("prices_symbol") }}' - name: TOKEN_ADDRESS - description: The specific address representing the asset in a specific platform. + description: '{{ doc("prices_token_address_evm") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' + - name: BLOCKCHAIN_ID + description: '{{ doc("prices_blockchain_id") }}' - name: DECIMALS - description: The number of decimal places the token needs adjusted where token values exist. + description: '{{ doc("prices_decimals_deprecation") }}' - name: DIM_ASSET_METADATA_ID description: '{{ doc("pk") }}' - name: INSERTED_TIMESTAMP diff --git a/models/gold/price/price__ez_asset_metadata.sql b/models/gold/price/price__ez_asset_metadata.sql index b145c352..a77af413 100644 --- a/models/gold/price/price__ez_asset_metadata.sql +++ b/models/gold/price/price__ez_asset_metadata.sql @@ -6,23 +6,32 @@ SELECT token_address, - id, + asset_id AS id, -- id column pending deprecation + asset_id, symbol, NAME, decimals, - COALESCE ( - asset_metadata_priority_id, - {{ dbt_utils.generate_surrogate_key( - ['token_address'] - ) }} - ) AS ez_asset_metadata_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp + blockchain, + FALSE AS is_native, + is_deprecated, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id AS ez_asset_metadata_id FROM - {{ ref('silver__asset_metadata_priority') }} \ No newline at end of file + {{ ref('silver__complete_token_asset_metadata') }} +UNION ALL +SELECT + NULL AS token_address, + asset_id AS id, -- id column pending deprecation + asset_id, + symbol, + NAME, + decimals, + blockchain, + TRUE AS is_native, + is_deprecated, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id AS ez_asset_metadata_id +FROM + {{ ref('silver__complete_native_asset_metadata') }} diff --git a/models/gold/price/price__ez_asset_metadata.yml b/models/gold/price/price__ez_asset_metadata.yml index a7a2c3ef..a2acc804 100644 --- a/models/gold/price/price__ez_asset_metadata.yml +++ b/models/gold/price/price__ez_asset_metadata.yml @@ -1,26 +1,31 @@ version: 2 models: - name: price__ez_asset_metadata - description: A convenience table holding prioritized asset metadata and other relevant details pertaining to each token_address. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - TOKEN_ADDRESS + description: Deprecating soon! Note, only the ID column is changing. The ez_asset_metadata table itself is not being deprecated. '{{ doc("prices_ez_asset_metadata_table_doc") }}' columns: - name: ID - description: The unique identifier representing the asset. + description: '{{ doc("prices_id_deprecation")}}' + - name: ASSET_ID + description: '{{ doc("prices_asset_id") }}' - name: NAME - description: The name of asset. + description: '{{ doc("prices_name") }}' - name: SYMBOL - description: The symbol of asset. + description: '{{ doc("prices_symbol") }}' - name: TOKEN_ADDRESS - description: The specific address representing the asset in a specific platform. + description: '{{ doc("prices_token_address_evm") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' - name: DECIMALS - description: The number of decimal places the token needs adjusted where token values exist. + description: '{{ doc("prices_decimals") }}' + - name: IS_NATIVE + description: '{{ doc("prices_is_native") }}' + - name: IS_DEPRECATED + description: '{{ doc("prices_is_deprecated") }}' - name: EZ_ASSET_METADATA_ID description: '{{ doc("pk") }}' - name: INSERTED_TIMESTAMP description: '{{ doc("inserted_timestamp") }}' - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' \ No newline at end of file + description: '{{ doc("modified_timestamp") }}' + \ No newline at end of file diff --git a/models/gold/price/price__ez_hourly_token_prices.yml b/models/gold/price/price__ez_hourly_token_prices.yml index e705476e..4ea34c47 100644 --- a/models/gold/price/price__ez_hourly_token_prices.yml +++ b/models/gold/price/price__ez_hourly_token_prices.yml @@ -1,29 +1,4 @@ version: 2 models: - name: price__ez_hourly_token_prices - description: A convenience table for determining token prices by address. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - HOUR - - TOKEN_ADDRESS - - columns: - - name: HOUR - description: Hour that the price was recorded at - - name: TOKEN_ADDRESS - description: Address of the token - - name: SYMBOL - description: Symbol of the token - - name: DECIMALS - description: The number of decimals for token contract - - name: PRICE - description: Closing price of the recorded hour in USD - - name: IS_IMPUTED - description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) - - name: EZ_HOURLY_TOKEN_PRICES_ID - description: '{{ doc("pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' \ No newline at end of file + description: Deprecating soon! Please use price.ez_prices_hourly or price.fact_prices_ohlc_hourly instead. \ No newline at end of file diff --git a/models/gold/price/price__ez_prices_hourly.sql b/models/gold/price/price__ez_prices_hourly.sql new file mode 100644 index 00000000..dfda724e --- /dev/null +++ b/models/gold/price/price__ez_prices_hourly.sql @@ -0,0 +1,39 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + HOUR, + token_address, + symbol, + NAME, + decimals, + price, + blockchain, + FALSE AS is_native, + is_imputed, + is_deprecated, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id AS ez_prices_hourly_id +FROM + {{ ref('silver__complete_token_prices') }} +UNION ALL +SELECT + HOUR, + NULL AS token_address, + symbol, + NAME, + decimals, + price, + blockchain, + TRUE AS is_native, + is_imputed, + is_deprecated, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id AS ez_prices_hourly_id +FROM + {{ ref('silver__complete_native_prices') }} diff --git a/models/gold/price/price__ez_prices_hourly.yml b/models/gold/price/price__ez_prices_hourly.yml new file mode 100644 index 00000000..96cf4e5e --- /dev/null +++ b/models/gold/price/price__ez_prices_hourly.yml @@ -0,0 +1,30 @@ +version: 2 +models: + - name: price__ez_prices_hourly + description: '{{ doc("prices_ez_prices_hourly_table_doc") }}' + + columns: + - name: HOUR + description: '{{ doc("prices_hour")}}' + - name: TOKEN_ADDRESS + description: '{{ doc("prices_token_address_evm") }}' + - name: SYMBOL + description: '{{ doc("prices_symbol") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' + - name: DECIMALS + description: '{{ doc("prices_decimals") }}' + - name: PRICE + description: '{{ doc("prices_price") }}' + - name: IS_NATIVE + description: '{{ doc("prices_is_native") }}' + - name: IS_IMPUTED + description: '{{ doc("prices_is_imputed") }}' + - name: IS_DEPRECATED + description: '{{ doc("prices_is_deprecated") }}' + - name: EZ_PRICES_HOURLY_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/price/price__fact_hourly_token_prices.yml b/models/gold/price/price__fact_hourly_token_prices.yml index 3b12c101..ab8f694d 100644 --- a/models/gold/price/price__fact_hourly_token_prices.yml +++ b/models/gold/price/price__fact_hourly_token_prices.yml @@ -1,22 +1,4 @@ version: 2 models: - name: price__fact_hourly_token_prices - description: A comprehensive fact table holding provider specific hourly token prices. - - columns: - - name: HOUR - description: Hour that the price was recorded at - - name: TOKEN_ADDRESS - description: Address of the token - - name: PROVIDER - description: Source of the token price. - - name: PRICE - description: Closing price of the recorded hour in USD - - name: IS_IMPUTED - description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) - - name: FACT_HOURLY_TOKEN_PRICES_ID - description: '{{ doc("pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' \ No newline at end of file + description: Deprecating soon! Please use price.ez_prices_hourly or price.fact_prices_ohlc_hourly instead. \ No newline at end of file diff --git a/models/gold/price/price__fact_prices_ohlc_hourly.sql b/models/gold/price/price__fact_prices_ohlc_hourly.sql new file mode 100644 index 00000000..2be7d759 --- /dev/null +++ b/models/gold/price/price__fact_prices_ohlc_hourly.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + asset_id, + recorded_hour AS HOUR, + OPEN, + high, + low, + CLOSE, + provider, + inserted_timestamp, + modified_timestamp, + complete_provider_prices_id AS fact_prices_ohlc_hourly_id +FROM + {{ ref('silver__complete_provider_prices') }} diff --git a/models/gold/price/price__fact_prices_ohlc_hourly.yml b/models/gold/price/price__fact_prices_ohlc_hourly.yml new file mode 100644 index 00000000..29ff0457 --- /dev/null +++ b/models/gold/price/price__fact_prices_ohlc_hourly.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: price__fact_prices_ohlc_hourly + description: '{{ doc("prices_fact_prices_ohlc_hourly_table_doc") }}' + + columns: + - name: HOUR + description: '{{ doc("prices_hour")}}' + - name: ASSET_ID + description: '{{ doc("prices_asset_id") }}' + - name: OPEN + description: '{{ doc("prices_open") }}' + - name: HIGH + description: '{{ doc("prices_high") }}' + - name: LOW + description: '{{ doc("prices_low") }}' + - name: CLOSE + description: '{{ doc("prices_close") }}' + - name: FACT_PRICES_OHLC_HOURLY_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.sql b/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.sql index 24e00ccc..2cb1a239 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_claimed_rewards.sql @@ -126,7 +126,7 @@ FROM base LEFT JOIN tokens t ON t.token_address = base.token_address - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} prices ON HOUR = DATE_TRUNC( 'hour', diff --git a/models/gold/protocols/velodrome/velodrome__ez_lp_actions.sql b/models/gold/protocols/velodrome/velodrome__ez_lp_actions.sql index dddf4a19..8722d99d 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_lp_actions.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_lp_actions.sql @@ -112,14 +112,14 @@ FROM ) = LOWER( pools.pool_address ) - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p0 ON p0.hour = DATE_TRUNC( 'hour', block_timestamp ) AND token0_address = p0.token_address - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p1 ON p1.hour = DATE_TRUNC( 'hour', diff --git a/models/gold/protocols/velodrome/velodrome__ez_swaps.sql b/models/gold/protocols/velodrome/velodrome__ez_swaps.sql index b47f03dc..eaeb93b0 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_swaps.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_swaps.sql @@ -225,7 +225,7 @@ FROM ) LEFT JOIN tokens ON tokens.token_address = fee_currency - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p0 ON p0.hour = DATE_TRUNC( 'hour', @@ -235,7 +235,7 @@ FROM WHEN amount0_in_unadj <> 0 THEN token0_address WHEN amount1_in_unadj <> 0 THEN token1_address END - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p1 ON p1.hour = DATE_TRUNC( 'hour', @@ -245,7 +245,7 @@ FROM WHEN amount0_out_unadj <> 0 THEN token0_address WHEN amount1_out_unadj <> 0 THEN token1_address END - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p2 ON p2.hour = DATE_TRUNC( 'hour', diff --git a/models/gold/protocols/velodrome/velodrome__ez_velo_locks.sql b/models/gold/protocols/velodrome/velodrome__ez_velo_locks.sql index 00ebddd4..3df0cb94 100644 --- a/models/gold/protocols/velodrome/velodrome__ez_velo_locks.sql +++ b/models/gold/protocols/velodrome/velodrome__ez_velo_locks.sql @@ -59,7 +59,7 @@ SELECT ) AS modified_timestamp FROM {{ ref('silver__velodrome_locks') }} l - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} prices ON HOUR = DATE_TRUNC( 'hour', diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql index 29f8672a..7a2e3016 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -32,6 +32,7 @@ SELECT FROM {{ ref('silver_stats__core_metrics_hourly') }} s - LEFT JOIN {{ ref('silver__hourly_prices_priority_eth') }} + LEFT JOIN {{ ref('silver__complete_token_prices') }} p - ON s.block_timestamp_hour = p.hour \ No newline at end of file + ON s.block_timestamp_hour = p.hour + AND p.token_address = '0x4200000000000000000000000000000000000006' \ No newline at end of file diff --git a/models/silver/core/silver__native_transfers.sql b/models/silver/core/silver__native_transfers.sql index 4ae5a47f..c91660bb 100644 --- a/models/silver/core/silver__native_transfers.sql +++ b/models/silver/core/silver__native_transfers.sql @@ -98,11 +98,13 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM eth_base A - LEFT JOIN {{ ref('silver__hourly_prices_priority_eth') }} + LEFT JOIN {{ ref('silver__complete_token_prices') }} + p ON DATE_TRUNC( 'hour', A.block_timestamp ) = HOUR + AND p.token_address = '0x4200000000000000000000000000000000000006' JOIN tx_table USING ( tx_hash, block_number diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index f4ee3327..7db266f5 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('silver__hourly_prices_priority') }} +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', @@ -85,7 +85,7 @@ token_transfers AS ( _inserted_timestamp FROM logs t - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p ON t.contract_address = p.token_address AND DATE_TRUNC( @@ -147,7 +147,7 @@ heal_model AS ( FROM {{ this }} t0 - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p ON t0.contract_address = p.token_address AND DATE_TRUNC( @@ -203,7 +203,7 @@ heal_model AS ( SELECT 1 FROM - {{ ref('silver__hourly_prices_priority') }} + {{ ref('silver__complete_token_prices') }} p WHERE p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) diff --git a/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql b/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql index 913f68bb..27f8ee27 100644 --- a/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql +++ b/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql @@ -646,7 +646,7 @@ FINAL AS ( all_protocols b LEFT JOIN {{ ref('silver__contracts') }} C ON b.token_address = C.contract_address - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p ON b.token_address = p.token_address AND DATE_TRUNC( diff --git a/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql b/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql index 213f7c3f..1a686525 100644 --- a/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql +++ b/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql @@ -22,7 +22,7 @@ prices AS ( token_address, price FROM - {{ ref('price__ez_hourly_token_prices') }} + {{ ref('price__ez_prices_hourly') }} WHERE token_address IN ( SELECT diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql index ee50a2a5..543027f9 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql @@ -230,12 +230,12 @@ FINAL AS ( 2 ) AS amount_usd, platform, - blockchain, + b.blockchain, b._LOG_ID, b._INSERTED_TIMESTAMP FROM borrow_union b - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p ON b.token_address = p.token_address AND DATE_TRUNC( diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql index 2a3a6fb8..e09e263a 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql @@ -227,12 +227,12 @@ FINAL AS ( 2 ) AS amount_usd, platform, - blockchain, + A.blockchain, A._LOG_ID, A._INSERTED_TIMESTAMP FROM deposit_union A - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p ON A.token_address = p.token_address AND DATE_TRUNC( diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql index 9f85428e..9780f6b0 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql @@ -122,12 +122,12 @@ FINAL AS ( 2 ) AS premium_amount_usd, platform, - blockchain, + f.blockchain, f._LOG_ID, f._INSERTED_TIMESTAMP FROM flashloan_union f - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p ON f.token_address = p.token_address AND DATE_TRUNC( diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql index dfa28b46..c84add53 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql @@ -243,7 +243,7 @@ prices AS ( SELECT * FROM - {{ ref('price__ez_hourly_token_prices') }} + {{ ref('price__ez_prices_hourly') }} p WHERE token_address IN ( diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql index 295fcebf..9e6a8833 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql @@ -236,12 +236,12 @@ FINAL AS ( 2 ) AS amount_usd, platform, - blockchain, + A.blockchain, A._LOG_ID, A._INSERTED_TIMESTAMP FROM repayments_union A - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p ON A.token_address = p.token_address AND DATE_TRUNC( diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql index 958394b2..212dde9c 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql @@ -243,12 +243,12 @@ FINAL AS ( 2 ) AS amount_usd, platform, - blockchain, + A.blockchain, A._log_id, A._inserted_timestamp FROM withdraws_union A - LEFT JOIN {{ ref('price__ez_hourly_token_prices') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p ON A.token_address = p.token_address AND DATE_TRUNC( diff --git a/models/silver/nft/sales/silver__complete_nft_sales.sql b/models/silver/nft/sales/silver__complete_nft_sales.sql index 2b51cdab..4de7c829 100644 --- a/models/silver/nft/sales/silver__complete_nft_sales.sql +++ b/models/silver/nft/sales/silver__complete_nft_sales.sql @@ -268,7 +268,7 @@ prices_raw AS ( decimals, price AS hourly_prices FROM - {{ ref('price__ez_hourly_token_prices') }} + {{ ref('price__ez_prices_hourly') }} WHERE token_address IN ( SELECT diff --git a/models/silver/prices/silver__asset_metadata_all_providers.sql b/models/silver/prices/_legacy/silver__asset_metadata_all_providers.sql similarity index 100% rename from models/silver/prices/silver__asset_metadata_all_providers.sql rename to models/silver/prices/_legacy/silver__asset_metadata_all_providers.sql diff --git a/models/silver/prices/silver__asset_metadata_all_providers.yml b/models/silver/prices/_legacy/silver__asset_metadata_all_providers.yml similarity index 100% rename from models/silver/prices/silver__asset_metadata_all_providers.yml rename to models/silver/prices/_legacy/silver__asset_metadata_all_providers.yml diff --git a/models/silver/prices/silver__asset_metadata_priority.sql b/models/silver/prices/_legacy/silver__asset_metadata_priority.sql similarity index 100% rename from models/silver/prices/silver__asset_metadata_priority.sql rename to models/silver/prices/_legacy/silver__asset_metadata_priority.sql diff --git a/models/silver/prices/silver__asset_metadata_priority.yml b/models/silver/prices/_legacy/silver__asset_metadata_priority.yml similarity index 100% rename from models/silver/prices/silver__asset_metadata_priority.yml rename to models/silver/prices/_legacy/silver__asset_metadata_priority.yml diff --git a/models/silver/prices/silver__hourly_prices_all_providers.sql b/models/silver/prices/_legacy/silver__hourly_prices_all_providers.sql similarity index 100% rename from models/silver/prices/silver__hourly_prices_all_providers.sql rename to models/silver/prices/_legacy/silver__hourly_prices_all_providers.sql diff --git a/models/silver/prices/silver__hourly_prices_all_providers.yml b/models/silver/prices/_legacy/silver__hourly_prices_all_providers.yml similarity index 100% rename from models/silver/prices/silver__hourly_prices_all_providers.yml rename to models/silver/prices/_legacy/silver__hourly_prices_all_providers.yml diff --git a/models/silver/prices/silver__hourly_prices_priority.sql b/models/silver/prices/_legacy/silver__hourly_prices_priority.sql similarity index 100% rename from models/silver/prices/silver__hourly_prices_priority.sql rename to models/silver/prices/_legacy/silver__hourly_prices_priority.sql diff --git a/models/silver/prices/silver__hourly_prices_priority.yml b/models/silver/prices/_legacy/silver__hourly_prices_priority.yml similarity index 100% rename from models/silver/prices/silver__hourly_prices_priority.yml rename to models/silver/prices/_legacy/silver__hourly_prices_priority.yml diff --git a/models/silver/prices/silver__hourly_prices_priority_eth.sql b/models/silver/prices/_legacy/silver__hourly_prices_priority_eth.sql similarity index 100% rename from models/silver/prices/silver__hourly_prices_priority_eth.sql rename to models/silver/prices/_legacy/silver__hourly_prices_priority_eth.sql diff --git a/models/silver/prices/silver__hourly_prices_priority_eth.yml b/models/silver/prices/_legacy/silver__hourly_prices_priority_eth.yml similarity index 100% rename from models/silver/prices/silver__hourly_prices_priority_eth.yml rename to models/silver/prices/_legacy/silver__hourly_prices_priority_eth.yml diff --git a/models/silver/prices/silver__complete_native_asset_metadata.sql b/models/silver/prices/silver__complete_native_asset_metadata.sql new file mode 100644 index 00000000..f06d5931 --- /dev/null +++ b/models/silver/prices/silver__complete_native_asset_metadata.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_native_asset_metadata_id', + tags = ['non_realtime'] +) }} + +SELECT + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_native_asset_metadata' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_native_asset_metadata.yml b/models/silver/prices/silver__complete_native_asset_metadata.yml new file mode 100644 index 00000000..1c1fdf1c --- /dev/null +++ b/models/silver/prices/silver__complete_native_asset_metadata.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: silver__complete_native_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - SYMBOL + + columns: + - name: PROVIDER + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_NATIVE_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_native_prices.sql b/models/silver/prices/silver__complete_native_prices.sql new file mode 100644 index 00000000..375cbed3 --- /dev/null +++ b/models/silver/prices/silver__complete_native_prices.sql @@ -0,0 +1,40 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_native_prices_id', + tags = ['non_realtime'] +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_native_prices' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_native_prices.yml b/models/silver/prices/silver__complete_native_prices.yml new file mode 100644 index 00000000..bb724335 --- /dev/null +++ b/models/silver/prices/silver__complete_native_prices.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: silver__complete_native_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - SYMBOL + + columns: + - name: HOUR + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: IS_IMPUTED + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_NATIVE_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_asset_metadata.sql b/models/silver/prices/silver__complete_provider_asset_metadata.sql new file mode 100644 index 00000000..a22da447 --- /dev/null +++ b/models/silver/prices/silver__complete_provider_asset_metadata.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_provider_asset_metadata_id', + tags = ['non_realtime'] +) }} + +SELECT + asset_id, + token_address, + NAME, + symbol, + platform, + platform_id, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_provider_asset_metadata' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_asset_metadata.yml b/models/silver/prices/silver__complete_provider_asset_metadata.yml new file mode 100644 index 00000000..8fa56fd8 --- /dev/null +++ b/models/silver/prices/silver__complete_provider_asset_metadata.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: silver__complete_provider_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ASSET_ID + - TOKEN_ADDRESS + - NAME + - SYMBOL + - PLATFORM + - PLATFORM_ID + - PROVIDER + columns: + - name: PROVIDER + tests: + - not_null + - name: ASSET_ID + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_PROVIDER_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_prices.sql b/models/silver/prices/silver__complete_provider_prices.sql new file mode 100644 index 00000000..c6aa062f --- /dev/null +++ b/models/silver/prices/silver__complete_provider_prices.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_provider_prices_id', + tags = ['non_realtime'] +) }} + +SELECT + p.asset_id, + recorded_hour, + OPEN, + high, + low, + CLOSE, + p.provider, + p.source, + p._inserted_timestamp, + p.inserted_timestamp, + p.modified_timestamp, + p.complete_provider_prices_id, + p._invocation_id +FROM + {{ ref( + 'bronze__complete_provider_prices' + ) }} + p + INNER JOIN {{ ref('bronze__complete_provider_asset_metadata') }} + m + ON p.asset_id = m.asset_id + +{% if is_incremental() %} +WHERE + p.modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY p.asset_id, recorded_hour, p.provider +ORDER BY + p.modified_timestamp DESC)) = 1 diff --git a/models/silver/prices/silver__complete_provider_prices.yml b/models/silver/prices/silver__complete_provider_prices.yml new file mode 100644 index 00000000..8db23c7e --- /dev/null +++ b/models/silver/prices/silver__complete_provider_prices.yml @@ -0,0 +1,25 @@ +version: 2 +models: + - name: silver__complete_provider_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ASSET_ID + - RECORDED_HOUR + - PROVIDER + columns: + - name: PROVIDER + tests: + - not_null + - name: ASSET_ID + tests: + - not_null + - name: RECORDED_HOUR + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_PROVIDER_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_token_asset_metadata.sql b/models/silver/prices/silver__complete_token_asset_metadata.sql new file mode 100644 index 00000000..2dfc8a35 --- /dev/null +++ b/models/silver/prices/silver__complete_token_asset_metadata.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_token_asset_metadata_id', + tags = ['non_realtime'] +) }} + +SELECT + LOWER( + A.token_address + ) AS token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + blockchain_name, + blockchain_id, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_token_asset_metadata' + ) }} A + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_token_asset_metadata.yml b/models/silver/prices/silver__complete_token_asset_metadata.yml new file mode 100644 index 00000000..3eea590e --- /dev/null +++ b/models/silver/prices/silver__complete_token_asset_metadata.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: silver__complete_token_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TOKEN_ADDRESS + - BLOCKCHAIN + + columns: + - name: PROVIDER + tests: + - not_null + - name: TOKEN_ADDRESS + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCKCHAIN_ID + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_TOKEN_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_token_prices.sql b/models/silver/prices/silver__complete_token_prices.sql new file mode 100644 index 00000000..f343d5a0 --- /dev/null +++ b/models/silver/prices/silver__complete_token_prices.sql @@ -0,0 +1,46 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_token_prices_id', + tags = ['non_realtime'] +) }} + +SELECT + HOUR, + LOWER( + p.token_address + ) AS token_address, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + blockchain_name, + blockchain_id, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_token_prices' + ) }} + p + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_token_prices.yml b/models/silver/prices/silver__complete_token_prices.yml new file mode 100644 index 00000000..d2ac57f5 --- /dev/null +++ b/models/silver/prices/silver__complete_token_prices.yml @@ -0,0 +1,41 @@ +version: 2 +models: + - name: silver__complete_token_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - TOKEN_ADDRESS + - BLOCKCHAIN + + columns: + - name: HOUR + tests: + - not_null + - name: TOKEN_ADDRESS + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCKCHAIN_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: IS_IMPUTED + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_TOKEN_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 5cba6993..b6685008 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -34,6 +34,12 @@ sources: - name: asset_metadata_all_providers - name: near_address_encoded - name: labels_combined + - name: complete_token_asset_metadata + - name: complete_token_prices + - name: complete_provider_asset_metadata + - name: complete_provider_prices + - name: complete_native_asset_metadata + - name: complete_native_prices - name: crosschain_public database: crosschain schema: bronze_public