An 4201/new columns (#213)

* stash

* typo

* name

* spacing

* event_index

* date

* fixes

* typo

---------

Co-authored-by: sam <[email protected]>

dbt_project.yml

@@ -50,6 +50,7 @@ query-comment:
 
 models:
   +copy_grants: true
+  +on_schema_change: "append_new_columns"
 
 # In this example config, we tell dbt to build all models in the example/ directory
 # as tables. These settings can be overridden in the individual model files

models/bronze/ethereum/bronze__state_hashes.sql

@@ -17,7 +17,13 @@ SELECT
     state_prev_total_elements,
     state_min_block,
     state_max_block,
-    _inserted_timestamp
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['state_block_number']
+    ) }} AS state_hashes_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     {{ source(
         'ethereum_silver',

models/bronze/ethereum/bronze__submission_hashes.sql

@@ -15,7 +15,13 @@ SELECT
     l1_submission_prev_total_elements,
     sub_min_block,
     sub_max_block,
-    _inserted_timestamp
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['l1_submission_block_number']
+    ) }} AS submission_hashes_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     {{ source(
         'ethereum_silver',

models/doc_descriptions/general/deprecation.md

@@ -1,12 +1,11 @@
-{% docs deprecation %}
+{% docs internal_column %}    
 
-Deprecating soon: This is a notice that we're only removing the below columns. Please migrate queries using these columns to `fact_decoded_event_logs`, `ez_decoded_event_logs` or use manual parsing of topics and data. The following columns will be deprecated on 7/16/23:
+Deprecated. This column is no longer used. Please remove from your query by Jan. 31 2024.'
 
-`Fact_event_logs` Columns:
-- `event_name`
-- `event_inputs`
-- `contract_name`
+{% enddocs %}
+
+{% docs amount_deprecation %}   
+
+This column is being deprecated for standardization purposes on Jan. 31 2024. Please use the equivalent column without the native asset prefix. For example, use `amount` instead of `eth_amount`.
 
-`Fact_transactions` Columns:
-- `tx_json`
 {% enddocs %}

models/doc_descriptions/general/export_columns.md

@@ -0,0 +1,19 @@
+{% docs pk %}
+
+The unique identifier for each row in the table.
+
+{% enddocs %}
+
+{% docs inserted_timestamp %}
+
+The utc timestamp at which the row was inserted into the table.
+
+{% enddocs %}
+
+{% docs modified_timestamp %}
+
+The utc timestamp at which the row was last modified.
+
+{% enddocs %}
+
+

models/doc_descriptions/nft/nft_intra_event_index.md

@@ -0,0 +1,5 @@
+{% docs nft_intra_event_index %}
+
+The order of events within a single event index. This is primarily used for ERC1155 NFT batch transfer events. 
+
+{% enddocs %}

models/doc_descriptions/traces/opt_traces_index.md

@@ -0,0 +1,5 @@
+{% docs opt_trace_index %}
+
+The index of the trace within the transaction.
+
+{% enddocs %}

models/gold/core/core__dim_contracts.sql

@@ -12,7 +12,15 @@ SELECT
     c0.block_number AS created_block_number,
     c0.block_timestamp AS created_block_timestamp,
     c0.tx_hash AS created_tx_hash,
-    c0.creator_address AS creator_address
+    c0.creator_address AS creator_address,
+    COALESCE (
+        c0.created_contracts_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['c0.created_contract_address']
+        ) }}
+    ) AS dim_contracts_id,
+    GREATEST(COALESCE(c0.inserted_timestamp, '2000-01-01'), COALESCE(c1.inserted_timestamp, '2000-01-01')) AS inserted_timestamp,
+    GREATEST(COALESCE(c0.modified_timestamp, '2000-01-01'), COALESCE(c1.modified_timestamp, '2000-01-01')) AS modified_timestamp
 FROM
     {{ ref('silver__created_contracts') }}
     c0

models/gold/core/core__dim_contracts.yml

@@ -20,3 +20,9 @@ models:
         description: 'The transaction hash when the contract was created'
       - name: CREATOR_ADDRESS
         description: 'The address of the creator of the contract'
+      - name: DIM_CONTRACTS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 

models/gold/core/core__dim_labels.sql

@@ -11,6 +11,20 @@ SELECT
     address_name,
     label_type,
     label_subtype,
-    project_name
+    project_name,
+    COALESCE (
+        labels_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['address']
+        ) }}
+    ) AS dim_labels_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__labels') }}

models/gold/core/core__dim_labels.yml

@@ -52,4 +52,9 @@ models:
               column_type_list:
                 - STRING
                 - VARCHAR
-
+      - name: DIM_LABELS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 

models/gold/core/core__ez_decoded_event_logs.sql

@@ -20,7 +20,16 @@ SELECT
     topics,
     DATA,
     event_removed,
-    tx_status
+    tx_status,
+    COALESCE (
+        decoded_logs_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_hash', 'event_index']
+        ) }}
+    ) AS ez_decoded_event_logs_id,
+    GREATEST(COALESCE(l.inserted_timestamp, '2000-01-01'), COALESCE(C.inserted_timestamp, '2000-01-01')) AS inserted_timestamp,
+    GREATEST(COALESCE(l.modified_timestamp, '2000-01-01'), COALESCE(C.modified_timestamp, '2000-01-01')) AS modified_timestamp
 FROM
-    {{ ref('silver__decoded_logs') }} 
-    LEFT JOIN {{ ref('silver__contracts') }} using (contract_address)
+    {{ ref('silver__decoded_logs') }}
+    l
+    LEFT JOIN {{ ref('silver__contracts') }} C USING (contract_address)

models/gold/core/core__ez_decoded_event_logs.yml

@@ -56,4 +56,10 @@ models:
       - name: EVENT_REMOVED
         description: '{{ doc("opt_event_removed") }}' 
       - name: TX_STATUS
-        description: '{{ doc("opt_tx_status") }}' 
+        description: '{{ doc("opt_tx_status") }}' 
+      - name: EZ_DECODED_EVENT_LOGS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 

models/gold/core/core__ez_eth_transfers.sql

@@ -1,104 +1,26 @@
 {{ config(
-    materialized = 'incremental',
-    incremental_strategy = 'delete+insert',
-    unique_key = 'block_number',
-    cluster_by = ['block_timestamp::DATE'],
-    tags = ['core','non_realtime','reorg'],
+    materialized = 'view',
     persist_docs ={ "relation": true,
     "columns": true }
 ) }}
 
-WITH eth_base AS (
-
-    SELECT
-        tx_hash,
-        block_number,
-        block_timestamp,
-        identifier,
-        from_address,
-        to_address,
-        eth_value,
-        _call_id,
-        _inserted_timestamp,
-        eth_value_precise_raw,
-        eth_value_precise,
-        tx_position,
-        trace_index
-    FROM
-        {{ ref('silver__traces') }}
-    WHERE
-        eth_value > 0
-        AND tx_status = 'SUCCESS'
-        AND trace_status = 'SUCCESS'
-        AND TYPE NOT IN (
-            'DELEGATECALL',
-            'STATICCALL'
-        )
-
-{% if is_incremental() %}
-AND _inserted_timestamp >= (
-    SELECT
-        MAX(_inserted_timestamp) - INTERVAL '72 hours'
-    FROM
-        {{ this }}
-)
-{% endif %}
-),
-tx_table AS (
-    SELECT
-        block_number,
-        tx_hash,
-        from_address AS origin_from_address,
-        to_address AS origin_to_address,
-        origin_function_signature
-    FROM
-        {{ ref('silver__transactions') }}
-    WHERE
-        tx_hash IN (
-            SELECT
-                DISTINCT tx_hash
-            FROM
-                eth_base
-        )
-
-{% if is_incremental() %}
-AND _inserted_timestamp >= (
-    SELECT
-        MAX(_inserted_timestamp) - INTERVAL '72 hours'
-    FROM
-        {{ this }}
-)
-{% endif %}
-)
 SELECT
-    tx_hash AS tx_hash,
-    block_number AS block_number,
-    block_timestamp AS block_timestamp,
-    identifier AS identifier,
+    tx_hash,
+    block_number,
+    block_timestamp,
+    identifier,
     origin_from_address,
     origin_to_address,
     origin_function_signature,
     from_address AS eth_from_address,
     to_address AS eth_to_address,
-    eth_value AS amount,
-    eth_value_precise_raw AS amount_precise_raw,
-    eth_value_precise AS amount_precise,
-    ROUND(
-        eth_value * price,
-        2
-    ) AS amount_usd,
+    amount,
+    amount_precise_raw,
+    amount_precise,
+    amount_usd,
     _call_id,
     _inserted_timestamp,
     tx_position,
     trace_index
 FROM
-    eth_base A
-    LEFT JOIN {{ ref('silver__hourly_prices_priority_eth') }}
-    ON DATE_TRUNC(
-        'hour',
-        A.block_timestamp
-    ) = HOUR
-    JOIN tx_table USING (
-        tx_hash,
-        block_number
-    )
+    {{ ref('silver__native_transfers') }}

models/gold/core/core__ez_eth_transfers.yml

@@ -1,7 +1,7 @@
 version: 2
 models:
   - name: core__ez_eth_transfers
-    description: '{{ doc("opt_ez_eth_transfers_table_doc") }}'
+    description: 'Deprecating soon! Migrate to `core.ez_native_transfers` by Jan. 31 2024.'
 
     columns:
       - name: BLOCK_NUMBER

models/gold/core/core__ez_native_transfers.sql

@@ -0,0 +1,38 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    tx_hash,
+    block_number,
+    block_timestamp,
+    tx_position,
+    trace_index,
+    identifier,
+    origin_from_address,
+    origin_to_address,
+    origin_function_signature,
+    from_address,
+    to_address,
+    amount,
+    amount_precise_raw,
+    amount_precise,
+    amount_usd,
+    COALESCE (
+        native_transfers_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_hash', 'trace_index']
+        ) }}
+    ) AS ez_native_transfers_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
+FROM
+    {{ ref('silver__native_transfers') }}

models/gold/core/core__ez_native_transfers.yml

@@ -0,0 +1,42 @@
+version: 2
+models:
+  - name: core__ez_native_transfers
+    description: '{{ doc("opt_ez_eth_transfers_table_doc") }}'
+
+    columns:
+      - name: TX_HASH
+        description: '{{ doc("opt_transfer_tx_hash") }}'
+      - name: BLOCK_NUMBER
+        description: '{{ doc("opt_block_number") }}'
+      - name: BLOCK_TIMESTAMP
+        description: '{{ doc("opt_block_timestamp") }}'
+      - name: TX_POSITION
+        description: '{{ doc("opt_tx_position") }}'
+      - name: TRACE_INDEX
+        description: '{{ doc("opt_trace_index") }}'
+      - name: IDENTIFIER
+        description: '{{ doc("opt_traces_identifier") }}'
+      - name: ORIGIN_FROM_ADDRESS
+        description: '{{ doc("opt_origin_from") }}'
+      - name: ORIGIN_TO_ADDRESS
+        description: '{{ doc("opt_origin_to") }}'
+      - name: ORIGIN_FUNCTION_SIGNATURE
+        description: '{{ doc("opt_origin_sig") }}'
+      - name: FROM_ADDRESS
+        description: '{{ doc("opt_transfer_from_address") }}'
+      - name: TO_ADDRESS
+        description: '{{ doc("opt_transfer_to_address") }}'
+      - name: AMOUNT
+        description: '{{ doc("opt_eth_amount") }}'
+      - name: AMOUNT_PRECISE_RAW
+        description: '{{ doc("precise_amount_unadjusted") }}'
+      - name: AMOUNT_PRECISE
+        description: '{{ doc("precise_amount_adjusted") }}'
+      - name: AMOUNT_USD
+        description: '{{ doc("opt_eth_amount_usd") }}' 
+      - name: EZ_NATIVE_TRANSFERS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}'