An 418/new cols part2 (#423)

* add new cols * new pk/date cols update * separate _generic logic * add back _log_id * add _inserted_timestamp back
FlipsideCrypto · Dec 11, 2023 · d58726e · d58726e
1 parent 42d48b9
commit d58726e
Show file tree

Hide file tree

Showing 92 changed files with 1,961 additions and 779 deletions.
diff --git a/models/gold/defi/defi__fact_liquidity_pool_actions.sql b/models/gold/defi/defi__fact_liquidity_pool_actions.sql
@@ -15,7 +15,21 @@ SELECT
     liquidity_pool_address,
     amount,
     mint,
-    _inserted_timestamp
+    _inserted_timestamp,
+    COALESCE (
+        liquidity_pool_actions_raydium_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['BLOCK_ID','TX_ID','INDEX','INNER_INDEX']
+        ) }}
+    ) AS fact_liquidity_pool_actions_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__liquidity_pool_actions_raydium') }}
 UNION
@@ -30,7 +44,21 @@ SELECT
     liquidity_pool_address,
     amount,
     mint,
-    _inserted_timestamp
+    _inserted_timestamp,
+    COALESCE (
+        liquidity_pool_actions_orca_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['BLOCK_ID','TX_ID','INDEX','INNER_INDEX']
+        ) }}
+    ) AS fact_liquidity_pool_actions_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__liquidity_pool_actions_orca') }}
 UNION
@@ -45,6 +73,20 @@ SELECT
     liquidity_pool_address,
     amount,
     mint,
-    _inserted_timestamp
+    _inserted_timestamp,
+    COALESCE (
+        liquidity_pool_actions_saber_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['BLOCK_ID','TX_ID','ACTION_INDEX']
+        ) }}
+    ) AS fact_liquidity_pool_actions_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__liquidity_pool_actions_saber') }}
diff --git a/models/gold/defi/defi__fact_liquidity_pool_actions.yml b/models/gold/defi/defi__fact_liquidity_pool_actions.yml
@@ -42,4 +42,10 @@ models:
       - name: MINT
         description: "{{ doc('mint') }}"
         tests:
-          - dbt_expectations.expect_column_to_exist
+          - dbt_expectations.expect_column_to_exist
+      - name: FACT_LIQUIDITY_POOL_ACTIONS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 
diff --git a/models/gold/defi/defi__fact_stake_pool_actions.sql b/models/gold/defi/defi__fact_stake_pool_actions.sql
@@ -1,23 +1,13 @@
 {{ config(
     materialized = 'view',
-    meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }}},
+    meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }} },
     tags = ['scheduled_non_core']
 ) }}
 
-{% for model_suffix in ["generic","socean","lido","eversol"] %}
+{% for model_suffix in ["socean","lido","eversol"] %}
 
     SELECT
-        CASE
-            WHEN '{{ model_suffix }}' = 'generic' THEN CASE
-                WHEN stake_pool = '7ge2xKsZXmqPxa3YmXxXmzCp9Hc2ezrTxh6PECaxCwrL' THEN 'daopool'
-                WHEN stake_pool = 'CtMyWsrUtAwXWiGr9WjHT5fC3p3fgV8cyGpLTo2LJzG1' THEN 'jpool'
-                WHEN stake_pool = 'stk9ApL5HeVAwPLr3TLhDXdZS8ptVu7zp6ov8HFDuMi' THEN 'blazestake'
-                WHEN stake_pool = 'Jito4APyf642JPZPx3hGc6WWJ8zPKtRbRs4P815Awbb' THEN 'jito'
-                WHEN stake_pool = 'CgnTSoL3DgY9SFHxcLj6CgCgKKoTBr6tp4CPAEWy25DE' THEN 'cogent'
-                WHEN stake_pool = 'LAinEtNLgpmCP9Rvsf5Hn8W6EhNiKLZQti1xfWMLy6X' THEN 'laine'
-            END
-            ELSE '{{ model_suffix }}'
-        END AS stake_pool_name,
+        '{{ model_suffix }}' AS stake_pool_name,
         tx_id,
         block_id,
         block_timestamp,
@@ -27,7 +17,21 @@
         address,
         stake_pool,
         amount,
-        'SOL' as token
+        'SOL' AS token,
+        COALESCE (
+         'stake_pool_actions_' + '{{ model_suffix }}' + '_id',
+            {{ dbt_utils.generate_surrogate_key(
+                ['tx_id','index']
+            ) }}
+        ) AS fact_stake_pool_actions_id,
+        COALESCE(
+            inserted_timestamp,
+            '2000-01-01'
+        ) AS inserted_timestamp,
+        COALESCE(
+            modified_timestamp,
+            '2000-01-01'
+        ) AS modified_timestamp
     FROM
         {{ ref(
             'silver__stake_pool_actions_' + model_suffix
@@ -37,20 +41,72 @@
         UNION ALL
         {% endif %}
     {% endfor %}
-    UNION ALL
-    SELECT
-        'marinade' as stake_pool_name,
-        tx_id,
-        block_id,
-        block_timestamp,
-        INDEX,
-        succeeded,
-        action,
-        address,
-        stake_pool,
-        amount,
-        token
-    FROM
-        {{ ref(
-            'silver__stake_pool_actions_marinade'
+UNION ALL
+SELECT
+    CASE
+        WHEN stake_pool = '7ge2xKsZXmqPxa3YmXxXmzCp9Hc2ezrTxh6PECaxCwrL' THEN 'daopool'
+        WHEN stake_pool = 'CtMyWsrUtAwXWiGr9WjHT5fC3p3fgV8cyGpLTo2LJzG1' THEN 'jpool'
+        WHEN stake_pool = 'stk9ApL5HeVAwPLr3TLhDXdZS8ptVu7zp6ov8HFDuMi' THEN 'blazestake'
+        WHEN stake_pool = 'Jito4APyf642JPZPx3hGc6WWJ8zPKtRbRs4P815Awbb' THEN 'jito'
+        WHEN stake_pool = 'CgnTSoL3DgY9SFHxcLj6CgCgKKoTBr6tp4CPAEWy25DE' THEN 'cogent'
+        WHEN stake_pool = 'LAinEtNLgpmCP9Rvsf5Hn8W6EhNiKLZQti1xfWMLy6X' THEN 'laine'
+    END AS stake_pool_name,
+    tx_id,
+    block_id,
+    block_timestamp,
+    INDEX,
+    succeeded,
+    action,
+    address,
+    stake_pool,
+    amount,
+    'SOL' AS token,
+    COALESCE (
+        stake_pool_actions_generic_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_id','index', 'inner_index']
+        ) }}
+    ) AS fact_stake_pool_actions_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
+FROM
+    {{ ref(
+        'silver__stake_pool_actions_generic'
+    ) }}
+UNION ALL
+SELECT
+    'marinade' AS stake_pool_name,
+    tx_id,
+    block_id,
+    block_timestamp,
+    INDEX,
+    succeeded,
+    action,
+    address,
+    stake_pool,
+    amount,
+    token,
+    COALESCE (
+        stake_pool_actions_marinade_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_id','index']
         ) }}
+    ) AS fact_stake_pool_actions_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
+FROM
+    {{ ref(
+        'silver__stake_pool_actions_marinade'
+    ) }}
diff --git a/models/gold/defi/defi__fact_stake_pool_actions.yml b/models/gold/defi/defi__fact_stake_pool_actions.yml
@@ -46,4 +46,10 @@ models:
       - name: TOKEN
         description: "Token utilized in the stake pool action"
         tests:
-          - dbt_expectations.expect_column_to_exist
+          - dbt_expectations.expect_column_to_exist
+      - name: FACT_STAKE_POOL_ACTIONS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 
diff --git a/models/gold/defi/defi__fact_swaps.sql b/models/gold/defi/defi__fact_swaps.sql
@@ -16,7 +16,21 @@ SELECT
     to_mint AS swap_to_mint,
     program_id,
     l.address_name AS swap_program,
-    _log_id
+    _log_id,
+    COALESCE (
+       swaps_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['block_id','tx_id','program_id']
+        ) }}
+    ) AS fact_swaps_id,
+    COALESCE(
+        s.inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        s.modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__swaps') }}
     s
@@ -36,7 +50,10 @@ SELECT
     to_mint AS swap_to_mint,
     program_id,
     l.address_name AS swap_program,
-    concat_ws('-',tx_id,swap_index) as _log_id
+    concat_ws('-',tx_id,swap_index) as _log_id,
+    swaps_intermediate_jupiterv6_id as fact_swaps_id,
+    s.inserted_timestamp,
+    s.modified_timestamp
 FROM
     {{ ref('silver__swaps_intermediate_jupiterv6') }}
     s
@@ -56,7 +73,10 @@ SELECT
     to_mint AS swap_to_mint,
     program_id,
     l.address_name AS swap_program,
-    concat_ws('-',tx_id,swap_index) as _log_id
+    concat_ws('-',tx_id,swap_index) as _log_id,
+    swaps_intermediate_jupiterv5_id as fact_swaps_id,
+    s.inserted_timestamp,
+    s.modified_timestamp
 FROM
     {{ ref('silver__swaps_intermediate_jupiterv5_1_view') }}
     s
@@ -76,7 +96,10 @@ SELECT
     to_mint AS swap_to_mint,
     program_id,
     l.address_name AS swap_program,
-    concat_ws('-',tx_id,swap_index) as _log_id
+    concat_ws('-',tx_id,swap_index) as _log_id,
+    swaps_intermediate_jupiterv5_id as fact_swaps_id,
+    s.inserted_timestamp,
+    s.modified_timestamp
 FROM
     {{ ref('silver__swaps_intermediate_jupiterv5_2_view') }}
     s

diff --git a/models/gold/defi/defi__fact_swaps.yml b/models/gold/defi/defi__fact_swaps.yml
@@ -50,4 +50,10 @@ models:
       - name: _LOG_ID
         description: "Combination of TX_ID and event index"
         tests: 
-          - not_null
+          - not_null
+      - name: FACT_SWAPS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 
diff --git a/models/gold/defi/defi__fact_token_burn_actions.sql b/models/gold/defi/defi__fact_token_burn_actions.sql
@@ -17,7 +17,21 @@ SELECT
     burn_authority,
     signers,
     DECIMAL,
-    mint_standard_type
+    mint_standard_type,
+    COALESCE (
+        token_burn_actions_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_id', 'index', 'inner_index', 'mint']
+        ) }}
+    ) AS fact_token_burn_actions_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 
 FROM
     {{ ref('silver__token_burn_actions') }}
diff --git a/models/gold/defi/defi__fact_token_burn_actions.yml b/models/gold/defi/defi__fact_token_burn_actions.yml
@@ -55,3 +55,9 @@ models:
         description: "{{ doc('mint_standard_type') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+      - name: FACT_TOKEN_BURN_ACTIONS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 
diff --git a/models/gold/defi/defi__fact_token_mint_actions.sql b/models/gold/defi/defi__fact_token_mint_actions.sql
@@ -17,7 +17,21 @@ SELECT
     mint_authority,
     signers,
     DECIMAL,
-    mint_standard_type
+    mint_standard_type,
+    COALESCE (
+        token_mint_actions_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_id', 'index', 'inner_index', 'mint']
+        ) }}
+    ) AS fact_token_mint_actions_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__token_mint_actions') }}
 

diff --git a/models/gold/defi/defi__fact_token_mint_actions.yml b/models/gold/defi/defi__fact_token_mint_actions.yml
@@ -55,4 +55,10 @@ models:
         description: "{{ doc('mint_standard_type') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+      - name: FACT_TOKEN_MINT_ACTIONS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}'