An 5449/update jup v4v5 gold (#707)

* move jupv4/v5 to gold jupiter table * add source * add cols to view * adjust test and limit cols * null cols;suceeded filter;doc update * convert to tables * table desc update jup v4/v5 removal
FlipsideCrypto · Nov 29, 2024 · d2404d2 · d2404d2
1 parent c9d89b3
commit d2404d2
Show file tree

Hide file tree

Showing 10 changed files with 310 additions and 67 deletions.
diff --git a/models/gold/defi/defi__fact_swaps.sql b/models/gold/defi/defi__fact_swaps.sql
@@ -41,65 +41,18 @@ SELECT
     modified_timestamp
 FROM
     {{ ref('silver__swaps') }}
-{% if is_incremental() %}
 WHERE
+    program_id != 'JUP4Fb2cqiRUcaTHdrPC8h2gNsA2ETXiPDD33WcGuJB'
+{% if is_incremental() %}
+AND
     modified_timestamp >= '{{ max_modified_timestamp }}'
 {% else %}
-WHERE
+AND
     modified_timestamp::date < '{{ backfill_to_date }}'
 {% endif %}
 )
 ,
 swaps_individual as (
-SELECT
-    block_timestamp,
-    block_id,
-    tx_id,
-    succeeded,
-    swapper,
-    from_amt AS swap_from_amount,
-    from_mint AS swap_from_mint,
-    to_amt AS swap_to_amount,
-    to_mint AS swap_to_mint,
-    program_id,
-    swap_index,
-    swaps_intermediate_jupiterv5_id as fact_swaps_id,
-    inserted_timestamp,
-    modified_timestamp
-FROM
-    {{ ref('silver__swaps_intermediate_jupiterv5_1_view') }}
-{% if is_incremental() %}
-WHERE
-    modified_timestamp >= '{{ max_modified_timestamp }}'
-{% else %}
-WHERE
-    modified_timestamp::date < '{{ backfill_to_date }}'
-{% endif %}
-UNION ALL
-SELECT
-    block_timestamp,
-    block_id,
-    tx_id,
-    succeeded,
-    swapper,
-    from_amt AS swap_from_amount,
-    from_mint AS swap_from_mint,
-    to_amt AS swap_to_amount,
-    to_mint AS swap_to_mint,
-    program_id,
-    swap_index,
-    swaps_intermediate_jupiterv5_id as fact_swaps_id,
-    inserted_timestamp,
-    modified_timestamp
-FROM
-    {{ ref('silver__swaps_intermediate_jupiterv5_2_view') }}
-{% if is_incremental() %}
-WHERE modified_timestamp >= '{{ max_modified_timestamp }}'
-{% else %}
-WHERE
-    modified_timestamp::date < '{{ backfill_to_date }}'
-{% endif %}
-UNION ALL
 SELECT
     block_timestamp,
     block_id,

diff --git a/models/gold/defi/defi__fact_swaps.yml b/models/gold/defi/defi__fact_swaps.yml
@@ -1,7 +1,7 @@
 version: 2
 models:
   - name: defi__fact_swaps
-    description: This table contains swaps performed on Jupiter (V4 and V5), Orca, Raydium, Saber, Bonkswap, Dooar, Phoenix and Meteora swap programs. Intermediate swaps are aggregated over the DEX programs, so the values showcase the final mint/amount swap values. Ie. a swap on Jupiter that swaps SOL->USDC->mSOL->ETH would show the initial amount in and the final amount out of SOL->ETH. For Phoenix, we are not capturing swaps where there are separate transactions for placing the order and filling the order. 
+    description: This table contains swaps performed on Jupiter (V4 and V5), Orca, Raydium, Saber, Bonkswap, Dooar, Phoenix and Meteora swap programs. Intermediate swaps are aggregated over the DEX programs, so the values showcase the final mint/amount swap values. Ie. a swap on Jupiter that swaps SOL->USDC->mSOL->ETH would show the initial amount in and the final amount out of SOL->ETH. For Phoenix, we are not capturing swaps where there are separate transactions for placing the order and filling the order. NOTE - Jupiter V4 and V5 swaps will be removed from this table on December 9th, and the corresponding swaps will exist in defi.fact_swaps_jupiter_summary. 
     recent_date_filter: &recent_date_filter
       config:
         where: block_timestamp >= current_date - 7
@@ -11,7 +11,6 @@ models:
     tests:
       - reference_tx_missing:
           reference_tables:
-            - 'silver__swaps'
             - 'silver__swaps_intermediate_bonkswap'
             - 'silver__swaps_intermediate_meteora'
             - 'silver__swaps_intermediate_dooar'

diff --git a/models/gold/defi/defi__fact_swaps_jupiter_inner.sql b/models/gold/defi/defi__fact_swaps_jupiter_inner.sql
@@ -1,9 +1,73 @@
 {{ config(
-    materialized = 'view',
+    materialized = 'incremental',
     meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }}},
+    unique_key = ['fact_swaps_jupiter_inner_id'],
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
+    cluster_by = ['block_timestamp::DATE','swap_program_id'],
+    merge_exclude_columns = ["inserted_timestamp"],
+    post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id, swapper, swap_from_mint, swap_to_mint, swap_program_id, aggregator_program_id, fact_swaps_jupiter_inner_id)'),
     tags = ['scheduled_non_core']
 ) }}
 
+{% if execute %}
+    {% if is_incremental() %}
+        {% set query %}
+            SELECT MAX(modified_timestamp) AS max_modified_timestamp
+            FROM {{ this }}
+        {% endset %}
+        {% set max_modified_timestamp = run_query(query).columns[0].values()[0] %}
+    {% endif %}
+{% endif %}
+
+-- Select from the non-active models only during the initial FR
+{% if not is_incremental() %}
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    index,
+    inner_index,
+    swap_index,
+    succeeded,
+    swapper,
+    from_mint AS swap_from_mint,
+    from_amt AS swap_from_amount,
+    to_mint AS swap_to_mint,
+    to_amt AS swap_to_amount,
+    inner_swap_program_id AS swap_program_id,
+    program_id AS aggregator_program_id,
+    {{ dbt_utils.generate_surrogate_key(['tx_id','index','inner_index']) }} AS fact_swaps_jupiter_inner_id,
+    inserted_timestamp,
+    modified_timestamp
+FROM
+    {{ ref('silver__swaps_intermediate_jupiterv4_view') }}
+WHERE
+    block_timestamp <= '2023-10-31'
+    and succeeded
+UNION ALL
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    index,
+    inner_index,
+    swap_index,
+    succeeded,
+    swapper,
+    from_mint AS swap_from_mint,
+    from_amount AS swap_from_amount,
+    to_mint AS swap_to_mint,
+    to_amount AS swap_to_amount,
+    swap_program_id,
+    aggregator_program_id,
+    swaps_inner_intermediate_jupiterv5_id AS fact_swaps_jupiter_inner_id,
+    inserted_timestamp,
+    modified_timestamp
+FROM
+    {{ ref('silver__swaps_inner_intermediate_jupiterv5') }}
+UNION ALL
+{% endif %}
+-- Only select from active models during incremental
 SELECT
     block_timestamp,
     block_id,
@@ -21,6 +85,36 @@ SELECT
     aggregator_program_id,
     swaps_inner_intermediate_jupiterv6_id AS fact_swaps_jupiter_inner_id,
     inserted_timestamp,
-    modified_timestamp,
+    modified_timestamp
+FROM
+    {{ ref('silver__swaps_inner_intermediate_jupiterv6') }}
+{% if is_incremental() %}
+WHERE modified_timestamp >= '{{ max_modified_timestamp }}'
+{% endif %}
+UNION ALL
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    index,
+    inner_index,
+    swap_index,
+    succeeded,
+    swapper,
+    from_mint AS swap_from_mint,
+    from_amount AS swap_from_amount,
+    to_mint AS swap_to_mint,
+    to_amount AS swap_to_amount,
+    swap_program_id,
+    aggregator_program_id,
+    swaps_inner_intermediate_jupiterv4_id AS fact_swaps_jupiter_inner_id,
+    inserted_timestamp,
+    modified_timestamp
 FROM
-    {{ ref('silver__swaps_inner_intermediate_jupiterv6') }}
+    {{ ref('silver__swaps_inner_intermediate_jupiterv4') }}
+WHERE 
+    block_timestamp > '2023-10-31'
+{% if is_incremental() %}
+AND modified_timestamp >= '{{ max_modified_timestamp }}'
+{% endif %}
+
diff --git a/models/gold/defi/defi__fact_swaps_jupiter_inner.yml b/models/gold/defi/defi__fact_swaps_jupiter_inner.yml
@@ -2,69 +2,93 @@ version: 2
 models:
   - name: defi__fact_swaps_jupiter_inner
     description: This table contain each intermediate swap that is a part of a Jupiter route. These are the individual steps that are executed by Jupiter to complete a swap.
+    recent_date_filter: &recent_date_filter
+      config:
+        where: modified_timestamp >= current_date - 7
+    tests:
+      - reference_tx_missing:
+          reference_tables:
+            - 'silver__swaps_inner_intermediate_jupiterv6'
+            - 'silver__swaps_inner_intermediate_jupiterv4'
+          id_column: 'tx_id'
     columns:
       - name: BLOCK_TIMESTAMP
         description: "{{ doc('block_timestamp') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: BLOCK_ID
         description: "{{ doc('block_id') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: TX_ID
         description: "{{ doc('tx_id') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: INDEX
         description: "{{ doc('event_index') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: INNER_INDEX
         description: "{{ doc('inner_index') }}. This is the inner index of the log event listing the inner swap"
         tests:
           - dbt_expectations.expect_column_to_exist
       - name: SWAP_INDEX
-        description: "{{ doc('swaps_swap_index') }} as it relates to the top level Jupiter V6 swap instruction"
+        description: "{{ doc('swaps_swap_index') }} as it relates to the top level Jupiter swap instruction"
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: SUCCEEDED
         description: "{{ doc('tx_succeeded') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: SWAP_PROGRAM_ID
         description: "{{ doc('program_id') }}. This is the AMM performing the swap."
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: AGGREGATOR_PROGRAM_ID
         description: "{{ doc('program_id') }}. This is the aggregator calling the different AMMs."
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: SWAPPER
         description: "{{ doc('swaps_swapper') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: SWAP_FROM_AMOUNT
         description:  "{{ doc('swaps_from_amt') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: SWAP_FROM_MINT
         description:  "{{ doc('swaps_from_mint') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: SWAP_TO_AMOUNT
         description:  "{{ doc('swaps_to_amt') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: SWAP_TO_MINT
         description:  "{{ doc('swaps_to_mint') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: FACT_SWAPS_JUPITER_INNER_ID
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: INSERTED_TIMESTAMP
         tests:
           - dbt_expectations.expect_column_to_exist
+          - not_null: *recent_date_filter
       - name: MODIFIED_TIMESTAMP
         description: '{{ doc("modified_timestamp") }}' 
         tests: