Merge branch 'main' into main

dbt_project.yml

@@ -774,9 +774,6 @@ models:
       optimism:
         +schema: ironbank_optimism
 
-    keep3r_network:
-      +schema: keep3r_network
-
     airswap:
       +schema: airswap
       ethereum:

macros/dune/source.sql

@@ -1,7 +1,7 @@
-{% macro source(source_name, table_name) %}
+{% macro source(source_name, table_name, database="delta_prod") %}
   {% set rel = builtins.source(source_name, table_name) %}
   {%- if target.type == 'trino' -%}
-    {% set newrel = rel.replace_path(database="delta_prod") %}
+    {% set newrel = rel.replace_path(database=database) %}
     {% do return(newrel) %}
   {%- else -%}
     {% do return(rel) %}

macros/models/_project/balancer/balancer_bpt_prices_macro.sql

@@ -26,6 +26,15 @@ WITH pool_labels AS (
         GROUP BY 1, 2, 3
     ),
 
+    gyro_prices AS (
+        SELECT
+            token_address,
+            decimals,
+            price
+        FROM {{ ref('gyroscope_gyro_tokens') }}
+        WHERE blockchain = '{{blockchain}}'
+    ),
+
     swaps_changes AS (
         SELECT
             day,
@@ -147,15 +156,15 @@ WITH pool_labels AS (
             '{{blockchain}}' as blockchain,
             b.pool_id,
             b.token,
-            cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals)) * COALESCE(p1.price, 0) AS protocol_liquidity_usd
+            cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals, p4.decimals)) * COALESCE(p1.price, p4.price, 0) AS protocol_liquidity_usd
         FROM calendar c
         LEFT JOIN cumulative_balance b ON b.day <= c.day
         AND c.day < b.day_of_next_change
         LEFT JOIN {{ source('tokens', 'erc20') }} t ON t.contract_address = b.token
         AND blockchain = '{{blockchain}}'
         LEFT JOIN prices p1 ON p1.day = b.day
         AND p1.token = b.token
-
+        LEFT JOIN gyro_prices p4 ON p4.token_address = b.token
         WHERE b.token != BYTEARRAY_SUBSTRING(b.pool_id, 1, 20)
     ),
 
@@ -175,7 +184,7 @@ WITH pool_labels AS (
         AND b.blockchain = q.chain
         LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(b.pool_id, 1, 20)
         WHERE q.name IS NOT NULL 
-        AND p.pool_type IN ('WP', 'WP2T') -- filters for weighted pools with pricing assets
+        AND p.pool_type IN ('weighted') -- filters for weighted pools with pricing assets
         AND w.blockchain = '{{blockchain}}'
         AND w.version = '{{version}}'
         GROUP BY 1, 2, 3, 4
@@ -214,7 +223,7 @@ WITH pool_labels AS (
         FROM {{ source('balancer_v2_' ~ blockchain, 'Vault_evt_Swap') }} v
         LEFT JOIN pool_labels l ON bytearray_substring(v.poolId, 1, 20) = l.pool_id
         WHERE v.tokenIn = bytearray_substring(v.poolId, 1, 20) OR v.tokenOut = bytearray_substring(v.poolId, 1, 20)
-        AND l.pool_type = 'LP'
+        AND l.pool_type = 'linear'
     ), 
 
     all_trades_info AS (
@@ -370,7 +379,10 @@ WITH pool_labels AS (
         l.version,
         18 AS decimals,
         l.pool_address AS contract_address,
-        CASE WHEN pl.pool_type = 'LP' AND median_price IS NOT NULL
+        pl.pool_type,
+        CASE WHEN pl.pool_type = 'linear' AND median_price IS NOT NULL
+        THEN p.median_price
+        WHEN l.liquidity = 0 AND median_price IS NOT NULL 
         THEN p.median_price
         ELSE l.liquidity / s.supply 
         END AS bpt_price
@@ -381,6 +393,6 @@ WITH pool_labels AS (
     AND l.day = s.day
     LEFT JOIN price_formulation p ON p.day = l.day AND p.contract_address = l.pool_address
     LEFT JOIN pool_labels pl ON pl.pool_id = l.pool_address
-    WHERE supply > 0
+    WHERE supply > 1
 
-    {% endmacro %}
+    {% endmacro %}

macros/models/_project/balancer/balancer_bpt_supply_macro.sql

@@ -67,8 +67,8 @@ WITH pool_labels AS (
     premints AS (
         SELECT 
             p.address AS bpt, 
-            CASE WHEN pool_type = 'LP' THEN CAST('5192296858534827628530496329220095' AS INT256) / POWER(10, 18)
-                WHEN pool_type IN ('SP') THEN CAST(m.delta AS INT256) / POWER(10, 18)
+            CASE WHEN pool_type IN ('linear') THEN CAST('5192296858534827628530496329220095' AS INT256) / POWER(10, 18)
+                WHEN pool_type IN ('stable') THEN CAST(m.delta AS INT256) / POWER(10, 18)
                 ELSE 0
             END AS preminted_bpts
         FROM pool_labels p 
@@ -81,7 +81,7 @@ WITH pool_labels AS (
             DATE_TRUNC('day', evt_block_time) AS block_date, 
             tokenOut,
             pool_type,
-            CASE WHEN pool_type IN ('WP', 'WP2T') 
+            CASE WHEN pool_type IN ('weighted') 
             THEN 0
             ELSE SUM(amountOut / POWER(10, 18)) 
             END AS ajoins
@@ -96,7 +96,7 @@ WITH pool_labels AS (
             DATE_TRUNC('day', evt_block_time) AS block_date, 
             tokenIn,
             pool_type,
-            CASE WHEN pool_type IN ('WP', 'WP2T') 
+            CASE WHEN pool_type IN ('weighted') 
             THEN 0
             ELSE SUM(amountIn / POWER(10, 18)) 
             END AS aexits
@@ -133,8 +133,8 @@ WITH pool_labels AS (
     LEFT JOIN joins_and_exits j ON c.day = j.block_date AND b.token = j.bpt
     LEFT JOIN premints p ON b.token = p.bpt
     LEFT JOIN pool_labels l ON b.token = l.address
-    WHERE l.pool_type IN ('WP', 'WP2T', 'LBP', 'IP', 'SP', 'LP')
+    WHERE l.pool_type IN ('weighted', 'LBP', 'investment', 'stable', 'linear', 'ECLP', 'managed', 'FX')
     GROUP BY 1, 2, 3, 4, 5
     HAVING SUM(b.supply - COALESCE(preminted_bpts, 0) + COALESCE(adelta, 0)) >= 0  --simple filter to remove outliers
 
-{% endmacro %}
+{% endmacro %}

macros/models/_project/balancer/balancer_liquidity_macro.sql

@@ -55,14 +55,14 @@ WITH pool_labels AS (
     ),
 
     bpt_prices AS(
-        SELECT 
+        SELECT DISTINCT
             day,
             contract_address AS token,
+            decimals,
             bpt_price
         FROM {{ ref('balancer_bpt_prices') }}
         WHERE blockchain = '{{blockchain}}'
         AND version = '{{version}}'
-        GROUP BY 1, 2, 3
     ),
 
     eth_prices AS (
@@ -74,6 +74,15 @@ WITH pool_labels AS (
         GROUP BY 1
     ),
 
+    gyro_prices AS (
+        SELECT
+            token_address,
+            decimals,
+            price
+        FROM {{ ref('gyroscope_gyro_tokens') }}
+        WHERE blockchain = '{{blockchain}}'
+    ),
+
     swaps_changes AS (
         SELECT
             day,
@@ -197,9 +206,9 @@ WITH pool_labels AS (
             b.token,
             symbol AS token_symbol,
             cumulative_amount as token_balance_raw,
-            cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals)) AS token_balance,
-            cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals)) * COALESCE(p1.price, p2.price, 0) AS protocol_liquidity_usd,
-            cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals)) * COALESCE(p1.price, p2.price, p3.bpt_price) AS pool_liquidity_usd
+            cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals, p3.decimals, p4.decimals)) AS token_balance,
+            cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals, p3.decimals, p4.decimals)) * COALESCE(p1.price, p2.price, p4.price, 0) AS protocol_liquidity_usd,
+            cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals, p3.decimals, p4.decimals)) * COALESCE(p1.price, p2.price, p3.bpt_price, p4.price, 0) AS pool_liquidity_usd
         FROM calendar c
         LEFT JOIN cumulative_balance b ON b.day <= c.day
         AND c.day < b.day_of_next_change
@@ -212,6 +221,7 @@ WITH pool_labels AS (
         AND p2.token = b.token
         LEFT JOIN bpt_prices p3 ON p3.day = b.day 
         AND p3.token = b.token
+        LEFT JOIN gyro_prices p4 ON p4.token_address = b.token
         WHERE b.token != BYTEARRAY_SUBSTRING(b.pool_id, 1, 20)
     ),
 
@@ -232,7 +242,7 @@ WITH pool_labels AS (
         AND b.blockchain = q.chain
         LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(b.pool_id, 1, 20)
         WHERE q.name IS NOT NULL 
-        AND p.pool_type IN ('WP', 'WP2T') -- filters for weighted pools with pricing assets
+        AND p.pool_type IN ('weighted') -- filters for weighted pools with pricing assets
         AND w.blockchain = '{{blockchain}}'
         AND w.version = '{{version}}'        
         GROUP BY 1, 2, 3, 4
@@ -254,6 +264,7 @@ WITH pool_labels AS (
         p.pool_symbol,
         '{{version}}' AS version,
         '{{blockchain}}' AS blockchain,
+        p.pool_type,
         c.token AS token_address,
         c.token_symbol,
         c.token_balance_raw,

macros/models/_project/balancer/balancer_protocol_fee_macro.sql

@@ -9,10 +9,11 @@ WITH pool_labels AS (
             SELECT
                 address,
                 name,
+                pool_type,
                 ROW_NUMBER() OVER (PARTITION BY address ORDER BY MAX(updated_at) DESC) AS num
             FROM {{ ref('labels_balancer_v2_pools') }}
             WHERE blockchain = '{{blockchain}}'
-            GROUP BY 1, 2) 
+            GROUP BY 1, 2, 3) 
         WHERE num = 1
     ),
 
@@ -64,19 +65,28 @@ WITH pool_labels AS (
         WHERE (price < previous_price * 1e4 AND price > previous_price / 1e4)
     ),
 
-    bpt_prices AS(
+    bpt_prices_1 AS ( --special calculation for this spell, in order to achieve completeness without relying on prices.usd
         SELECT 
+            l.day,
+            s.token_address AS token,
+            18 AS decimals,
+            SUM(protocol_liquidity_usd / supply) AS price
+        FROM {{ ref('balancer_liquidity') }} l
+        LEFT JOIN {{ ref('balancer_bpt_supply') }} s ON s.token_address = l.pool_address 
+        AND l.blockchain = s.blockchain AND s.day = l.day AND s.supply > 0
+        WHERE l.blockchain = '{{blockchain}}'
+        AND l.version = '{{version}}'
+        GROUP BY 1, 2, 3
+    ),
+
+    bpt_prices AS (
+        SELECT  
             day,
-            contract_address AS token,
-            bpt_price AS price,
-            decimals
-        FROM {{ ref('balancer_bpt_prices') }}
-        WHERE blockchain = '{{blockchain}}'
-        AND version = '{{version}}'
-        {% if is_incremental() %}
-        AND {{ incremental_predicate('day') }}
-        {% endif %}
-        GROUP BY 1, 2, 3, 4
+            token,
+            decimals,
+            price,
+            LEAD(DAY, 1, NOW()) OVER (PARTITION BY token ORDER BY DAY) AS day_of_next_change
+        FROM bpt_prices_1
     ),
 
     daily_protocol_fee_collected AS (
@@ -129,7 +139,8 @@ WITH pool_labels AS (
             AND p2.day = d.day
         LEFT JOIN bpt_prices p3
             ON p3.token = d.token_address
-            AND p3.day = d.day
+            AND p3.day <= d.day
+            AND d.day < p3.day_of_next_change     
         LEFT JOIN {{ source('tokens', 'erc20') }} t 
             ON t.contract_address = d.token_address
             AND t.blockchain = '{{blockchain}}'
@@ -156,6 +167,7 @@ WITH pool_labels AS (
         l.name AS pool_symbol,
         '{{version}}' as version,
         '{{blockchain}}' as blockchain,
+        l.pool_type,
         f.token_address,
         f.token_symbol,
         SUM(f.token_amount_raw) as token_amount_raw,
@@ -168,6 +180,6 @@ WITH pool_labels AS (
         ON r.day = f.day
     LEFT JOIN pool_labels l
         ON BYTEARRAY_SUBSTRING(f.pool_id,1,20) = l.address
-    GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 12
+    GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 13
 
 {% endmacro %}

macros/models/_project/oneinch/AR/oneinch_ar_handle_generic.sql

@@ -5,7 +5,8 @@
         method,
         method_data,
         blockchain,
-        traces_cte
+        traces_cte,
+        start_date
     )
 %}
 
@@ -43,6 +44,8 @@ from (
     from {{ source('oneinch_' + blockchain, contract + '_call_' + method) }}
     {% if is_incremental() %}
         where {{ incremental_predicate('call_block_time') }}
+    {% else %}
+        where call_block_time >= timestamp '{{ start_date }}'
     {% endif %}
 )
 join traces_cte using(call_block_number, call_tx_hash, call_trace_address)

macros/models/_project/oneinch/AR/oneinch_ar_handle_unoswap.sql

@@ -7,7 +7,8 @@
         blockchain,
         traces_cte,
         pools_list,
-        native
+        native,
+        start_date
     )
 %}
 
@@ -48,7 +49,11 @@ select
     , call_error
     , call_type
     , ordinary
-    , transform(pools, x -> cast(x['pool'] as varbinary)) as pools
+    , transform(pools, x -> map_from_entries(array[
+        ('type', substr(cast(x['pool_type'] as varbinary), 1, 1))
+        , ('info', substr(cast(x['pool'] as varbinary), 1, 12))
+        , ('address', substr(cast(x['pool'] as varbinary), 13))
+    ])) as pools
     , remains
     , '{{ method_data.router_type }}' as router_type
 from (
@@ -96,11 +101,13 @@ from (
         join traces_cte using(call_block_number, call_tx_hash, call_trace_address)
         {% if is_incremental() %}
             where {{ incremental_predicate('call_block_time') }}
+        {% else %}
+            where call_block_time >= timestamp '{{ start_date }}'
         {% endif %}
     )
 )
-left join (select pool_address as first_pool, array[token0, token1] as first_pool_tokens from pools_list) using(first_pool) -- replace array[token0, token1] with an array with tokens when it appears in the dex.raw_pools table
-left join (select pool_address as last_pool, array[token0, token1] as last_pool_tokens from pools_list) using(last_pool) -- replace array[token0, token1] with an array with tokens when it appears in the dex.raw_pools table
+left join (select pool_address as first_pool, tokens as first_pool_tokens from pools_list) using(first_pool)
+left join (select pool_address as last_pool, tokens as last_pool_tokens from pools_list) using(last_pool)
 
 
 

macros/models/_project/oneinch/AR/oneinch_ar_macro.sql

@@ -223,11 +223,10 @@ with
 pools_list as (
     select
         pool as pool_address
-        , token0
-        , token1
+        , tokens
     from {{ ref('dex_raw_pools') }}
-    where type = 'uniswap_compatible'
-    group by 1, 2, 3
+    where type in ('uniswap_compatible', 'curve_compatible')
+    group by 1, 2
 )
 
 
@@ -269,7 +268,8 @@ pools_list as (
                         method=method,
                         method_data=method_data,
                         blockchain=blockchain,
-                        traces_cte=traces_cte
+                        traces_cte=traces_cte,
+                        start_date=contract_data['start'],
                     ) 
                 }} 
             {% elif method_data.router_type in ['unoswap'] %}
@@ -282,7 +282,8 @@ pools_list as (
                         blockchain=blockchain,
                         traces_cte=traces_cte,
                         pools_list=pools_list,
-                        native=native
+                        native=native,
+                        start_date=contract_data['start'],
                     )
                 }}
             {% endif %}