Skip to content

Commit

Permalink
Merge branch 'main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
Hosuke authored Dec 18, 2023
2 parents c088a05 + 4e882dc commit 10f9491
Show file tree
Hide file tree
Showing 686 changed files with 49,482 additions and 9,575 deletions.
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ Welcome to [Spellbook](https://youtu.be/o7p0BNt7NHs). Cast a magical incantation
- Don't know where to start? The docs below will guide you, but as a summary:
- Want to make an incremental improvement to one of our spells? (add a new project, fix a bug you found), simply open a PR with your changes.
- Follow the guide for [Submitting a PR](), [Setting up your dev environment]() and [Using dbt to write spells]() if you find yourself lost.
- Not sure how to start? Follow the walkthrough [here](https://dune.com/docs/data-tables/spellbook/contributing/).
- Not sure how to start? Follow the walkthrough [here](https://dune.com/docs/spellbook/).
- Make sure to open a draft PR if you will work on your spell for longer than a few days, to avoid duplicated work
- Do you want to get started building spells and you don't know what to build? Check [Issues]() to see what the community needs.
- Check the Discussions section to see what problems the community is trying to solve (i.e. non-incremental changes) or propose your own!
Expand Down Expand Up @@ -45,7 +45,7 @@ Spellbook is built for and by the community, you are welcome to close any gaps t
- **Propose** changes to spellbook - [Discussions](https://github.com/duneanalytics/spellbook/discussions) are where we bring up, challenge and develop ideas to continue building spellbook. If you want to make a major change to a spell (e.g. major overhaul to a sector, launching a new sector, designing a new organic volume filter, etc.).

### Submitting a PR
Want to get right to work? Follow the guide [here](https://dune.com/docs/data-tables/spellbook/contributing/#7-steps-to-adding-a-spell) to get started.
Want to get right to work? Follow the guide [here](https://dune.com/docs/spellbook/?h=7+steps+adding+a+spell) to get started.

### Testing your spell
You don't need a complex local setup to test spells against Dune's engine. Once you send a PR, our CI pipeline will run and test it, and, if the job finishes successfully, you'll be able to query the data your PR created directly from dune.com.
Expand Down Expand Up @@ -94,7 +94,7 @@ pipenv install
If the install fails, one likely reason is our script looks for a static python version and the likelihood of an error for a wrong python version is pretty high. If that error occurs, check your python version with:

```console
py --version
python --version
```

Now use any text editor program to change the python version in the pipfile within the spellbook directory to your python version. You need to have at least python 3.9.
Expand Down
2 changes: 2 additions & 0 deletions dbt_project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -808,6 +808,8 @@ models:
+schema: chainlink_arbitrum
avalanche_c:
+schema: chainlink_avalanche_c
base:
+schema: chainlink_base
bnb:
+schema: chainlink_bnb
ethereum:
Expand Down
168 changes: 168 additions & 0 deletions macros/models/_project/balancer/balancer_protocol_fee_macro.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
{% macro
balancer_protocol_fee_macro(
blockchain, version
)
%}

WITH pool_labels AS (
SELECT * FROM (
SELECT
address,
name,
ROW_NUMBER() OVER (PARTITION BY address ORDER BY MAX(updated_at) DESC) AS num
FROM {{ ref('labels_balancer_v2_pools') }}
WHERE blockchain = '{{blockchain}}'
GROUP BY 1, 2)
WHERE num = 1
),

prices AS (
SELECT
date_trunc('day', minute) AS day,
contract_address AS token,
decimals,
AVG(price) AS price
FROM {{ source('prices', 'usd') }}
WHERE blockchain = '{{blockchain}}'
{% if is_incremental() %}
AND {{ incremental_predicate('minute') }}
{% endif %}
GROUP BY 1, 2, 3

),

dex_prices_1 AS (
SELECT
date_trunc('day', hour) AS DAY,
contract_address AS token,
approx_percentile(median_price, 0.5) AS price,
sum(sample_size) AS sample_size
FROM {{ ref('dex_prices') }}
{% if is_incremental() %}
WHERE {{ incremental_predicate('hour') }}
{% endif %}
GROUP BY 1, 2
HAVING sum(sample_size) > 3
),

dex_prices AS (
SELECT
*,
LEAD(DAY, 1, NOW()) OVER (
PARTITION BY token
ORDER BY
DAY
) AS day_of_next_change
FROM dex_prices_1
),

bpt_prices AS(
SELECT
date_trunc('day', hour) AS day,
contract_address AS token,
approx_percentile(median_price, 0.5) AS price
FROM {{ ref('balancer_bpt_prices') }}
WHERE blockchain = '{{blockchain}}'
{% if is_incremental() %}
AND {{ incremental_predicate('hour') }}
{% endif %}
GROUP BY 1, 2
),

daily_protocol_fee_collected AS (
SELECT
date_trunc('day', evt_block_time) AS day,
poolId AS pool_id,
token AS token_address,
SUM(protocol_fees) AS protocol_fee_amount_raw
FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_PoolBalanceChanged') }} b
CROSS JOIN unnest("protocolFeeAmounts", "tokens") AS t(protocol_fees, token)
{% if is_incremental() %}
WHERE {{ incremental_predicate('b.evt_block_time') }}
{% endif %}
GROUP BY 1, 2, 3

UNION ALL

SELECT
date_trunc('day', t.evt_block_time) AS day,
poolId AS pool_id,
b.poolAddress AS token_address,
sum(value) AS protocol_fee_amount_raw
FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_PoolRegistered') }} b
INNER JOIN {{ source('erc20_' + blockchain, 'evt_transfer') }} t
ON t.contract_address = b.poolAddress
AND t."from" = 0x0000000000000000000000000000000000000000
AND t.to = 0xce88686553686DA562CE7Cea497CE749DA109f9F --ProtocolFeesCollector address, which is the same across all chains
{% if is_incremental() %}
WHERE {{ incremental_predicate('t.evt_block_time') }}
AND {{ incremental_predicate('b.evt_block_time') }}
{% endif %}
GROUP BY 1, 2, 3
),

decorated_protocol_fee AS (
SELECT
d.day,
d.pool_id,
d.token_address,
t.symbol AS token_symbol,
SUM(d.protocol_fee_amount_raw) AS token_amount_raw,
SUM(d.protocol_fee_amount_raw / power(10, COALESCE(t.decimals,p1.decimals))) AS token_amount,
CASE
WHEN BYTEARRAY_SUBSTRING(d.pool_id, 1, 20) = d.token_address -- fees paid in BPTs
THEN 0
ELSE
SUM(COALESCE(p1.price, p2.price, p3.price) * protocol_fee_amount_raw / POWER(10, COALESCE(t.decimals,p1.decimals)))
END AS protocol_fee_collected_usd
FROM daily_protocol_fee_collected d
LEFT JOIN prices p1
ON p1.token = d.token_address
AND p1.day = d.day
LEFT JOIN dex_prices p2
ON p2.token = d.token_address
AND p2.day = d.day
LEFT JOIN bpt_prices p3
ON p3.token = CAST(d.token_address AS VARCHAR)
AND p3.day = d.day
LEFT JOIN {{ ref('tokens_erc20') }} t
ON t.contract_address = d.token_address
AND t.blockchain = '{{blockchain}}'
GROUP BY 1, 2, 3, 4
),

revenue_share as(
SELECT
day,
CASE
WHEN day < DATE '2022-07-03' THEN 0.25 -- veBAL release
WHEN day >= DATE '2022-07-03' AND day < DATE '2023-01-23' THEN 0.25 -- BIP 19
WHEN day >= DATE '2023-01-23' AND day < DATE '2023-07-24' THEN 0.35 -- BIP 161
WHEN day >= DATE '2023-07-24' THEN 0.175 -- BIP 371
END AS treasury_share
FROM UNNEST(SEQUENCE(DATE '2022-03-01', CURRENT_DATE, INTERVAL '1' DAY)) AS date(day)
)


SELECT
f.day,
f.pool_id,
BYTEARRAY_SUBSTRING(f.pool_id,1,20) as pool_address,
l.name AS pool_symbol,
'{{version}}' as version,
'{{blockchain}}' as blockchain,
f.token_address,
f.token_symbol,
SUM(f.token_amount_raw) as token_amount_raw,
SUM(f.token_amount) as token_amount,
SUM(f.protocol_fee_collected_usd) as protocol_fee_collected_usd,
r.treasury_share,
SUM(f.protocol_fee_collected_usd) * r.treasury_share as treasury_revenue_usd
FROM decorated_protocol_fee f
INNER JOIN revenue_share r
ON r.day = f.day
LEFT JOIN pool_labels l
ON BYTEARRAY_SUBSTRING(f.pool_id,1,20) = l.address
GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 12

{% endmacro %}
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ select
, {{ method_data.get("dst_amount_min", "null") }} as dst_amount_min
, call_gas_used
, call_output
, call_error
, null as ordinary
, null as pools
, remains
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ select
, dst_amount_min
, call_gas_used
, call_output
, call_error
, ordinary
, pools
, remains
Expand All @@ -63,6 +64,7 @@ from (
, {{ method_data.get("dst_amount_min", "null") }} as dst_amount_min
, call_gas_used
, call_output
, call_error
, if(cardinality(call_pools) > 0, true, false) as ordinary -- if call_pools is not empty
, if(cardinality(call_pools) > 0
, try(substr(cast(call_pools[1] as varbinary), 13)) -- get first pool from call_pools
Expand Down
33 changes: 21 additions & 12 deletions macros/models/_project/oneinch/AR/oneinch_ar_macro.sql
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@
},
"AggregationRouterV5": {
"version": "5",
"blockchains": ["ethereum", "bnb", "polygon", "arbitrum", "optimism", "avalanche_c", "gnosis", "fantom", "base"],
"blockchains": ["ethereum", "bnb", "polygon", "arbitrum", "optimism", "avalanche_c", "gnosis", "fantom", "base", "zksync"],
"start": "2022-11-04",
"methods": {
"swap": samples["swap_2"],
Expand Down Expand Up @@ -237,6 +237,7 @@ pools_list as (
, length(input) as call_input_length
, substr(input, length(input) - mod(length(input) - 4, 32) + 1) as remains
, output as call_output
, error as call_error
from {{ source(blockchain, 'traces') }}
where
{% if is_incremental() %}
Expand Down Expand Up @@ -288,38 +289,46 @@ select
, tx_from
, tx_to
, tx_success
, tx_nonce
, tx_gas_price as gas_price
, tx_priority_fee_per_gas as priority_fee_per_gas
, contract_name
, 'AR' as protocol
, protocol_version
, method
, call_selector
, call_trace_address
, call_from
, call_to
, call_trace_address
, call_selector
, src_token_address
, dst_token_address
, call_success
, call_gas_used
, call_output
, call_error
, src_receiver
, dst_receiver
, src_amount
, dst_amount
, dst_amount_min
, src_token_address
, dst_token_address
, src_amount -- will be removed soon
, dst_amount -- will be removed soon
, dst_amount_min -- will be removed soon
, src_amount as src_token_amount
, dst_amount as dst_token_amount
, dst_amount_min as dst_token_amount_min
, ordinary
, pools
, router_type
, call_success
, call_gas_used
, concat(cast(length(remains) as bigint), if(length(remains) > 0
, transform(sequence(1, length(remains), 4), x -> bytearray_to_bigint(reverse(substr(reverse(remains), x, 4))))
, array[bigint '0']
)) as remains
, call_output
, date_trunc('minute', block_time) as minute
, date(date_trunc('month', block_time)) as block_month
from (
{{
add_tx_columns(
model_cte = 'calls'
, blockchain = blockchain
, columns = ['from', 'to', 'success']
, columns = ['from', 'to', 'success', 'nonce', 'gas_price', 'priority_fee_per_gas']
)
}}
)
Expand Down
Loading

0 comments on commit 10f9491

Please sign in to comment.