Merge pull request #76 from FlipsideCrypto/AN-4285/revalidate-staking…

…-and-gov AN-4285/revalidate-staking-and-gov
FlipsideCrypto · Jan 29, 2024 · 83a6b8e · 83a6b8e
2 parents 614c1c9 + 7c8083c
commit 83a6b8e
Show file tree

Hide file tree

Showing 20 changed files with 1,038 additions and 527 deletions.
diff --git a/models/bronze/bronze_api/bronze_api__get_validator_metadata.sql b/models/bronze/bronze_api/bronze_api__get_validator_metadata.sql
@@ -4,12 +4,29 @@
 
 WITH call AS (
     SELECT
-        ethereum.streamline.udf_api(
+        live.udf_api(
             'POST',
             'https://api.axelarscan.io',{},{'path':'/cosmos/staking/v1beta1/validators','module':'lcd'}
         ) AS resp,
         SYSDATE() AS _inserted_timestamp
+),
+call_2 AS (
+    SELECT
+        live.udf_api(
+            'POST',
+            'https://api.axelarscan.io',{},{'path':'/cosmos/staking/v1beta1/validators?pagination.key=FMrVj6JowYsKRr2zLFCWofPkvPYV','module':'lcd'}
+        ) AS resp,
+        SYSDATE() AS _inserted_timestamp
 )
+SELECT
+    i.value :operator_address :: STRING AS address,
+    i.value AS data,
+    _inserted_timestamp
+FROM call_2, 
+LATERAL FLATTEN(
+    input => resp :data :validators
+) i
+UNION ALL
 SELECT
     i.value :operator_address :: STRING AS address,
     i.value AS data,

diff --git a/models/gold/gov/gov__fact_staking.sql b/models/gold/gov/gov__fact_staking.sql
@@ -18,10 +18,10 @@ SELECT
     redelegate_source_validator_address,
     completion_time,
     msg_group AS _msg_group,
-    COALESCE(
+    COALESCE (
         staking_id,
         {{ dbt_utils.generate_surrogate_key(
-            ['tx_id','msg_group','action','currency','delegator_address','validator_address']
+            ['tx_id','msg_index']
         ) }}
     ) AS fact_staking_id,
     COALESCE(

diff --git a/models/silver/core/balance/silver__balances_staked.sql b/models/silver/core/balance/silver__balances_staked.sql
@@ -0,0 +1,31 @@
+{{ config(
+    materialized = 'view',
+    tags = ['daily']
+) }}
+
+SELECT
+    block_id,
+    block_timestamp,
+    tx_id,
+    msg_group,
+    delegator_address AS address,
+    CASE
+        WHEN action = 'delegate' THEN amount
+        ELSE - amount
+    END AS amount,
+    SUM(
+        CASE
+            WHEN action = 'delegate' THEN amount
+            ELSE - amount
+        END
+    ) over(
+        PARTITION BY address,
+        currency
+        ORDER BY
+            block_timestamp,
+            msg_group rows unbounded preceding
+    ) AS balance,
+    currency,
+    _inserted_timestamp
+FROM
+    {{ ref('silver__staking') }}
diff --git a/models/silver/core/balance/silver__balances_staked.yml b/models/silver/core/balance/silver__balances_staked.yml
@@ -0,0 +1,55 @@
+version: 2
+models:
+  - name: silver__balances_staked
+    description: A table of each staked balance change for wallets on the Axelar chain. 
+    columns:
+      - name: BLOCK_ID 
+        description: "{{ doc('block_id') }}"
+        tests: 
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests: 
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+      - name: ADDRESS
+        description: "{{ doc('address') }}"
+        tests: 
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: axelar[0-9a-z]{38,38}
+      - name: BALANCE 
+        description: The amount of the currency the wallet held at the given time
+        tests: 
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: CURRENCY
+        description: "{{ doc('currency') }}"
+        tests: 
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('inserted_timestamp') }}"
+        tests: 
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+                - TIMESTAMP_LTZ
+
diff --git a/models/silver/core/balance/silver__daily_balances_staked.sql b/models/silver/core/balance/silver__daily_balances_staked.sql
@@ -0,0 +1,185 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = ['date', 'address', 'currency'],
+    incremental_strategy = 'merge',
+    merge_exclude_columns = ["inserted_timestamp"],
+    cluster_by = ['DATE'],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(address);",
+    tags = ['daily']
+) }}
+
+WITH all_staked AS (
+
+    SELECT
+        block_id,
+        block_timestamp,
+        address,
+        balance,
+        currency
+    FROM
+        {{ ref('silver__balances_staked') }}
+)
+
+{% if is_incremental() %},
+recent AS (
+    SELECT
+        DATE,
+        address,
+        balance,
+        currency
+    FROM
+        {{ this }}
+    WHERE
+        DATE = (
+            SELECT
+                DATEADD('day', -1, MAX(DATE))
+            FROM
+                {{ this }})
+        ),
+        NEW AS (
+            SELECT
+                block_timestamp :: DATE AS DATE,
+                address,
+                balance,
+                currency,
+                1 AS RANK
+            FROM
+                all_staked
+            WHERE
+                block_timestamp :: DATE >= (
+                    SELECT
+                        DATEADD('day', -1, MAX(DATE))
+                    FROM
+                        {{ this }}) qualify(ROW_NUMBER() over (PARTITION BY block_timestamp :: DATE, address, currency
+                    ORDER BY
+                        block_timestamp DESC)) = 1
+                ),
+                incremental AS (
+                    SELECT
+                        DATE,
+                        address,
+                        balance,
+                        currency
+                    FROM
+                        (
+                            SELECT
+                                DATE,
+                                address,
+                                balance,
+                                currency,
+                                2 AS RANK
+                            FROM
+                                recent
+                            UNION
+                            SELECT
+                                DATE,
+                                address,
+                                balance,
+                                currency,
+                                1 AS RANK
+                            FROM
+                                NEW
+                        ) qualify(ROW_NUMBER() over (PARTITION BY DATE, address, currency
+                    ORDER BY
+                        RANK ASC)) = 1
+                )
+            {% endif %},
+            base AS (
+
+{% if is_incremental() %}
+SELECT
+    DATE AS block_timestamp, address, balance, currency
+FROM
+    incremental
+{% else %}
+SELECT
+    block_timestamp, address, balance, currency
+FROM
+    all_staked
+{% endif %}),
+address_ranges AS (
+    SELECT
+        address,
+        currency,
+        MIN(
+            block_timestamp :: DATE
+        ) AS min_block_date,
+        MAX (
+            CURRENT_TIMESTAMP :: DATE
+        ) AS max_block_date
+    FROM
+        base
+    GROUP BY
+        address,
+        currency
+),
+ddate AS (
+    SELECT
+        date_day :: DATE AS DATE
+    FROM
+        {{ source(
+            'crosschain',
+            'dim_dates'
+        ) }}
+    GROUP BY
+        DATE
+),
+all_dates AS (
+    SELECT
+        d.date,
+        A.address,
+        A.currency
+    FROM
+        ddate d
+        LEFT JOIN address_ranges A
+        ON d.date BETWEEN A.min_block_date
+        AND A.max_block_date
+    WHERE
+        A.address IS NOT NULL
+),
+sei_balances AS (
+    SELECT
+        block_timestamp,
+        address,
+        balance,
+        currency
+    FROM
+        base qualify(ROW_NUMBER() over (PARTITION BY block_timestamp :: DATE, address, currency
+    ORDER BY
+        block_timestamp DESC)) = 1
+),
+balance_temp AS (
+    SELECT
+        d.date,
+        d.address,
+        b.balance,
+        d.currency
+    FROM
+        all_dates d
+        LEFT JOIN sei_balances b
+        ON d.date = b.block_timestamp :: DATE
+        AND d.address = b.address
+        AND d.currency = b.currency
+)
+SELECT
+    DATE,
+    'staked' AS balance_type,
+    address,
+    currency,
+    LAST_VALUE(
+        balance ignore nulls
+    ) over(
+        PARTITION BY address,
+        currency,
+        balance_type
+        ORDER BY
+            DATE ASC rows unbounded preceding
+    ) AS balance,
+    {{ dbt_utils.generate_surrogate_key(
+        ['address','currency','date']
+    ) }} AS daily_balances_staked_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    balance_temp
diff --git a/models/silver/core/balance/silver__daily_balances_staked.yml b/models/silver/core/balance/silver__daily_balances_staked.yml
@@ -0,0 +1,53 @@
+version: 2
+models:
+  - name: silver__daily_balances_staked
+    description: A table that contains a daily balance entry for both staked and liquid balance of wallets on the Axelar chain. 
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - DATE
+            - ADDRESS
+            - BALANCE_TYPE
+            - CURRENCY
+    columns:
+      - name: DATE
+        description: The day the balance was recorded on. 
+        tests: 
+          - not_null
+      - name: BALANCE_TYPE
+        description: Either "staked" or "liquid" corresponding to how the currency is in the wallet. 
+        tests: 
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+          - dbt_expectations.expect_column_values_to_be_in_set:
+              value_set: ['staked', 'liquid', 'locked liquidity', 'superfluid staked']
+      - name: ADDRESS
+        description: "{{ doc('address') }}"
+        tests: 
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: axelar[0-9a-z]{39,39}
+      - name: BALANCE 
+        description: The amount of the currency the wallet held at the given time
+        tests: 
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: CURRENCY
+        description: "{{ doc('currency') }}"
+        tests: 
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+