transfers (#91)

FlipsideCrypto · Oct 24, 2023 · 06a7879 · 06a7879
1 parent 276725f
commit 06a7879
Show file tree

Hide file tree

Showing 7 changed files with 316 additions and 74 deletions.
diff --git a/.github/workflows/dbt_run_heal_models.yml b/.github/workflows/dbt_run_heal_models.yml
@@ -0,0 +1,45 @@
+name: dbt_run_heal_models
+run-name: dbt_run_heal_models
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs at 04:55 on Wednesday (see https://crontab.guru)
+    - cron: '55 4 * * 3'
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -m tag:heal --var '{"HEAL_MODEL":True}'
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -63,4 +63,5 @@ vars:
   STREAMLINE_INVOKE_STREAMS: False
   STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False
   WAIT: 0
-  OBSERV_FULL_TEST: False
+  OBSERV_FULL_TEST: False
+  HEAL_MODEL: False
diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql
@@ -8,47 +8,24 @@ SELECT
     block_number,
     block_timestamp,
     tx_hash,
+    event_index,
     origin_function_signature,
     origin_from_address,
     origin_to_address,
-    t.contract_address,
+    contract_address,
     from_address,
     to_address,
-    raw_amount,
     raw_amount_precise,
-    C.token_decimals AS decimals,
-    C.token_symbol AS symbol,
-    price AS token_price,
-    CASE
-        WHEN C.token_decimals IS NOT NULL THEN raw_amount / pow(
-            10,
-            C.token_decimals
-        )
-        ELSE NULL
-    END AS amount,
-    CASE
-        WHEN C.token_decimals IS NOT NULL
-        AND price IS NOT NULL THEN amount * price
-        ELSE NULL
-    END AS amount_usd,
-    CASE
-        WHEN C.token_decimals IS NULL THEN 'false'
-        ELSE 'true'
-    END AS has_decimal,
-    CASE
-        WHEN price IS NULL THEN 'false'
-        ELSE 'true'
-    END AS has_price,
-    _log_id
+    raw_amount,
+    amount_precise,
+    amount,
+    amount_usd,
+    decimals,
+    symbol,
+    token_price,
+    has_decimal,
+    has_price,
+    _log_id,
+    _inserted_timestamp
 FROM
-    {{ ref('core__fact_token_transfers') }}
-    t
-    LEFT JOIN {{ ref('price__ez_hourly_token_prices') }}
-    p
-    ON t.contract_address = p.token_address
-    AND DATE_TRUNC(
-        'hour',
-        t.block_timestamp
-    ) = HOUR
-    LEFT JOIN {{ ref('silver__contracts') }} C
-    ON t.contract_address = C.contract_address
+    {{ ref('silver__transfers') }}
diff --git a/models/gold/core/core__ez_token_transfers.yml b/models/gold/core/core__ez_token_transfers.yml
@@ -10,35 +10,41 @@ models:
         description: '{{ doc("gno_block_timestamp") }}'
       - name: TX_HASH
         description: '{{ doc("gno_transfer_tx_hash") }}'
+      - name: EVENT_INDEX
+        description: '{{ doc("gno_event_index") }}'
+      - name: ORIGIN_FUNCTION_SIGNATURE
+        description: '{{ doc("gno_origin_sig") }}'
+      - name: ORIGIN_FROM_ADDRESS
+        description: '{{ doc("gno_origin_from") }}'
+      - name: ORIGIN_TO_ADDRESS
+        description: '{{ doc("gno_origin_to") }}'
       - name: CONTRACT_ADDRESS
         description: '{{ doc("gno_transfer_contract_address") }}'
       - name: FROM_ADDRESS
         description: '{{ doc("gno_transfer_from_address") }}'
       - name: TO_ADDRESS
         description: '{{ doc("gno_transfer_to_address") }}'
+      - name: RAW_AMOUNT_PRECISE
+        description: '{{ doc("gno_transfer_raw_amount_precise") }}'
       - name: RAW_AMOUNT
         description: '{{ doc("gno_transfer_raw_amount") }}'
-      - name: RAW_AMOUNT_PRECISE
-        description: '{{ doc("precise_amount_unadjusted") }}'
-      - name: DECIMALS
-        description: 'The number of decimal places this contract needs adjusted where token values exist.'
-      - name: SYMBOL
-        description: 'The symbol belonging to the address of the token.'
+      - name: AMOUNT_PRECISE
+        description: '{{ doc("gno_transfer_amount_precise") }}'
       - name: TOKEN_PRICE
         description: '{{ doc("gno_transfer_token_price") }}'
       - name: AMOUNT
         description: '{{ doc("gno_transfer_amount") }}'
       - name: AMOUNT_USD
         description: '{{ doc("gno_transfer_amount_usd") }}'
+      - name: DECIMALS
+        description: '{{ doc("gno_decimals") }}'
+      - name: SYMBOL
+        description: '{{ doc("gno_symbol") }}'
+      - name: TOKEN_PRICE
+        description: '{{ doc("gno_transfer_token_price") }}'
       - name: HAS_DECIMAL
         description: '{{ doc("gno_transfer_has_decimal") }}'
       - name: HAS_PRICE
         description: '{{ doc("gno_transfer_has_price") }}'
       - name: _LOG_ID
-        description: '{{ doc("gno_log_id_transfers") }}'
-      - name: ORIGIN_FUNCTION_SIGNATURE
-        description: '{{ doc("gno_origin_sig") }}'
-      - name: ORIGIN_FROM_ADDRESS
-        description: '{{ doc("gno_eth_origin_from") }}'
-      - name: ORIGIN_TO_ADDRESS
-        description: '{{ doc("gno_eth_origin_to") }}'
+        description: '{{ doc("gno_log_id_transfers") }}'
diff --git a/models/gold/core/core__fact_token_transfers.sql b/models/gold/core/core__fact_token_transfers.sql
@@ -8,6 +8,7 @@ SELECT
     block_number,
     block_timestamp,
     tx_hash,
+    event_index,
     origin_function_signature,
     origin_from_address,
     origin_to_address,

diff --git a/models/gold/core/core__fact_token_transfers.yml b/models/gold/core/core__fact_token_transfers.yml
@@ -10,6 +10,14 @@ models:
         description: '{{ doc("gno_block_timestamp") }}'
       - name: TX_HASH
         description: '{{ doc("gno_transfer_tx_hash") }}'
+      - name: EVENT_INDEX
+        description: '{{ doc("gno_event_index") }}'
+      - name: ORIGIN_FUNCTION_SIGNATURE
+        description: '{{ doc("gno_origin_sig") }}'
+      - name: ORIGIN_FROM_ADDRESS
+        description: '{{ doc("gno_origin_from") }}'
+      - name: ORIGIN_TO_ADDRESS
+        description: '{{ doc("gno_origin_to") }}'
       - name: CONTRACT_ADDRESS
         description: '{{ doc("gno_transfer_contract_address") }}'
       - name: FROM_ADDRESS
@@ -19,12 +27,6 @@ models:
       - name: RAW_AMOUNT
         description: '{{ doc("gno_transfer_raw_amount") }}'
       - name: RAW_AMOUNT_PRECISE
-        description: '{{ doc("precise_amount_unadjusted") }}'
+        description: '{{ doc("gno_transfer_raw_amount_precise") }}'
       - name: _LOG_ID
-        description: '{{ doc("gno_log_id_transfers") }}'
-      - name: ORIGIN_FUNCTION_SIGNATURE
-        description: '{{ doc("gno_origin_sig") }}'
-      - name: ORIGIN_FROM_ADDRESS
-        description: '{{ doc("gno_eth_origin_from") }}'
-      - name: ORIGIN_TO_ADDRESS
-        description: '{{ doc("gno_eth_origin_to") }}'
+        description: '{{ doc("gno_log_id_transfers") }}'