AN-4936/Backfill Migration Final (#335)

* del cw models * core views * defi views * gov view * nft_views * rm ref to old cw models * del internal cols * nv 13-15 * nv 13-15 * nv 17-18 * nv 20 * del temp logic * upd tests on allday metadata * upd to warn
FlipsideCrypto · Jul 1, 2024 · 30ce62f · 30ce62f
1 parent 3bf75df
commit 30ce62f
Show file tree

Hide file tree

Showing 83 changed files with 120 additions and 6,664 deletions.
diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml
@@ -48,7 +48,7 @@ jobs:
         run: >
           dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; 
           dbt seed; 
-          dbt run -s tag:scheduled_core tag:streamline_complete "flow_models,models/gold" --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}'
+          dbt run -s tag:scheduled_core tag:streamline_complete "flow_models,models/gold"
 
       - name: Store logs
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/dbt_run_scheduled_non_core.yml b/.github/workflows/dbt_run_scheduled_non_core.yml
@@ -44,7 +44,7 @@ jobs:
         run: >
           dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; 
           dbt seed; 
-          dbt run -s tag:scheduled_non_core --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}'
+          dbt run -s tag:scheduled_non_core
 
       - name: Store logs
         uses: actions/upload-artifact@v3

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -2,7 +2,7 @@
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
 name: "flow_models"
-version: "1.7.0"
+version: "1.8.0"
 config-version: 2
 
 require-dbt-version: ">=1.7.0"

diff --git a/models/gold/core/core__dim_contract_labels.sql b/models/gold/core/core__dim_contract_labels.sql
@@ -3,61 +3,16 @@
     tags = ['scheduled']
 ) }}
 
-WITH chainwalkers AS (
-
-    SELECT
-        NULL AS event_contract_id,
-        event_contract,
-        contract_name,
-        account_address,
-        _inserted_timestamp,
-        NULL AS inserted_timestamp,
-        NULL AS modified_timestamp
-    FROM
-        {{ ref('silver__contract_labels') }}
-),
-streamline AS (
-    SELECT
-        event_contract_id,
-        event_contract,
-        contract_name,
-        account_address,
-        _inserted_timestamp,
-        inserted_timestamp,
-        modified_timestamp
-    FROM
-        {{ ref('silver__contract_labels_s') }}
-),
-FINAL AS (
-    SELECT
-        *
-    FROM
-        chainwalkers
-    UNION ALL
-    SELECT
-        *
-    FROM
-        streamline
-)
 SELECT
+    event_contract_id,
     event_contract,
     contract_name,
     account_address,
+    inserted_timestamp,
+    modified_timestamp,
     COALESCE (
         event_contract_id,
         {{ dbt_utils.generate_surrogate_key(['event_contract']) }}
-    ) AS dim_contract_labels_id,
-    COALESCE (
-        inserted_timestamp,
-        _inserted_timestamp
-    ) AS inserted_timestamp,
-    COALESCE (
-        modified_timestamp,
-        _inserted_timestamp
-    ) AS modified_timestamp
+    ) AS dim_contract_labels_id
 FROM
-    FINAL qualify ROW_NUMBER() over (
-        PARTITION BY event_contract
-        ORDER BY
-            _inserted_timestamp DESC
-    ) = 1
+    {{ ref('silver__contract_labels_s') }}
diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql
@@ -3,68 +3,6 @@
     tags = ['ez', 'scheduled']
 ) }}
 
-WITH chainwalkers AS (
-
-    SELECT
-        NULL AS token_transfers_id,
-        block_height,
-        block_timestamp,
-        tx_id,
-        sender,
-        recipient,
-        token_contract,
-        amount,
-        tx_succeeded,
-        _inserted_timestamp,
-        NULL AS inserted_timestamp,
-        NULL AS modified_timestamp
-    FROM
-        {{ ref('silver__token_transfers') }}
-    WHERE
-        token_contract NOT IN (
-            'A.c38aea683c0c4d38.ZelosAccountingToken',
-            'A.f1b97c06745f37ad.SwapPair'
-        )
-        AND block_height < {{ var(
-            'STREAMLINE_START_BLOCK'
-        ) }}
-),
-streamline AS (
-    SELECT
-        token_transfers_id,
-        block_height,
-        block_timestamp,
-        tx_id,
-        sender,
-        recipient,
-        token_contract,
-        amount,
-        tx_succeeded,
-        _inserted_timestamp,
-        inserted_timestamp,
-        modified_timestamp
-    FROM
-        {{ ref('silver__token_transfers_s') }}
-    WHERE
-        token_contract NOT IN (
-            'A.c38aea683c0c4d38.ZelosAccountingToken',
-            'A.f1b97c06745f37ad.SwapPair'
-        )
-        AND block_height >= {{ var(
-            'STREAMLINE_START_BLOCK'
-        ) }}
-),
-FINAL AS (
-    SELECT
-        *
-    FROM
-        streamline
-    UNION ALL
-    SELECT
-        *
-    FROM
-        chainwalkers
-)
 SELECT
     block_height,
     block_timestamp,
@@ -80,13 +18,12 @@ SELECT
             ['tx_id','sender', 'recipient','token_contract', 'amount']
         ) }}
     ) AS ez_token_transfers_id,
-    COALESCE (
-        inserted_timestamp,
-        _inserted_timestamp
-    ) AS inserted_timestamp,
-    COALESCE (
-        modified_timestamp,
-        _inserted_timestamp
-    ) AS modified_timestamp
+    inserted_timestamp,
+    modified_timestamp
 FROM
-    FINAL
+    {{ ref('silver__token_transfers_s') }}
+WHERE
+    token_contract NOT IN (
+        'A.c38aea683c0c4d38.ZelosAccountingToken',
+        'A.f1b97c06745f37ad.SwapPair'
+    )
diff --git a/models/gold/core/core__ez_token_transfers.yml b/models/gold/core/core__ez_token_transfers.yml
@@ -15,7 +15,6 @@ models:
             - amount
           config:
             severity: warn
-            error_if: ">10"
 
     columns:
       - name: TX_ID

diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql
@@ -3,81 +3,21 @@
     tags = ['scheduled']
 ) }}
 
-WITH chainwalkers AS (
-
-    SELECT
-        NULL AS blocks_id,
-        block_height,
-        block_timestamp,
-        network,
-        network_version,
-        chain_id,
-        tx_count,
-        id,
-        parent_id,
-        _inserted_timestamp,
-        NULL AS inserted_timestamp,
-        NULL AS modified_timestamp
-    FROM
-        {{ ref('silver__blocks') }}
-    WHERE
-        block_height < {{ var(
-            'STREAMLINE_START_BLOCK'
-        ) }}
-),
-streamline AS (
-    SELECT
-        blocks_id,
-        block_height,
-        block_timestamp,
-        'mainnet' AS network,
-        network_version,
-        'flow' AS chain_id,
-        tx_count,
-        id,
-        parent_id,
-        _inserted_timestamp,
-        inserted_timestamp,
-        modified_timestamp
-    FROM
-        {{ ref('silver__streamline_blocks') }}
-    WHERE
-        block_height >= {{ var(
-            'STREAMLINE_START_BLOCK'
-        ) }}
-),
-FINAL AS (
-    SELECT
-        *
-    FROM
-        chainwalkers
-    UNION ALL
-    SELECT
-        *
-    FROM
-        streamline
-)
 SELECT
-    blocks_id,
-    block_height,
+    block_height :: INT AS block_height,
     block_timestamp,
-    network,
+    'mainnet' AS network,
     network_version,
-    chain_id,
+    'flow' AS chain_id,
     tx_count,
     id,
     parent_id,
     COALESCE (
         blocks_id,
         {{ dbt_utils.generate_surrogate_key(['block_height']) }}
     ) AS fact_blocks_id,
-    COALESCE (
-        inserted_timestamp,
-        _inserted_timestamp
-    ) AS inserted_timestamp,
-    COALESCE (
-        modified_timestamp,
-        _inserted_timestamp
-    ) AS modified_timestamp
+    _inserted_timestamp,
+    inserted_timestamp,
+    modified_timestamp
 FROM
-    FINAL
+    {{ ref('silver__streamline_blocks') }}
diff --git a/models/gold/core/core__fact_events.sql b/models/gold/core/core__fact_events.sql
@@ -3,64 +3,10 @@
     tags = ['scheduled']
 ) }}
 
-WITH chainwalkers AS (
-
-    SELECT
-        NULL AS streamline_event_id,
-        tx_id,
-        block_timestamp,
-        block_height,
-        tx_succeeded,
-        event_index,
-        event_contract,
-        event_type,
-        event_data,
-        _inserted_timestamp,
-        NULL AS inserted_timestamp,
-        NULL AS modified_timestamp
-    FROM
-        {{ ref('silver__events_final') }}
-    WHERE
-        block_height < {{ var(
-            'STREAMLINE_START_BLOCK'
-        ) }}
-),
-streamline AS (
-    SELECT
-        streamline_event_id,
-        tx_id,
-        block_timestamp,
-        block_height,
-        tx_succeeded,
-        event_index,
-        event_contract,
-        event_type,
-        event_data,
-        _inserted_timestamp,
-        inserted_timestamp,
-        modified_timestamp
-    FROM
-        {{ ref('silver__streamline_events') }}
-    WHERE
-        block_height >= {{ var(
-            'STREAMLINE_START_BLOCK'
-        ) }}
-),
-FINAL AS (
-    SELECT
-        *
-    FROM
-        chainwalkers
-    UNION ALL
-    SELECT
-        *
-    FROM
-        streamline
-)
 SELECT
     tx_id,
     block_timestamp,
-    block_height,
+    block_height :: INT AS block_height,
     tx_succeeded,
     event_index,
     event_contract,
@@ -70,13 +16,8 @@ SELECT
         streamline_event_id,
         {{ dbt_utils.generate_surrogate_key(['tx_id']) }}
     ) AS fact_events_id,
-    COALESCE (
-        inserted_timestamp,
-        _inserted_timestamp
-    ) AS inserted_timestamp,
-    COALESCE (
-        modified_timestamp,
-        _inserted_timestamp
-    ) AS modified_timestamp
+    _inserted_timestamp,
+    inserted_timestamp,
+    modified_timestamp
 FROM
-    FINAL
+    {{ ref('silver__streamline_events') }}