AN-4242/abi-decoded-logs2 (#214)

* abi pipeline + decodedlogs2

* typo

* range

.github/workflows/dbt_run_streamline_temp_decoder_2.yml

@@ -0,0 +1,45 @@
+name: dbt_run_streamline_temp_decoder2
+run-name: dbt_run_streamline_temp_decoder2
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs "every 1 hours" (see https://crontab.guru)
+    - cron: '5 */1 * * *'
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod_backfill
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -m "optimism_models,tag:decoded_logs2"

models/gold/core/core__dim_contract_abis.sql

@@ -0,0 +1,16 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    contract_address,
+    DATA AS abi,
+    abi_source,
+    bytecode,
+    abis_id AS dim_contract_abis_id,
+    inserted_timestamp,
+    modified_timestamp
+FROM
+    {{ ref('silver__abis') }}

models/gold/core/core__dim_contract_abis.yml

@@ -0,0 +1,19 @@
+version: 2
+models:
+  - name: core__dim_contract_abis
+    description: >
+      'This table contains the contract ABIs that we have sourced from Optimistic-Etherscan, the community, or bytecode matched. This table is the source of ABIs used in the `core__ez_decoded_event_logs` and `core__fact_decoded_event_logs` tables.
+      We first try to source ABIs from Optimistic-Etherscan. If we cannot find an ABI on Optimistic-Etherscan, we will rely on user submissions. To add a contract to this table, please visit [here](https://science.flipsidecrypto.xyz/abi-requestor/).
+      If we are unable to locate an ABI for a contract from Optimistic-Etherscan or the community, we will try to find an ABI to use by matching the contract bytecode to a known contract bytecode we do have an ABI for.' 
+       
+    columns:
+      - name: CONTRACT_ADDRESS
+        description: 'The address of the contract.'
+      - name: ABI
+        description: 'The JSON ABI for the contract.'
+      - name: ABI_SOURCE
+        description: 'The source of the ABI. This can be `Optimistic-Etherscan`, `user_submitted`, or `bytecode_matched`.'
+      - name: BYTECODE
+        description: 'The deployed bytecode of the contract.'
+
+

models/silver/abis/silver__abis.sql

@@ -1,6 +1,7 @@
 {{ config (
     materialized = "incremental",
     unique_key = "contract_address",
+    merge_exclude_columns = ["inserted_timestamp"],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(contract_address)",
     tags = ['abis']
 ) }}
@@ -17,6 +18,8 @@ WITH override_abis AS (
         1 AS priority
     FROM
         {{ ref('silver__override_abis') }}
+    WHERE
+        contract_address IS NOT NULL
 ),
 verified_abis AS (
     SELECT
@@ -62,12 +65,15 @@ user_abis AS (
 {% if is_incremental() %}
 AND _inserted_timestamp >= (
     SELECT
-        COALESCE(MAX(_inserted_timestamp), '1970-01-01' :: TIMESTAMP)
+        MAX(
+            _inserted_timestamp
+        )
     FROM
         {{ this }}
     WHERE
-        abi_source = 'user')
-    {% endif %}
+        abi_source = 'user'
+)
+{% endif %}
 ),
 bytecode_abis AS (
     SELECT
@@ -81,7 +87,7 @@ bytecode_abis AS (
     FROM
         {{ ref('silver__bytecode_abis') }}
     WHERE
-        NOT bytecode_dupe
+        1 = 1
 
 {% if is_incremental() %}
 AND _inserted_timestamp >= (
@@ -162,10 +168,14 @@ SELECT
     p.abi_source,
     p.discord_username,
     p.abi_hash,
-    created_contract_input AS bytecode
+    created_contract_input AS bytecode,
+    {{ dbt_utils.generate_surrogate_key(
+        ['contract_address']
+    ) }} AS abis_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     priority_abis p
     LEFT JOIN {{ ref('silver__created_contracts') }}
     ON p.contract_address = created_contract_address
-WHERE
-    p.contract_address IS NOT NULL

models/silver/abis/silver__bytecode_abis.sql

@@ -1,20 +1,33 @@
 {{ config (
     materialized = "incremental",
-    unique_key = "abi_id",
+    unique_key = "contract_address",
     tags = ['abis']
 ) }}
 
-WITH bytecodes AS (
+WITH contracts_with_abis AS (
+    -- Identifying contracts with verified ABIs
 
     SELECT
-        created_contract_address AS contract_address,
-        A.data AS abi,
-        created_contract_input AS bytecode,
-        abi_hash
+        created_contract_address AS contract_address
     FROM
         {{ ref('silver__created_contracts') }}
-        LEFT JOIN {{ ref('silver__verified_abis') }} A
+        JOIN {{ ref('silver__verified_abis') }} A
         ON A.contract_address = created_contract_address
+),
+contracts_without_abis AS (
+    -- Contracts that are missing ABIs
+    SELECT
+        created_contract_address AS contract_address,
+        created_contract_input AS bytecode
+    FROM
+        {{ ref('silver__created_contracts') }}
+    WHERE
+        created_contract_address NOT IN (
+            SELECT
+                contract_address
+            FROM
+                contracts_with_abis
+        )
 
 {% if is_incremental() %}
 AND created_contract_address NOT IN (
@@ -25,63 +38,39 @@ AND created_contract_address NOT IN (
 )
 {% endif %}
 ),
-contracts_missing_abis AS (
-    SELECT
-        contract_address,
-        bytecode
-    FROM
-        bytecodes
-    WHERE
-        abi_hash IS NULL
-),
-bytecode_abis AS (
-    SELECT
-        *,
-        ROW_NUMBER() over(
-            PARTITION BY bytecode
-            ORDER BY
-                abi_length DESC
-        ) AS abi_row_no
-    FROM
-        (
-            SELECT
-                DISTINCT bytecode,
-                abi,
-                abi_hash,
-                LENGTH(abi) AS abi_length
-            FROM
-                bytecodes
-            WHERE
-                abi_hash IS NOT NULL
-        )
-),
-dupe_check AS (
+unique_bytecode_abis AS (
+    -- Bytecodes from created_contracts with a unique ABI
     SELECT
-        bytecode,
-        COUNT(*) AS num_abis
+        cc.created_contract_input AS bytecode,
+        va.data AS abi,
+        va.abi_hash
     FROM
-        bytecode_abis
+        {{ ref('silver__created_contracts') }}
+        cc
+        JOIN {{ ref('silver__verified_abis') }}
+        va
+        ON cc.created_contract_address = va.contract_address
     GROUP BY
-        bytecode
+        cc.created_contract_input,
+        va.data,
+        va.abi_hash
     HAVING
-        COUNT(*) > 1
-)
+        COUNT(
+            DISTINCT va.data
+        ) = 1 -- Ensuring there's only one ABI per bytecode
+) -- Final matching
 SELECT
     contract_address,
     abi,
     abi_hash,
-    SYSDATE() AS _inserted_timestamp,
-    abi_row_no,
-    CONCAT(
-        contract_address,
-        '-',
-        abi_row_no
-    ) AS abi_id,
-    CASE
-        WHEN num_abis > 1 THEN TRUE
-        ELSE FALSE
-    END AS bytecode_dupe
+
+{% if is_incremental() %}
+SYSDATE()
+{% else %}
+    TO_TIMESTAMP_NTZ('2000-01-01 00:00:00')
+{% endif %}
+
+AS _inserted_timestamp
 FROM
-    contracts_missing_abis
-    JOIN bytecode_abis USING (bytecode)
-    LEFT JOIN dupe_check USING (bytecode)
+    contracts_without_abis
+    JOIN unique_bytecode_abis USING (bytecode)

models/silver/abis/silver__bytecode_abis.yml

@@ -4,4 +4,5 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
-            - ABI_ID
+            - CONTRACT_ADDRESS
+            - ABI_HASH