Merge pull request #8 from FlipsideCrypto/an-5379-testing-updates

Testing strategy

.github/workflows/dbt_test_recent.yml

@@ -0,0 +1,47 @@
+name: dbt_test_recent
+run-name: dbt_test_recent
+
+on:
+  workflow_dispatch:
+  # schedule:
+  #   # Daily at 8:30am UTC
+  #   - cron: '30 8 * * *'
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "${{ vars.PYTHON_VERSION }}"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -m "aleo_models,tag:recent_test"
+          dbt test -m "aleo_models,tag:recent_test"

data/github_actions__workflows.csv

@@ -1,4 +1,6 @@
 workflow_name,workflow_schedule
 dbt_run_streamline_blocks_realtime,"5,35 * * * *"
 dbt_run_core,"20,50 * * * *"
-dbt_test_tasks,"0,30 * * * *"
+dbt_test_tasks,"0,30 * * * *"
+dbt_run_non_core,"30,50 * * * *"
+dbt_test_recent,"20 */4 * * *"

models/descriptions/author.md

@@ -0,0 +1,5 @@
+{% docs author %}
+
+The author of the block round, transition, or batch type object.
+
+{% enddocs %}

models/descriptions/batch_id.md

@@ -0,0 +1,5 @@
+{% docs batch_id %}
+
+Batch ID associated with the transaction.
+
+{% enddocs %}

models/descriptions/round.md

@@ -0,0 +1,5 @@
+{% docs round %}
+
+Prover round data associated with the transaction.
+
+{% enddocs %}

models/gold/core/core__dim_programs.yml

@@ -8,61 +8,29 @@ models:
         description: "{{ doc('deployment_block_id') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
       - name: DEPLOYMENT_BLOCK_TIMESTAMP
         description: "{{ doc('deployment_block_timestamp') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - TIMESTAMP_NTZ
       - name: PROGRAM_ID
         description: "{{ doc('program_id') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-          - unique
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - VARCHAR
       - name: EDITION
         description: "{{ doc('edition') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
       - name: PROGRAM
         description: "{{ doc('program') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - VARCHAR
       - name: VERIFYING_KEYS
         description: "{{ doc('verifying_keys') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
       - name: dim_program_id
         description: '{{ doc("pk") }}'
-        tests:
-          - dbt_expectations.expect_column_to_exist
-          - not_null
       - name: INSERT_TIMESTAMP
         description: '{{ doc("inserted_timestamp") }}'
-        tests:
-          - dbt_expectations.expect_column_to_exist
-          - not_null
       - name: MODIFIED_TIMESTAMP
-        description: '{{ doc("modified_timestamp") }}'
-        tests:
-          - dbt_expectations.expect_column_to_exist
-          - not_null
+        description: '{{ doc("modified_timestamp") }}'

models/gold/core/core__fact_block_round_batches.yml

@@ -3,84 +3,41 @@ version: 2
 models:
   - name: core__fact_block_round_batches
     description: Records of rounds and batches within blocks on the Aleo network, capturing the consensus and block production process from genesis onward.
-    tests:
-      - dbt_utils.unique_combination_of_columns:
-          combination_of_columns:
-            - batch_id
-            - block_id
     columns:
-      - name: block_id
+      - name: BLOCK_ID
         description: "{{ doc('block_id') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
       - name: BLOCK_TIMESTAMP
         description: "{{ doc('block_timestamp') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
           - dbt_expectations.expect_row_values_to_have_recent_data:
               datepart: day
               interval: 1
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - TIMESTAMP_NTZ
       - name: ROUND
-        # description: "{{ doc('network_id') }}"
-        # tests:
-        #   - dbt_expectations.expect_column_to_exist
-        #   - not_null
-        #   - dbt_expectations.expect_column_values_to_be_in_type_list:
-        #       column_type_list:
-        #         - STRING
-        #         - VARCHAR
-      - name: batch_id
-        # description: "{{ doc('tx_count') }}"
-        # tests: 
-        #   - dbt_expectations.expect_column_to_exist
-        #   - not_null
-        #   - dbt_expectations.expect_column_values_to_be_in_type_list:
-        #       column_type_list:
-        #         - NUMBER
-        #         - FLOAT 
-      - name: author
-        # description: "{{ doc('previous_hash') }}"
-        # tests:
-        #   - dbt_expectations.expect_column_to_exist
-        #   - not_null
-      - name: committee_id
-        # description: "{{ doc('proving_round') }}"
-        # tests: 
-        #   - dbt_expectations.expect_column_to_exist
-        #   - not_null
-        #   - dbt_expectations.expect_column_values_to_be_in_type_list:
-        #       column_type_list:
-        #         - NUMBER
-        #         - FLOAT
-      - name: transmission_ids
-        # description: "{{ doc('prover_rounds') }}"
-        # tests: 
-        #   - dbt_expectations.expect_column_to_exist
-        #   - not_null
-        #   - dbt_expectations.expect_column_values_to_be_in_type_list:
-        #       column_type_list:
-        #         - VARIANT
-      - name: fact_block_round_batches_id
-        description: '{{ doc("pk") }}'   
-        tests: 
+        description: "{{ doc('round') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: BATCH_ID
+        description: "{{ doc('batch_id') }}"
+        tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-          - unique
+      - name: AUTHOR
+        description: "The author of the block round."
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: COMMITTEE_ID
+        description: "The committee ID of the block round."
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: TRANSMISSION_IDS
+        description: "The transmission IDs of the block round."
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: FACT_BLOCK_ROUND_BATCHES_ID
+        description: '{{ doc("pk") }}'   
       - name: INSERTED_TIMESTAMP
         description: '{{ doc("inserted_timestamp") }}'   
-        tests: 
-          - dbt_expectations.expect_column_to_exist
-          - not_null
       - name: MODIFIED_TIMESTAMP
         description: '{{ doc("modified_timestamp") }}'          
-        tests: 
-          - dbt_expectations.expect_column_to_exist
-          - not_null

models/gold/core/core__fact_block_solutions.yml

@@ -3,87 +3,53 @@ version: 2
 models:
   - name: core__fact_block_solutions
     description: Records of all prover solutions for each block, including block details, rewards, and solution-specific information.
-    tests:
-      - dbt_utils.unique_combination_of_columns:
-          combination_of_columns:
-            - solution_id
-            - block_id
     columns:
-      - name: block_id
+      - name: BLOCK_ID
         description: "{{ doc('block_id') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
       - name: BLOCK_TIMESTAMP
         description: "{{ doc('block_timestamp') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
           - dbt_expectations.expect_row_values_to_have_recent_data:
               datepart: day
               interval: 1
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - TIMESTAMP_NTZ
-      - name: block_puzzle_reward
+      - name: BLOCK_PUZZLE_REWARD
         description: "{{ doc('puzzle_reward') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
-      - name: address
+      - name: ADDRESS
         description: "{{ doc('address') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-      - name: counter
+      - name: COUNTER
         description: "A counter value associated with the solution."
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-      - name: epoch_hash
+      - name: EPOCH_HASH
         description: "The hash of the epoch in which the solution was found."
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-      - name: solution_id
+      - name: SOLUTION_ID
         description: "{{ doc('solution_id') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-          - unique
-      - name: target
+      - name: TARGET
         description: "The target difficulty for the block solution."
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-      - name: reward_raw
+      - name: REWARD_RAW
         description: "The raw value of the reward for solving the block."
         tests:
           - dbt_expectations.expect_column_to_exist
-          - not_null
-      - name: reward
+      - name: REWARD
         description: "The processed or formatted value of the reward for solving the block."
         tests:
           - dbt_expectations.expect_column_to_exist
-      - name: fact_block_solutions_id
+      - name: FACT_BLOCK_SOLUTIONS_ID
         description: '{{ doc("pk") }}'   
-        tests: 
-          - dbt_expectations.expect_column_to_exist
-          - not_null
-          - unique
       - name: INSERTED_TIMESTAMP
         description: '{{ doc("inserted_timestamp") }}'   
-        tests: 
-          - dbt_expectations.expect_column_to_exist
-          - not_null
       - name: MODIFIED_TIMESTAMP
         description: '{{ doc("modified_timestamp") }}'          
-        tests: 
-          - dbt_expectations.expect_column_to_exist
-          - not_null