Merge pull request #3 from FlipsideCrypto/workflow_deployment

finishing touches across many models. also introduces dim_labels

.github/workflows/dbt_alter_gha_task.yml

@@ -0,0 +1,46 @@
+name: dbt_alter_gha_task
+run-name: dbt_alter_gha_task
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    inputs:
+      workflow_name:
+        type: string
+        description: Name of the workflow to perform the action on, no .yml extension
+        required: true
+      task_action:
+        type: choice 
+        description: Action to perform
+        required: true
+        options:
+          - SUSPEND
+          - RESUME
+        default: SUSPEND
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7
+    with:
+      workflow_name: |
+         ${{ inputs.workflow_name }}
+      task_action: |
+         ${{ inputs.task_action }}
+      environment: workflow_prod
+    secrets: inherit

.github/workflows/dbt_integration_test.yml

@@ -0,0 +1,17 @@
+name: dbt_run_integration_test
+run-name: ${{ github.event.inputs.branch }}
+
+on:
+  workflow_dispatch:
+
+concurrency: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt.yml@main
+    with:
+      command: >
+        dbt test --selector 'integration_tests'
+      environment:  ${{ github.ref == 'refs/heads/main' && 'workflow_prod' || 'workflow_dev' }}
+      warehouse: ${{ vars.WAREHOUSE }}
+    secrets: inherit

.github/workflows/dbt_run_adhoc.yml

@@ -0,0 +1,66 @@
+name: dbt_run_adhoc
+run-name: ${{ inputs.dbt_command }}
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    inputs:
+      environment:
+        type: choice
+        description: DBT run environment
+        required: true
+        options:
+          - dev
+          - prod
+        default: dev
+      warehouse:
+        type: choice
+        description: Snowflake warehouse
+        required: true
+        options:
+          - DBT
+          - DBT_CLOUD
+          - DBT_EMERGENCY
+        default: DBT
+      dbt_command:
+        type: string
+        description: "DBT Run Command"
+        required: true
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ inputs.warehouse }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_${{ inputs.environment }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          ${{ inputs.dbt_command }}

.github/workflows/dbt_run_core.yml

@@ -0,0 +1,46 @@
+name: dbt_run_core
+run-name: dbt_run_core
+
+on:
+  workflow_dispatch:
+  # schedule:
+  #   # Runs "at minute 10, every hour" (see https://crontab.guru)
+  #   - cron: '10 * * * *'
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "${{ vars.PYTHON_VERSION }}"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -m "aleo_models,tag:core" 

.github/workflows/dbt_run_dev_refresh.yml

@@ -0,0 +1,45 @@
+name: dbt_run_dev_refresh
+run-name: dbt_run_dev_refresh
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs "at 5:10 UTC" (see https://crontab.guru)
+    - cron: '10 5 * * *'
+
+env:
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "${{ vars.PYTHON_VERSION }}"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run-operation run_sp_create_prod_clone

.github/workflows/dbt_run_streamline_blocks_realtime.yml

@@ -5,8 +5,8 @@ on:
   workflow_dispatch:
     branches:
       - "main"
-  schedule:
-    - cron: '0 * * * *'
+  #schedule:
+  #  - cron: '0 * * * *'
 
 env:
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"

.github/workflows/dbt_test_monthly.yml

@@ -0,0 +1,45 @@
+name: dbt_test_monthly
+run-name: dbt_test_monthly
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs 2nd of month at 6AM” (see https://crontab.guru)
+    - cron: '0 6 2 * *'
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "${{ vars.PYTHON_VERSION }}"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt test -m  "aleo_models,tag:full_test"

.github/workflows/dbt_test_tasks.yml

@@ -0,0 +1,27 @@
+name: dbt_test_tasks
+run-name: dbt_test_tasks
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7
+    secrets: inherit

data/github_actions__workflows.csv

@@ -0,0 +1,4 @@
+workflow_name,workflow_schedule
+dbt_run_streamline_blocks_realtime,"5,35 * * * *"
+dbt_run_core,"20,50 * * * *"
+dbt_test_tasks,"0,30 * * * *"

models/descriptions/address.md

@@ -0,0 +1,5 @@
+{% docs address %}
+
+The unique address referring to an entity on the blockchain. 
+
+{% enddocs %}

models/descriptions/solution_id.md

@@ -0,0 +1,5 @@
+{% docs solution_id %}
+
+A unique identifier for the prover solution. 
+
+{% enddocs %}