Merge branch 'duneanalytics:main' into main

.github/pull_request_template.md

@@ -0,0 +1,55 @@
+## Thank you for contributing to Spellbook 🪄
+
+### Update!
+Please build spells in the proper [subproject](../dbt_subprojects/) directory. For more information, please see the main [readme](../README.md), which also links to a GH discussion with the option to ask questions.
+
+### Contribution type
+Please check the type of contribution this pull request is for:
+
+- [ ] New spell(s)
+- [ ] Adding to existing spell lineage
+- [ ] Bug fix
+
+**Note:** You can safely discard any section below which doesn't apply based on selection above
+
+---
+
+### For new spell(s)
+If you are building new spell(s), please provide the following information:
+
+- **Spell name(s):** *schema.table_name*
+- **Description:** [Detailed description of the new spell(s) and their purpose]
+- **Who are the new spell(s) for?** [Internal team or general community]
+- **How will the new spell(s) be used downstream?** [Description of downstream usage]
+- **Implementation details:** [Information on how the spell(s) are implemented]
+- **Test instructions:** [How to test the new spell(s)]
+- **Related issue(s):** [Link to related issues, if any]
+
+---
+
+### For adding to existing spell lineage
+If you are adding to an existing spell lineage, please provide the following information:
+
+- **Description:** [Description of the changes made]
+
+---
+
+### For bug fixes
+If you are fixing a bug, please provide the following information:
+
+- **Description:** [Description of the bug fix]
+- **Steps to reproduce:** [How to reproduce the bug]
+- **Implementation details:** [Information on how the bug was fixed]
+- **Test instructions:** [How to test the fix]
+- **Related issue(s):** [Link to related issues, if any]
+
+---
+
+### Additional information
+Please provide any additional information that might help us review your pull request:
+
+- [Any additional information]
+
+---
+
+Thank you for your contribution!

.github/workflows/commit_manifest.yml

@@ -9,13 +9,13 @@ on:
 concurrency:
   group: ${{ github.workflow }}
   cancel-in-progress: true
-      
+
 jobs:
   commit_manifest:
     runs-on: [ self-hosted, linux, spellbook-trino-ci ]
     strategy:
       matrix:
-        project: [ 'tokens', 'spellbook' ]
+        project: [ 'tokens', 'daily_spellbook', 'nft', 'dex', 'solana', 'hourly_spellbook']
       max-parallel: 1
 
     steps:
@@ -27,26 +27,22 @@ jobs:
 
     - name: Add git_sha to schema
       run: "/runner/change_schema.sh wizard"
-    
+
     - name: Setup variables
       run: |
         echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
         echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
-        if [[ "${{ matrix.project }}" == "tokens" ]]; then
-          echo "PROJECT_DIR=tokens" >> $GITHUB_ENV
-          echo "S3_LOCATION=s3://manifest-spellbook-dunesql/tokens" >> $GITHUB_ENV
-        elif [[ "${{ matrix.project }}" == "spellbook" ]]; then
-          echo "PROJECT_DIR=." >> $GITHUB_ENV
-          echo "S3_LOCATION=s3://manifest-spellbook-dunesql/spellbook" >> $GITHUB_ENV
-        fi
+        echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{matrix.project}}" >> $GITHUB_ENV
+        PROJECT_DIR=dbt_subprojects/${{ matrix.project }}
+        echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
 
     - name: dbt dependencies
       working-directory: ${{env.PROJECT_DIR}}
       run: "dbt deps"
-    
+
     - name: Run dbt debug on Trino if dunesql engine is used, and retry until cluster is available
       run: "./scripts/ensure_cluster.sh"
-          
+
     - name: dbt compile to create prod manifest from main
       run: "dbt compile --target-path . $PROFILE --project-dir ${PROJECT_DIR}"
 
@@ -57,4 +53,4 @@ jobs:
       run: "aws s3 cp $S3_LOCATION/manifest.json $S3_LOCATION/manifest_$GIT_SHA.json"
 
     - name: upload manifest
-      run: "aws s3 cp $PROJECT_DIR/manifest.json $S3_LOCATION/manifest.json"
+      run: "aws s3 cp $PROJECT_DIR/manifest.json $S3_LOCATION/manifest.json"

.github/workflows/daily_spellbook.yml

@@ -0,0 +1,20 @@
+name: DBT Daily Spellbook run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - dbt_subprojects/daily_spellbook/**
+      - .github/workflows/daily_spellbook.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'daily_spellbook'
+

.github/workflows/dbt_run.yml

@@ -0,0 +1,67 @@
+name: DBT Run Reusable Workflow
+
+on:
+  workflow_call:
+    inputs:
+      project:
+        description: 'DBT run for project'
+        required: true
+        type: string
+
+jobs:
+  dbt-test:
+    runs-on: [ self-hosted, linux, spellbook-trino-ci ]
+    timeout-minutes: 90
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Setup variables
+        run: | # Spellbook is a special case because it's not a subdirectory
+          echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
+          echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
+          echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{inputs.project}}" >> $GITHUB_ENV
+          PROJECT_DIR=dbt_subprojects/${{ inputs.project }}
+          echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
+      - name: Add git_sha to schema
+        run: "/runner/change_schema.sh git_dunesql_$GIT_SHA"
+
+      - name: Get latest manifest
+        run: "aws s3 cp $S3_LOCATION/manifest.json $PROJECT_DIR/manifest.json"
+        continue-on-error: true # Temprorary until all projects have manifest
+
+      - name: dbt dependencies
+        working-directory: ${{env.PROJECT_DIR}}
+        run: "dbt deps"
+
+      - name: Activate DuneSQL Cluster
+        run: "./scripts/ensure_cluster.sh"
+
+      - name: dbt compile to create manifest to compare to
+        working-directory: ${{env.PROJECT_DIR}}
+        run: "dbt --warn-error compile"
+
+      - name: check schemas
+        run: "./scripts/check_schema.sh"
+
+      - name: dbt seed
+        run: "dbt seed $PROFILE --select @state:modified --exclude tag:prod_exclude tag:remove --state . --project-dir ${PROJECT_DIR}"
+
+      - name: dbt run initial model(s)
+        run: "dbt -x run $PROFILE --select state:modified --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"
+
+      - name: dbt test initial model(s)
+        run: "dbt test $PROFILE --select state:new state:modified --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"
+
+      - name: Set environment variable for incremental model count
+        run: |
+          echo "INC_MODEL_COUNT=$(echo dbt ls $PROFILE --select state:modified,config.materialized:incremental --state . --resource-type model  --project-dir ${PROJECT_DIR} | wc -l)" >> $GITHUB_ENV
+
+      - name: dbt run incremental model(s) if applicable
+        if: env.INC_MODEL_COUNT > 0
+        run: "dbt run $PROFILE --select state:modified,config.materialized:incremental --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"
+
+      - name: dbt test incremental model(s) if applicable
+        if: env.INC_MODEL_COUNT > 0
+        run: "dbt test $PROFILE --select state:modified,config.materialized:incremental --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"

.github/workflows/dex.yml

@@ -0,0 +1,19 @@
+name: DBT DEX sector run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - dbt_subprojects/dex/**
+      - .github/workflows/dex.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'dex'

.github/workflows/hourly_spellbook.yml

@@ -0,0 +1,20 @@
+name: DBT Hourly Spellbook run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - dbt_subprojects/hourly_spellbook/**
+      - .github/workflows/hourly_spellbook.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'hourly_spellbook'
+

.github/workflows/nft.yml

@@ -0,0 +1,20 @@
+name: DBT NFT run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - dbt_subprojects/nft/**
+      - .github/workflows/nft.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'nft'
+

.github/workflows/prices_check.yml

@@ -5,7 +5,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - 'models/prices/*/*.sql'
+      - 'dbt_subprojects/tokens/models/prices/*/*.sql'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -30,7 +30,7 @@ jobs:
       - name: Get prices diff
         shell: bash
         run: |
-          git diff ${{ github.event.pull_request.base.sha}}..${{ github.event.pull_request.head.sha }} models/prices/**/*.sql | grep '^\+ ' > scripts/new_lines.txt
+          git diff ${{ github.event.pull_request.base.sha}}..${{ github.event.pull_request.head.sha }} dbt_subprojects/tokens/models/prices/**/*.sql | grep '^\+ ' > scripts/new_lines.txt
 
       - name: Run validations
         id: validations

.github/workflows/solana.yml

@@ -0,0 +1,19 @@
+name: DBT Solana sector run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - dbt_subprojects/solana/**
+      - .github/workflows/solana.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'solana'