From 1255e9112f3af6c746ebbbf22866390321bbe7a6 Mon Sep 17 00:00:00 2001
From: Chris Black <chris@ckblack.org>
Date: Tue, 20 Aug 2024 20:24:08 -0700
Subject: [PATCH] build db:ci automatically

---
 .github/workflows/dbdump.yml | 49 ++++++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/dbdump.yml b/.github/workflows/dbdump.yml
index 3a3c4878..1bf6824b 100644
--- a/.github/workflows/dbdump.yml
+++ b/.github/workflows/dbdump.yml
@@ -35,7 +35,7 @@ jobs:
           psql -h localhost -U postgres -d bety -c "CREATE EXTENSION postgis;"
 
       - name: Sync with EBI
-        run: script/load.bety.sh -a "postgres" -p "-h localhost" -d "bety" -o bety -m 99 -r 0 -c -w https://ebi-forecast.igb.illinois.edu/pecan/dump/bety.tar.gz
+        run: script/load.bety.sh -a "postgres" -p "-h localhost" -d "bety" -o bety -m 99 -r 0 -c -w http://www.betydb.org/dump/bety.tar.gz
 
       - name: Sync with BU
         run: script/load.bety.sh -a "postgres" -p "-h localhost" -d "bety" -o bety -m 99 -r 1
@@ -44,7 +44,7 @@ jobs:
         run: script/load.bety.sh -a "postgres" -p "-h localhost" -d "bety" -o bety -m 99 -r 2 -w ftp://anon:anon@ftp.test.bnl.gov/outgoing/betydb/bety.tar.gz
 
       - name: Sync with Wisconsin
-        run: script/load.bety.sh -a "postgres" -p "-h localhost" -d "bety" -o bety -m 99 -r 5 -w http://fen.aos.wisc.edu:6480/sync/dump/bety.tar.gz
+        run: script/load.bety.sh -a "postgres" -p "-h localhost" -d "bety" -o bety -m 99 -r 5 -w http://tree.aos.wisc.edu:6480/sync/dump/bety.tar.gz
 
       - name: Dump Database
         run: |
@@ -56,6 +56,45 @@ jobs:
           cd initdb
           docker build --tag image --file Dockerfile .
 
+      - name: Build smaller dump for CI runs
+        # update on scheduled runs, not every PR
+        if: github.event.schedule != ''
+        run: |
+          # Nuke some large tables not used in testing
+          #  (to avoid foreign key constraints when deleting their child inputs)
+          docker exec ${PG} psql -h localhost -U bety -c "
+            TRUNCATE runs,likelihoods,benchmarks CASCADE"
+          # Drop unneeded records from inputs table
+          # (loop is because many are referenced only by descendent inputs
+          #   that are themselves deleted here.)
+          while [ "$DEL_RES" != "DELETE 0" ]; do
+            DEL_RES=$(
+              docker exec ${PG} psql -h localhost -U bety -c "
+                DELETE FROM inputs WHERE
+                  id NOT IN (
+                    SELECT DISTINCT container_id FROM dbfiles
+                    WHERE container_type = 'Input'
+                    AND container_id IS NOT null)
+                  AND id NOT IN (
+                    SELECT DISTINCT parent_id FROM inputs
+                    WHERE parent_id IS NOT null);"
+              ) || break
+              echo ${DEL_RES}
+          done
+          # CI tests don't need any records from the excluded tables, just schemas
+          docker exec ${PG} pg_dump \
+            -h localhost -U bety \
+            --exclude-table-data='runs*' \
+            --exclude-table-data='inputs_runs*' \
+            --exclude-table-data='likelihoods*' \
+            --exclude-table-data='ensembles*' \
+            --exclude-table-data='posteriors_ensembles*' \
+            --exclude-table-data='benchmarks*' \
+            --exclude-table-data='reference_runs*' \
+            -F c \
+            bety > initdb/db.dump
+          docker build --tag image_ci --file Dockerfile .
+
       - name: Login into registry
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
@@ -75,3 +114,9 @@ jobs:
               docker push pecan/db:$T
             fi
           done
+          if [ -n "$(docker image ls -q image_ci)" ]; then
+            docker tag image_ci $IMAGE_ID:ci
+            docker push $IMAGE_ID:ci
+            docker tag image_ci pecan/db:ci
+            docker push pecan/db:ci
+          fi