Debug workflows

johnshaughnessy · Nov 3, 2023 · 2f04b28 · 2f04b28
1 parent 1237302
commit 2f04b28
Show file tree

Hide file tree

Showing 6 changed files with 57 additions and 51 deletions.
diff --git a/.github/workflows/backup-postgres.yml → .github/workflows/create-postgres-backup.yml b/.github/workflows/backup-postgres.yml → .github/workflows/create-postgres-backup.yml
@@ -1,4 +1,4 @@
-name: Backup Database
+name: Create Postgres Backup
 
 on:
   workflow_dispatch:
@@ -31,10 +31,10 @@ jobs:
             docker ps --all
             CONTAINER_NAME=\$(docker ps --format '{{.Names}}' | grep db)
             echo "Container name is: \$CONTAINER_NAME"
-            # Run pg_dump inside the container to create the backup
             echo "Creating database backup..."
             docker exec \$CONTAINER_NAME pg_dump -U postgres -h localhost trackdb | gzip > /tmp/$BACKUP_NAME
             echo "Done creating database backup."
+            docker ps --all
           ENDSSH
 
           scp -i ~/.ssh/private_key.pem ${{ secrets.SERVER_USER }}@${{ secrets.SERVER_HOST }}:/tmp/$BACKUP_NAME .

diff --git a/.github/workflows/deploy-server-image.yml b/.github/workflows/deploy-server-image.yml
@@ -81,4 +81,7 @@ jobs:
             docker compose -f track/base.yml -f track/prod.yml pull
             docker compose -f track/base.yml -f track/prod.yml down
             docker compose -f track/base.yml -f track/prod.yml up -d
+
+            echo "Status:"
+            docker compose -f track/base.yml -f track/prod.yml ps
           ENDSSH
diff --git a/.github/workflows/restore_from_backup.yml → .github/workflows/restore-from-backup.yml b/.github/workflows/restore_from_backup.yml → .github/workflows/restore-from-backup.yml
@@ -1,4 +1,4 @@
-name: Update Staging Database
+name: Restore from backup
 
 on:
   workflow_dispatch:
@@ -21,6 +21,7 @@ jobs:
 
       - name: Update from backup
         run: |
+          set -x
           echo "Authenticating with GCP."
           echo '${{ secrets.GCP_SA_KEY }}' > /tmp/gcp-sa-key.json
           gcloud auth activate-service-account --key-file=/tmp/gcp-sa-key.json
@@ -45,13 +46,15 @@ jobs:
           fi
 
           ssh -i ~/.ssh/private_key.pem ${{ secrets.SERVER_USER }}@${{ secrets.SERVER_HOST }} << ENDSSH
+            set -x
             echo "Removing old backups."
             rm -r /home/${{ secrets.SERVER_USER }}/track/backups/ || true
             mkdir -p /home/${{ secrets.SERVER_USER }}/track/backups/
           ENDSSH
 
           scp -i ~/.ssh/private_key.pem $BACKUP_FILE ${{ secrets.SERVER_USER }}@${{ secrets.SERVER_HOST }}:/home/${{ secrets.SERVER_USER }}/track/$BACKUP_FILE
           ssh -i ~/.ssh/private_key.pem ${{ secrets.SERVER_USER }}@${{ secrets.SERVER_HOST }} << ENDSSH
+            set -x
             echo "Restoring from backup: ./track/$BACKUP_FILE"
             ls ./track/backups/
 
@@ -63,53 +66,41 @@ jobs:
             echo "Bringing down services."
             docker compose -f track/base.yml -f track/prod.yml down
 
-            # echo "Starting db service in isolation."
-            # docker compose -f track/base.yml -f track/prod.yml up -d db
-
-            # echo "Preparing for restore."
-            # docker compose -f track/base.yml -f track/prod.yml exec db psql -U postgres -c "DROP DATABASE IF EXISTS trackdb;"
-            # docker compose -f track/base.yml -f track/prod.yml exec db psql -U postgres -c "CREATE DATABASE trackdb;"
-
-            # echo "Restoring from backup."
-            # docker compose -f track/base.yml -f track/prod.yml cp ./track/$BACKUP_FILE db:/backup
-            # docker compose -f track/base.yml -f track/prod.yml exec db psql -U postgres -d trackdb -f /backup
-
-            # echo "Bringing down db."
-            # docker compose -f track/base.yml -f track/prod.yml down
-
-            # echo "Bringing up all services."
-            # docker compose -f track/base.yml -f track/prod.yml up -d
-
-            # echo "Status:"
-            # docker compose -f track/base.yml -f track/prod.yml ps
+            echo "Status:"
+            docker compose -f track/base.yml -f track/prod.yml ps
 
             echo "Starting db service in isolation."
-            docker compose -f track/base.yml -f track/prod.yml up -d db || { echo "Failed to start db service"; exit 1; }
+            docker compose -f track/base.yml -f track/prod.yml up -d db
 
-            echo "Waiting for database to initialize..."
-            sleep 3  # Wait to ensure db is up. Adjust time as needed.
+            echo "Status:"
+            docker compose -f track/base.yml -f track/prod.yml ps
 
             echo "Preparing for restore."
-            docker compose -f track/base.yml -f track/prod.yml exec db psql -U postgres -c "DROP DATABASE IF EXISTS trackdb;" || { echo "Failed to drop database"; exit 1; }
-            sleep 1
-            docker compose -f track/base.yml -f track/prod.yml exec db psql -U postgres -c "CREATE DATABASE trackdb;" || { echo "Failed to create database"; exit 1; }
+            #docker compose -f track/base.yml -f track/prod.yml exec db echo "hello world!"
+            echo "A"
+            #docker compose -f track/base.yml -f track/prod.yml exec -T db echo "hello world!" || true
+            output=$(docker compose -f track/base.yml -f track/prod.yml exec db echo "hello world!") || true
+            echo "$output"
+            echo "B"
 
-            echo "Waiting for new database to be ready..."
-            sleep 3  # Wait for a few seconds. Adjust time as needed.
+
+            docker compose -f track/base.yml -f track/prod.yml exec -T db echo "hello world 2!"
+            docker compose -f track/base.yml -f track/prod.yml exec -T db psql -U postgres -c "DROP DATABASE IF EXISTS trackdb;"
+            docker compose -f track/base.yml -f track/prod.yml exec -T db psql -U postgres -c "CREATE DATABASE trackdb;"
 
             echo "Restoring from backup."
-            docker compose -f track/base.yml -f track/prod.yml cp ./track/$BACKUP_FILE db:/backup || { echo "Failed to copy backup file"; exit 1; }
-            docker compose -f track/base.yml -f track/prod.yml exec db psql -U postgres -d trackdb -f /backup || { echo "Failed to restore database from backup"; exit 1; }
+            docker compose -f track/base.yml -f track/prod.yml cp ./track/$BACKUP_FILE db:/backup
+            docker compose -f track/base.yml -f track/prod.yml exec db psql -U postgres -d trackdb -f /backup
 
-            sleep 3  # Wait for a few seconds. Adjust time as needed.
             echo "Bringing down db."
-            docker compose -f track/base.yml -f track/prod.yml down || { echo "Failed to bring down db"; exit 1; }
+            docker compose -f track/base.yml -f track/prod.yml down
 
             echo "Bringing up all services."
-            docker compose -f track/base.yml -f track/prod.yml up -d || { echo "Failed to bring up all services"; exit 1; }
+            docker compose -f track/base.yml -f track/prod.yml up -d
 
             echo "Status:"
             docker compose -f track/base.yml -f track/prod.yml ps
 
 
+
           ENDSSH
diff --git a/ops/.bashrc b/ops/.bashrc
@@ -114,6 +114,5 @@ if ! shopt -oq posix; then
         . /etc/bash_completion
     fi
 fi
-. "$HOME/.cargo/env"
 
 source "$HOME/track/alias"
diff --git a/ops/curl_weight_api.sh b/ops/curl_weight_api.sh
@@ -8,10 +8,10 @@ if [ -z "$HOST" ]; then
 fi
 
 echo "Listing weights:"
-curl -X GET $HOST/api/weights
+curl -X GET http://$HOST/api/weights
 
-# echo "Creating weight:"
-# curl -X POST $HOST/api/weights -H "Content-Type: application/json" -d '{"weight_kg": 65.1, "measured_at": "2022-11-01T12:34:56" }'
+echo "Creating weight:"
+curl -X POST $HOST/api/weights -H "Content-Type: application/json" -d '{"weight_kg": 65.1, "measured_at": "2022-11-01T12:34:56" }'
 
-# echo "Listing weights:"
-# curl -X GET $HOST/api/weights
+echo "Listing weights:"
+curl -X GET $HOST/api/weights
diff --git a/ops/inspect_backup.sh b/ops/inspect_backup.sh
@@ -1,5 +1,23 @@
 #!/usr/bin/env sh
-#
+
+POSTGRES_PASSWORD=$1
+BACKUP_FILE=$2
+
+if [ -z "$BACKUP_FILE" ]; then
+    BACKUP_FILE=$(gsutil ls -l gs://ocho-osai/track/pg_data/ | grep -v "TOTAL:" | sort -k2,2 -r | awk '{ $1=$2=""; print $0 }' | sed 's/^[ \t]*//' | head -n 1 | tr -d '\n')
+    echo "Warning: No BACKUP_FILE provided. Fetching the most recent one: $BACKUP_FILE"
+    gsutil cp $BACKUP_FILE ./backups/
+    echo "base: $(basename $BACKUP_FILE)"
+    BACKUP_FILE=$(basename $BACKUP_FILE)
+    echo "$BACKUP_FILE"
+    BACKUP_FILE=./backups/$BACKUP_FILE
+    echo "$BACKUP_FILE"
+fi
+
+if [ -z "$BACKUP_FILE" ] || [ -z "$POSTGRES_PASSWORD" ]; then
+    echo "Warning: Both BACKUP_FILE and POSTGRES_PASSWORD need to be provided."
+    cat <<EOF
+
 # List backups with:
 #
 #   gsutil ls gs://ocho-osai/track/pg_data/
@@ -10,16 +28,11 @@
 #
 # Run this script with
 #
-#   ./inspect_backup.sh gs://ocho-osai/track/pg_data/pg_backup_2023-11-02T14-51-44.sql.gz
+#   ./inspect_backup.sh password gs://ocho-osai/track/pg_data/pg_backup_2023-11-02T14-51-44.sql.gz
 #
 # From there, you can run psql commands to inspect the data.
-#
 
-BACKUP_FILE=$1
-POSTGRES_PASSWORD=$2
-
-if [ -z "$BACKUP_FILE" ] || [ -z "$POSTGRES_PASSWORD" ]; then
-    echo "Warning: Both BACKUP_FILE and POSTGRES_PASSWORD need to be provided."
+EOF
     exit 1
 fi
 
@@ -51,7 +64,7 @@ docker run --name temp-postgres \
     -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
     -v "$PWD/$DECOMPRESSED_FILE:/docker-entrypoint-initdb.d/dump.sql" \
     -v "$VOLUME_NAME:/var/lib/postgresql/data" \
-    -d trackdb
+    -d postgres
 
 # Check if container started successfully
 if [ $? -ne 0 ]; then
@@ -61,4 +74,4 @@ fi
 
 echo "PostgreSQL container started successfully."
 
-docker exec -it temp-postgres bash -c "sleep 1; psql -U postgres -d trackdb"
+docker exec -it temp-postgres bash -c "sleep 1; psql -U postgres"