Merge pull request #3 from pvcy/jc/restore-from-backup

Optionally restore database during startup

.github/workflows/anonymize.yml

@@ -1,7 +1,7 @@
-name: Anonymize database and export it
+name: Anonymize database
 on:
   pull_request:
-    branches: [ main ]
+    branches: [ develop ]
 jobs:
   anonymize:
     runs-on: ubuntu-latest
@@ -16,9 +16,3 @@ jobs:
           db-password: ${{ secrets.PG_PASSWORD }}
           client-id: ${{ secrets.DB_SSH_KEY }}
           client-secret: ${{ secrets.DB_SSH_SECRET }}
-  dump-database:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Dump database to blob storage
-      run: |
-        some bash command to call pgdump

.github/workflows/snapshot-data.yml

@@ -1,7 +1,7 @@
-name: Deploy app to Cloud Run
+name: Backup database to blob cloud storage
 on:
   pull_request:
-    branches: [ main ]
+    branches: [ develop ]
 # on:
 #   schedule: 
 #     - cron: '0 0 * * *'
@@ -21,22 +21,6 @@ jobs:
         service_account_key: ${{ secrets.GCP_SA_KEY }}
         export_default_credentials: true
 
-    - name: Build and push database image
-      run: |
-        cd ./db
-        docker build -t destination-db .
-        gcloud auth configure-docker
-        docker tag destination-db:latest gcr.io/database-migration-anonymized/destination-db
-        docker push gcr.io/database-migration-anonymized/destination-db
-
-    - name: Deploy to Cloud Run # service must already exist
-      run: |
-        gcloud run deploy destination-database \
-          --image=gcr.io/database-migration-anonymized/destination-db \
-          --platform managed \
-          --allow-unauthenticated \
-          --region us-west1
-
     - name: Backup database
       run: |
         pg_dump -h db \
@@ -47,7 +31,7 @@ jobs:
 
     - name: Copy backup to storage
       run: |
-        gsutil cp db_backup.tar gs://anonymize-db-backups
+        gsutil cp db_backup.tar gs://anonymize-db-backups/us-west1/
 
 
 

.github/workflows/test-migration.yml

@@ -1,42 +1,55 @@
-name: Create Preview and Test Migration
+name: Standup Database and Test Migration
 on:
   pull_request:
     branches: [ main ]
-    paths: 
-    - 'users-api/src/migrations/**'
+    # paths: 
+    # - 'users-api/src/migrations/**'
 
 jobs:
-  preview:
+  check-migrations:
     runs-on: ubuntu-latest
-    steps:
-      - name: Login
-        uses: okteto/[email protected]
-        with:
-          token: ${{ secrets.OKTETO_GCP_TOKEN }}
-          url: ${{ secrets.OKTETO_GCP_URL }}
-
-      - name: Deploy preview environment
-        uses: okteto/[email protected]
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          name: anonymize-pr-${{ github.event.number }}
-          scope: global
-          timeout: 15m
-
-  load-data:
-    name: Load data from snapshot
-    runs-on: ubuntu-latest
-    steps:
-      - id: load-data-from-snapshot
-        run: |
-          echo "some bash command goes here"
+    env:
+      GCP_PROEJCT_ID: "database-migration-anonymized"
+      DB_HOST: "users-db"
 
-  tests:
-    name: Verify migration works
-    runs-on: ubuntu-latest
-    needs: preview
     steps:
-      - id: run-migration
-        name: Run Sequelize migration
-        run: npx sequelize-cli db:migrate
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Authenticate with GCP
+      uses: google-github-actions/[email protected]
+      with:
+        project_id: $GCP_PROEJCT_ID
+        service_account_key: ${{ secrets.GCP_SA_KEY }}
+        export_default_credentials: true
+
+    - name: Pull PostgreSQL image from GCP
+      run: |
+        docker pull gcr.io/$GCP_PROEJCT_ID/$DB_HOST:latest
+
+    - name: Start PostgreSQL container
+      env:
+        POSTGRES_PASSWORD: ${DB_PASSWORD:-dMVZFeBWLOzYRV71}
+        POSTGRES_DB: "postgres"
+      run: |
+        docker run -d --name $DB_HOST -e POSTGRES_PASSWORD=$DB_PASSWORD: -e POSTGRES_DB=$POSTGRES_DB -p 5432:5432 gcr.io/$GCP_PROEJCT_ID/$DB_HOST:latest
+
+    - name: Install Sequelize CLI and dependencies
+      run: |
+        npm install sequelize-cli
+        npm install # Assuming you have package.json with necessary dependencies
+
+    - name: Wait for Postgres to be ready
+      run: |
+        for i in {1..10}; do
+          nc -z localhost 5432 && echo "Postgres is up" && break
+          echo "Waiting for Postgres..."
+          sleep 10
+        done
+
+    # - name: Run Sequelize migrations
+    #   run: |
+    #     npx sequelize-cli db:migrate --config path-to-your-sequelize-config
+
+    - name: Cleanup
+      run: docker stop $DB_HOST

db/Dockerfile

@@ -1,11 +1,12 @@
 FROM postgres:latest
 
-# SSH
-RUN apt-get update && apt-get install -y openssh-server
-RUN echo "root:Docker!" | chpasswd
-RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
-RUN mkdir /var/run/sshd
+RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
 
-EXPOSE 22
+# Add the loading script to the container
+COPY ./db/restore-backup.sh /restore-backup.sh
 
-CMD service ssh start && docker-entrypoint.sh postgres
+# Make the script executable
+RUN chmod +x /restore-backup.sh
+
+# Use the script as the entrypoint
+ENTRYPOINT ["/restore-backup.sh"]

db/restore-backup.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Start Postgres in the background to ensure it's ready to accept connections
+docker-entrypoint.sh postgres &
+
+# The process ID of the PostgreSQL process, so we can wait for it later
+POSTGRES_PID=$!
+
+# Wait for Postgres to be ready
+until pg_isready -h localhost -U $POSTGRES_USER; do
+  sleep 1
+done
+
+if [ "$RESTORE_FROM_BACKUP" == "True" ]; then
+  # Download the dump from the public GCS bucket using curl
+  echo "Download backup"
+  curl -o /tmp/dump.backup $DB_BACKUP_URL
+
+  # Restore the dump using pg_restore (or psql depending on the dump format)
+  echo "Restore backup"
+  pg_restore -U $POSTGRES_USER -d $POSTGRES_DB /tmp/dump.backup
+fi
+
+# Wait for the PostgreSQL process to complete
+wait $POSTGRES_PID

docker-compose.yml

@@ -7,11 +7,13 @@ services:
     environment:
       DB_USERNAME: "postgres"
       DB_PASSWORD: ${DB_PASSWORD:-dMVZFeBWLOzYRV71} #It is strongly recommended to provide DB_PASSWORD in the environment rather than use this default.
-      LOAD_DATA: 
+      POSTGRES_DB: "postgres"
+      DB_HOST: "users-db"
+      LOAD_DATA: "False"
     ports:
       - 8080:8080
     depends_on:
-      - db
+      - users-db
 
   users-ui:
     build:
@@ -22,9 +24,15 @@ services:
     depends_on:
       - users-api
 
-  db:
-    image: postgres
+  users-db:
+    build:
+      context: .
+      dockerfile: db/Dockerfile
     environment:
+      POSTGRES_USER: "postgres"
       POSTGRES_PASSWORD: ${DB_PASSWORD:-dMVZFeBWLOzYRV71}
+      POSTGRES_DB: "postgres"
+      DB_BACKUP_URL: "https://storage.googleapis.com/anonymize-db-backups/us-west1/backup.sql"
+      RESTORE_FROM_BACKUP: "True"
     ports:
       - 5432:5432

users-api/Dockerfile

@@ -10,4 +10,3 @@ ADD ./users-api/data/users.json ./users.json
 EXPOSE 8080
 
 CMD ["./docker-entrypoint.sh"]
-# CMD [ "node", "server.js" ]

users-api/src/load-data.js

@@ -8,8 +8,8 @@ const connectToDatabase = async () => {
     retries++;
     console.log(`Attempt ${retries} to connect to database`);
     client = new Client({
-      host: "db",
-      database: "postgres",
+      host: process.env.DB_HOST,
+      database: process.env.POSTGRES_DB,
       user: process.env.DB_USERNAME,
       password: process.env.DB_PASSWORD,
     });

users-api/src/server.js

@@ -7,11 +7,11 @@ const HOST = "0.0.0.0";
 const { Sequelize, DataTypes } = require("sequelize");
 
 const sequelize = new Sequelize(
-  "postgres",
+  process.env.POSTGRES_DB,
   process.env.DB_USERNAME,
   process.env.DB_PASSWORD,
   {
-    host: "db",
+    host: process.env.DB_HOST,
     dialect: "postgres",
   }
 );