From ef6234349665ece1b4c6d8f4199d2906b1a15eee Mon Sep 17 00:00:00 2001 From: John Craft Date: Mon, 23 Oct 2023 20:04:07 -0700 Subject: [PATCH] Optionally restore db from backup at start --- .github/workflows/anonymize.yml | 10 +--- .github/workflows/snapshot-data.yml | 22 ++------ .github/workflows/test-migration.yml | 81 ++++++++++++++++------------ db/Dockerfile | 15 +++--- db/restore-backup.sh | 25 +++++++++ docker-compose.yml | 16 ++++-- users-api/Dockerfile | 1 - users-api/src/load-data.js | 4 +- users-api/src/server.js | 4 +- 9 files changed, 101 insertions(+), 77 deletions(-) create mode 100644 db/restore-backup.sh diff --git a/.github/workflows/anonymize.yml b/.github/workflows/anonymize.yml index 644fb96..9d7ad94 100644 --- a/.github/workflows/anonymize.yml +++ b/.github/workflows/anonymize.yml @@ -1,7 +1,7 @@ -name: Anonymize database and export it +name: Anonymize database on: pull_request: - branches: [ main ] + branches: [ develop ] jobs: anonymize: runs-on: ubuntu-latest @@ -16,9 +16,3 @@ jobs: db-password: ${{ secrets.PG_PASSWORD }} client-id: ${{ secrets.DB_SSH_KEY }} client-secret: ${{ secrets.DB_SSH_SECRET }} - dump-database: - runs-on: ubuntu-latest - steps: - - name: Dump database to blob storage - run: | - some bash command to call pgdump \ No newline at end of file diff --git a/.github/workflows/snapshot-data.yml b/.github/workflows/snapshot-data.yml index 100116f..45e09b0 100644 --- a/.github/workflows/snapshot-data.yml +++ b/.github/workflows/snapshot-data.yml @@ -1,7 +1,7 @@ -name: Deploy app to Cloud Run +name: Backup database to blob cloud storage on: pull_request: - branches: [ main ] + branches: [ develop ] # on: # schedule: # - cron: '0 0 * * *' @@ -21,22 +21,6 @@ jobs: service_account_key: ${{ secrets.GCP_SA_KEY }} export_default_credentials: true - - name: Build and push database image - run: | - cd ./db - docker build -t destination-db . - gcloud auth configure-docker - docker tag destination-db:latest gcr.io/database-migration-anonymized/destination-db - docker push gcr.io/database-migration-anonymized/destination-db - - - name: Deploy to Cloud Run # service must already exist - run: | - gcloud run deploy destination-database \ - --image=gcr.io/database-migration-anonymized/destination-db \ - --platform managed \ - --allow-unauthenticated \ - --region us-west1 - - name: Backup database run: | pg_dump -h db \ @@ -47,7 +31,7 @@ jobs: - name: Copy backup to storage run: | - gsutil cp db_backup.tar gs://anonymize-db-backups + gsutil cp db_backup.tar gs://anonymize-db-backups/us-west1/ diff --git a/.github/workflows/test-migration.yml b/.github/workflows/test-migration.yml index e317064..3f7878b 100644 --- a/.github/workflows/test-migration.yml +++ b/.github/workflows/test-migration.yml @@ -1,42 +1,55 @@ -name: Create Preview and Test Migration +name: Standup Database and Test Migration on: pull_request: branches: [ main ] - paths: - - 'users-api/src/migrations/**' + # paths: + # - 'users-api/src/migrations/**' jobs: - preview: + check-migrations: runs-on: ubuntu-latest - steps: - - name: Login - uses: okteto/context@2.19.2 - with: - token: ${{ secrets.OKTETO_GCP_TOKEN }} - url: ${{ secrets.OKTETO_GCP_URL }} - - - name: Deploy preview environment - uses: okteto/deploy-preview@2.19.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - name: anonymize-pr-${{ github.event.number }} - scope: global - timeout: 15m - - load-data: - name: Load data from snapshot - runs-on: ubuntu-latest - steps: - - id: load-data-from-snapshot - run: | - echo "some bash command goes here" + env: + GCP_PROEJCT_ID: "database-migration-anonymized" + DB_HOST: "users-db" - tests: - name: Verify migration works - runs-on: ubuntu-latest - needs: preview steps: - - id: run-migration - name: Run Sequelize migration - run: npx sequelize-cli db:migrate + - name: Checkout code + uses: actions/checkout@v2 + + - name: Authenticate with GCP + uses: google-github-actions/setup-gcloud@v0.3.0 + with: + project_id: $GCP_PROEJCT_ID + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + + - name: Pull PostgreSQL image from GCP + run: | + docker pull gcr.io/$GCP_PROEJCT_ID/$DB_HOST:latest + + - name: Start PostgreSQL container + env: + POSTGRES_PASSWORD: ${DB_PASSWORD:-dMVZFeBWLOzYRV71} + POSTGRES_DB: "postgres" + run: | + docker run -d --name $DB_HOST -e POSTGRES_PASSWORD=$DB_PASSWORD: -e POSTGRES_DB=$POSTGRES_DB -p 5432:5432 gcr.io/$GCP_PROEJCT_ID/$DB_HOST:latest + + - name: Install Sequelize CLI and dependencies + run: | + npm install sequelize-cli + npm install # Assuming you have package.json with necessary dependencies + + - name: Wait for Postgres to be ready + run: | + for i in {1..10}; do + nc -z localhost 5432 && echo "Postgres is up" && break + echo "Waiting for Postgres..." + sleep 10 + done + + # - name: Run Sequelize migrations + # run: | + # npx sequelize-cli db:migrate --config path-to-your-sequelize-config + + - name: Cleanup + run: docker stop $DB_HOST diff --git a/db/Dockerfile b/db/Dockerfile index 92d70d0..6d71224 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -1,11 +1,12 @@ FROM postgres:latest -# SSH -RUN apt-get update && apt-get install -y openssh-server -RUN echo "root:Docker!" | chpasswd -RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN mkdir /var/run/sshd +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* -EXPOSE 22 +# Add the loading script to the container +COPY ./db/restore-backup.sh /restore-backup.sh -CMD service ssh start && docker-entrypoint.sh postgres +# Make the script executable +RUN chmod +x /restore-backup.sh + +# Use the script as the entrypoint +ENTRYPOINT ["/restore-backup.sh"] diff --git a/db/restore-backup.sh b/db/restore-backup.sh new file mode 100644 index 0000000..59099d6 --- /dev/null +++ b/db/restore-backup.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Start Postgres in the background to ensure it's ready to accept connections +docker-entrypoint.sh postgres & + +# The process ID of the PostgreSQL process, so we can wait for it later +POSTGRES_PID=$! + +# Wait for Postgres to be ready +until pg_isready -h localhost -U $POSTGRES_USER; do + sleep 1 +done + +if [ "$RESTORE_FROM_BACKUP" == "True" ]; then + # Download the dump from the public GCS bucket using curl + echo "Download backup" + curl -o /tmp/dump.backup $DB_BACKUP_URL + + # Restore the dump using pg_restore (or psql depending on the dump format) + echo "Restore backup" + pg_restore -U $POSTGRES_USER -d $POSTGRES_DB /tmp/dump.backup +fi + +# Wait for the PostgreSQL process to complete +wait $POSTGRES_PID diff --git a/docker-compose.yml b/docker-compose.yml index e6a220a..ae361ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,11 +7,13 @@ services: environment: DB_USERNAME: "postgres" DB_PASSWORD: ${DB_PASSWORD:-dMVZFeBWLOzYRV71} #It is strongly recommended to provide DB_PASSWORD in the environment rather than use this default. - LOAD_DATA: + POSTGRES_DB: "postgres" + DB_HOST: "users-db" + LOAD_DATA: "False" ports: - 8080:8080 depends_on: - - db + - users-db users-ui: build: @@ -22,9 +24,15 @@ services: depends_on: - users-api - db: - image: postgres + users-db: + build: + context: . + dockerfile: db/Dockerfile environment: + POSTGRES_USER: "postgres" POSTGRES_PASSWORD: ${DB_PASSWORD:-dMVZFeBWLOzYRV71} + POSTGRES_DB: "postgres" + DB_BACKUP_URL: "https://storage.googleapis.com/anonymize-db-backups/us-west1/backup.sql" + RESTORE_FROM_BACKUP: "True" ports: - 5432:5432 diff --git a/users-api/Dockerfile b/users-api/Dockerfile index 7dc801d..5bc7d7d 100644 --- a/users-api/Dockerfile +++ b/users-api/Dockerfile @@ -10,4 +10,3 @@ ADD ./users-api/data/users.json ./users.json EXPOSE 8080 CMD ["./docker-entrypoint.sh"] -# CMD [ "node", "server.js" ] diff --git a/users-api/src/load-data.js b/users-api/src/load-data.js index 029b3b0..2341206 100644 --- a/users-api/src/load-data.js +++ b/users-api/src/load-data.js @@ -8,8 +8,8 @@ const connectToDatabase = async () => { retries++; console.log(`Attempt ${retries} to connect to database`); client = new Client({ - host: "db", - database: "postgres", + host: process.env.DB_HOST, + database: process.env.POSTGRES_DB, user: process.env.DB_USERNAME, password: process.env.DB_PASSWORD, }); diff --git a/users-api/src/server.js b/users-api/src/server.js index ea8c990..122f3a3 100644 --- a/users-api/src/server.js +++ b/users-api/src/server.js @@ -7,11 +7,11 @@ const HOST = "0.0.0.0"; const { Sequelize, DataTypes } = require("sequelize"); const sequelize = new Sequelize( - "postgres", + process.env.POSTGRES_DB, process.env.DB_USERNAME, process.env.DB_PASSWORD, { - host: "db", + host: process.env.DB_HOST, dialect: "postgres", } );