From 1067cd308c49c071677f31f9e48fe4ff19a0e93f Mon Sep 17 00:00:00 2001 From: John Craft Date: Thu, 14 Nov 2024 18:28:39 -0800 Subject: [PATCH] Build database container image within workflow instead of pulling from registry (#16) * Build db container image within workflow * Make the app easier to use - Display total user count on top of page - Update the README - Move db password to .env file - Fix paths in Dockerfiles after updating context in docker-compose * Copyedit the README --- .github/workflows/test-migration.yml | 23 ++++------------------- README.md | 15 ++++++++------- db/Dockerfile | 2 +- docker-compose.yml | 16 ++++++++-------- users-api/Dockerfile | 6 +++--- users-api/src/server.js | 8 ++++---- users-ui/Dockerfile | 2 +- users-ui/src/index.html | 18 +++++++++++++++++- 8 files changed, 46 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test-migration.yml b/.github/workflows/test-migration.yml index 75d8dee..0a95d3a 100644 --- a/.github/workflows/test-migration.yml +++ b/.github/workflows/test-migration.yml @@ -18,26 +18,11 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - name: Authenticate with GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v1 - - - name: Use gcloud CLI - run: gcloud info - - - name: Configure Docker to authenticate with GCR - run: | - gcloud auth configure-docker - - - name: Pull PostgreSQL image from GCP + - name: Build PostgreSQL container run: | - docker pull gcr.io/$GCP_PROEJCT_ID/$DB_HOST:latest + docker build -t $DB_HOST:latest ./db - name: Start PostgreSQL container run: | @@ -48,7 +33,7 @@ jobs: -e RESTORE_FROM_BACKUP=True \ -e DB_BACKUP_URL=$DB_BACKUP_URL \ -p 5432:5432 \ - gcr.io/$GCP_PROEJCT_ID/$DB_HOST:latest + $DB_HOST:latest - name: Install Sequelize CLI and dependencies run: | diff --git a/README.md b/README.md index 6a4cbf1..9246f8d 100644 --- a/README.md +++ b/README.md @@ -6,18 +6,19 @@ This application is based off of the reference app at [pvcy/anonymize-demo](http ## Overview This sample app runs a GitHub workflow to verify success when pull requests contain database migration code. -The app uses two GitHub workflows, `build-and-push.yml` and `test-migration.yml`, to coordinate everything. The first workflow, `build-and-push.yml`, builds and stores a container image from the `/db` directory which will load data from a GCS bucket at startup. The second worklow, `test-migration.yml`, detects when migration changes in pull requests exist and runs the database container with a full copy of production data against which migrations can be tested. The success or failur of the migration is added as a comment to the pull request. +The app uses a GitHub workflow, `test-migration.yml`, to detect when pull requests contain database migrations and runs the database container with a full copy of production data to test against. The success or failure of the migration is added as a comment to the pull request. ## Assumptions -* This app assumes there is a `pg_dump` from PostgreSQL stored in a Google Cloud Storage (GCS) bucket. * This is a Node app and uses [Sequelize](https://sequelize.org/) to run the database migration. * The workflow variable `DB_BACKUP_URL` needs to be pointed at the bucket containing the backup. * You must have [Docker](https://www.docker.com/) installed to run the app locally. -* This app loads the database backup into a PostgreSQL container when the workflow `test-migration` runs. If the data is exceptionally large, it may not load in time. +* This app assumes there is a `pg_dump` from PostgreSQL stored in a Google Cloud Storage (GCS) bucket. +* This app loads the database backup into a PostgreSQL container when the workflow runs. If the data is exceptionally large, it may not load in time. ## Getting started -1. Create a backup of your production database with `pg_dump` and store it in a GCS bucket. -2. Run the PostgreSQL database locally with the command `docker compose up --build`. This will launch the three containers defined in `/docker-compose.yml` and load the sample data (`/users-api/data/users.json`) into the database container. -3. From the command line, run the migration with `$\users-api\src npx sequelize-cli db:migrate`. The migration should pass locally. -4. Create a migration \ No newline at end of file +1. Create a backup of your production database with `pg_dump` and store it in a bucket (GCS, S3, etc.). +2. Make sure the environment variable `LOAD_DATA` is set to `True` in the `/docker-compose.yml` file. And make sure the environment variable `RESTORE_FROM_BACKUP` is set to `False`. +3. Run the PostgreSQL database locally with the command `docker compose up --build`. This will launch the three containers defined in `/docker-compose.yml` and load the sample data (`/users-api/data/users.json`) into the database container. +4. From the command line, run the migration with `$\users-api\src npx sequelize-cli db:migrate`. The migration should pass locally. +5. Create a pull request containing the new migration file. The workflow will run and comment on the pull request with the results of the migration test. diff --git a/db/Dockerfile b/db/Dockerfile index 6d71224..973969b 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -3,7 +3,7 @@ FROM postgres:latest RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* # Add the loading script to the container -COPY ./db/restore-backup.sh /restore-backup.sh +COPY ./restore-backup.sh /restore-backup.sh # Make the script executable RUN chmod +x /restore-backup.sh diff --git a/docker-compose.yml b/docker-compose.yml index 54f03d7..12b46cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,11 +2,11 @@ version: "3.7" services: users-api: build: - context: . - dockerfile: users-api/Dockerfile + context: users-api + dockerfile: Dockerfile environment: DB_USERNAME: "postgres" - DB_PASSWORD: ${DB_PASSWORD:-dMVZFeBWLOzYRV71} #It is strongly recommended to provide DB_PASSWORD in the environment rather than use this default. + DB_PASSWORD: ${DB_PASSWORD} POSTGRES_DB: "postgres" DB_HOST: "users-db" LOAD_DATA: "True" @@ -17,8 +17,8 @@ services: users-ui: build: - context: . - dockerfile: users-ui/Dockerfile + context: users-ui + dockerfile: Dockerfile ports: - 5000:80 depends_on: @@ -26,11 +26,11 @@ services: users-db: build: - context: . - dockerfile: db/Dockerfile + context: db + dockerfile: Dockerfile environment: POSTGRES_USER: "postgres" - POSTGRES_PASSWORD: ${DB_PASSWORD:-dMVZFeBWLOzYRV71} + POSTGRES_PASSWORD: ${DB_PASSWORD} POSTGRES_DB: "postgres" DB_BACKUP_URL: "https://storage.googleapis.com/anonymize-db-backups/us-west1/backup.sql" RESTORE_FROM_BACKUP: "False" diff --git a/users-api/Dockerfile b/users-api/Dockerfile index 1771461..afd4c9b 100644 --- a/users-api/Dockerfile +++ b/users-api/Dockerfile @@ -1,11 +1,11 @@ FROM node:18 WORKDIR /app -COPY ./users-api/src/package*.json ./ +COPY ./src/package*.json ./ RUN npm install -COPY ./users-api/src . -COPY ./users-api/data/ . +COPY ./src . +COPY ./data/ . EXPOSE 8080 diff --git a/users-api/src/server.js b/users-api/src/server.js index 700df15..f2fdf20 100644 --- a/users-api/src/server.js +++ b/users-api/src/server.js @@ -75,18 +75,18 @@ app.use( app.get("/users", async (req, res) => { const page = parseInt(req.query.page, 10) || 1; - const pageSize = 25; + const pageSize = 100; try { - const users = await User.findAll({ + const { count, rows } = await User.findAndCountAll({ offset: (page - 1) * pageSize, limit: pageSize, }); res.json({ - totalUsers: users.length, + totalUsers: count, page: page, pageSize: pageSize, - users: users, + users: rows, }); } catch (err) { console.error(err); diff --git a/users-ui/Dockerfile b/users-ui/Dockerfile index d34b64d..d2ccdb7 100644 --- a/users-ui/Dockerfile +++ b/users-ui/Dockerfile @@ -4,6 +4,6 @@ WORKDIR /usr/share/nginx # Remove default nginx static assets RUN rm -rf ./* -COPY ./users-ui/src ./html +COPY ./src ./html ENTRYPOINT ["nginx", "-g", "daemon off;"] diff --git a/users-ui/src/index.html b/users-ui/src/index.html index 94f2c8f..226afe8 100644 --- a/users-ui/src/index.html +++ b/users-ui/src/index.html @@ -45,12 +45,22 @@ text-align: right; padding: 3px; } + + .header-container { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 10px; + }

List of Users

- +
+
Total Users:
+ +
@@ -77,6 +87,8 @@

List of Users

fetch(API_ENDPOINT) .then(response => response.json()) .then(envelope => { + document.getElementById('total-users').textContent = formatNumber(envelope.totalUsers); + const tableBody = document.querySelector('#usersTable tbody'); envelope.users.forEach(user => { const row = document.createElement('tr'); @@ -165,6 +177,10 @@

List of Users

controlsDiv.appendChild(nextLink); } + + function formatNumber(num) { + return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); + }