Merge pull request #1 from pvcy/jc/add-user-pagination-controls

Add pagination controls to user list

.github/workflows/snapshot-data.yml

@@ -4,16 +4,47 @@ on:
     - cron: '0 0 * * *'
 
 jobs:
-  anonymize:
+  deploy:
     runs-on: ubuntu-latest
+
     steps:
-      - name: Anonymize Data
-        uses: pvcy/anonymize-project@latest
-        with:
-          project-id: 4e1213f4-my-project-uuid-0242ac120002
-          db-host: my_postgres_host.host.com
-          db-port: 5432
-          db-username: postgres_username
-          db-password: postgres_password
-          client-id: ClientIDFromYourPDAccount
-          client-secret: ClientSecretFromYourPDAccount
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Google Cloud SDK
+      uses: google-github-actions/[email protected]
+      with:
+        project_id: database-migration-anonymized
+        service_account_key: ${{ secrets.GCP_SA_KEY }}
+        export_default_credentials: true
+
+    - name: Build and Push Docker Image
+      run: |
+        cd ./db
+        docker build -t destination-db .
+        gcloud auth configure-docker
+        docker tag destination-db:latest gcr.io/database-migration-anonymized/destination-db
+        docker push gcr.io/database-migration-anonymized/destination-db
+
+    - name: Deploy to CloudRun
+      run: |
+        gcloud run deploy my-pg-service --image=gcr.io/database-migration-anonymized/destination-db --platform managed --allow-unauthenticated
+  # anonymize:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Anonymize Data
+  #       uses: pvcy/anonymize-project@latest
+  #       with:
+  #         project-id: 4e1213f4-my-project-uuid-0242ac120002
+  #         db-host: my_postgres_host.host.com
+  #         db-port: 5432
+  #         db-username: postgres_username
+  #         db-password: postgres_password
+  #         client-id: ClientIDFromYourPDAccount
+  #         client-secret: ClientSecretFromYourPDAccount
+  # dump-database:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #   - name: Dump database to blob storage
+  #     run: |
+  #       some bash command to call pgdump

.github/workflows/create-preview.yml → .github/workflows/test-migration.yml

@@ -2,6 +2,8 @@ name: Create Preview and Test Migration
 on:
   pull_request:
     branches: [ main ]
+    paths: 
+    - 'users-api/src/migrations/**'
 
 jobs:
   preview:
@@ -11,35 +13,30 @@ jobs:
         uses: okteto/[email protected]
         with:
           token: ${{ secrets.OKTETO_GCP_TOKEN }}
-          url: ${{secrets.OKTETO_GCP_URL}}
+          url: ${{ secrets.OKTETO_GCP_URL }}
 
       - name: Deploy preview environment
         uses: okteto/[email protected]
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          name: pr-${{ github.event.number }}
+          name: anonymize-pr-${{ github.event.number }}
           scope: global
           timeout: 15m
-          variables: "DOPPLER_CONFIG=dev,DOPPLER_TOKEN=${{ secrets.DOPPLER_TOKEN }}"
 
   load-data:
     name: Load data from snapshot
     runs-on: ubuntu-latest
     steps:
       - id: load-data-from-snapshot
         run: |
-          some bash command goes here
+          echo "some bash command goes here"
 
   tests:
-    name: Run integration tests
+    name: Verify migration works
     runs-on: ubuntu-latest
     needs: preview
     steps:
-      - id: run-alembic-tests
-        name: Run Alembic migration tests
-        env:
-          DOPPLER_TOKEN: ${{ secrets.DOPPLER_TOKEN }}
-          DOPPLER_PROJECT: my-big-proj
-          DOPPLER_CONFIG: dev
-        run: doppler run -- pytest test/alembic
+      - id: run-migration
+        name: Run Sequelize migration
+        run: npx sequelize-cli db:migrate

README.md

@@ -1,6 +1,6 @@
-# Anonymizing Data in Ephemeral Environments
+# Stateful Validation of Database Migrations
 
-This is a demo application for ephemeral development environments, such as testing, staging, and sandbox setups. It can be used for reference or as a starting point to build vendor specific architectures that work with Privacy Dynamics. The application is designed to be as simple and basic as possible.
+This application is a demonstration of how to validate database migrations will succeed before being run in production environments. The application uses real, anonymized production data to verify migrations work and don't fail against outlier data.
 
 ## A reference architecture
 The reference architecture we propose is designed specifically for ephemeral development environments. These environments often contain subsets of production data, making it essential to safeguard Personally Identifiable Information (PII) and other sensitive details.
@@ -27,5 +27,5 @@ Once connectivity to the databases is established, a base environment can be use
 ## Getting started
 You can run a single instance of the application locally. All you need to get started is [Docker](https://www.docker.com/) installed on your local machine.
 
-1. Start the application with `docker compose -up --build`. This will seed the PostgreSQL database with data from the JSON file.
+1. Start the application with `docker compose up --build`. This will seed the PostgreSQL database with data from the JSON file.
 2. Navigate to `http://localhost:5000` in your browser.

db/Dockerfile

@@ -0,0 +1,11 @@
+FROM postgres:latest
+
+# SSH
+RUN apt-get update && apt-get install -y openssh-server
+RUN echo "root:Docker!" | chpasswd
+RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN mkdir /var/run/sshd
+
+EXPOSE 22
+
+CMD service ssh start && docker-entrypoint.sh postgres

users-api/src/migrations/20230921200634-create-user.js

@@ -3,7 +3,7 @@
 module.exports = {
   async up(queryInterface, Sequelize) {
     await queryInterface.createTable('Users', {
-      id: {
+      user_id: {
         allowNull: false,
         autoIncrement: true,
         primaryKey: true,
@@ -12,16 +12,33 @@ module.exports = {
       first_name: {
         type: Sequelize.STRING
       },
-      email: {
-        type: Sequelize.STRING
+      last_name: {
+        type: DataTypes.STRING,
+        allowNull: false,
+      },
+      phone: {
+        type: DataTypes.STRING,
+        allowNull: false,
+      },
+      city: {
+        type: DataTypes.STRING,
+        allowNull: false,
+      },
+      state: {
+        type: DataTypes.STRING,
+        allowNull: false,
+      },
+      zip: {
+        type: DataTypes.STRING,
+        allowNull: false,
       },
-      createdAt: {
+      gender: {
+        type: DataTypes.STRING,
         allowNull: false,
-        type: Sequelize.DATE
       },
-      updatedAt: {
+      age: {
+        type: DataTypes.INTEGER,
         allowNull: false,
-        type: Sequelize.DATE
       }
     });
   },