Add SQL-based migration script, with migration dir (#878)

* refactor: remove alembic from repo (dep + refs)

* build: optimise ci dockerfile, use system pip pkgs

* build: fix mv quoting for ci img pkg move

* build: add base fmtm db schema

* build: improve entrypoint wait_for_db func

* build: update compose depends, run app aafter migrations

* build: add bash script to handle migrations at startup

* refactor: remove password field from users table (migration)

* build: update schema check, check if public.projects exists

* build: syntax error for migrations table, formatting script

* build: return success on schema preexist or create

* build: move success notice to wrapper pgsql func (migrations)

* docs: add info on manual db migration

docker-compose.deploy.yml

@@ -91,6 +91,7 @@ services:
       - fmtm_tiles:/opt/tiles
     depends_on:
       - fmtm-db
+      - migrations
       - traefik
     env_file:
       - .env
@@ -109,13 +110,12 @@ services:
     image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
     container_name: fmtm_migrations
     depends_on:
-      - api
+      - fmtm-db
     env_file:
       - .env
     networks:
       - fmtm-net
     entrypoint: ["/migrate-entrypoint.sh"]
-    command: ["alembic", "upgrade", "head"]
     restart: "on-failure:3"
 
   ui:

docker-compose.noodk.yml

@@ -58,6 +58,7 @@ services:
       - ./src/backend/app:/opt/app
     depends_on:
       - fmtm-db
+      - migrations
     env_file:
       - .env
     ports:
@@ -71,13 +72,12 @@ services:
     image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
     container_name: fmtm_migrations
     depends_on:
-      - api
+      - fmtm-db
     env_file:
       - .env
     networks:
       - fmtm-net
     entrypoint: ["/migrate-entrypoint.sh"]
-    command: ["alembic", "upgrade", "head"]
     restart: "on-failure:3"
 
   ui:

docker-compose.yml

@@ -64,6 +64,7 @@ services:
       # - ../osm-fieldwork/osm_fieldwork:/home/appuser/.local/lib/python3.10/site-packages/osm_fieldwork
     depends_on:
       - fmtm-db
+      - migrations
       - central-proxy
     env_file:
       - .env
@@ -78,13 +79,12 @@ services:
     image: "ghcr.io/hotosm/fmtm/backend:debug"
     container_name: fmtm_migrations
     depends_on:
-      - api
+      - fmtm-db
     env_file:
       - .env
     networks:
       - fmtm-dev
     entrypoint: ["/migrate-entrypoint.sh"]
-    command: ["alembic", "upgrade", "head"]
     restart: "on-failure:3"
 
   ui:

docs/dev/Database-Tips.md

@@ -1,122 +1,196 @@
-# Access the database (psql)
+# Database Tips
 
-**Option 1** (when the docker container is running) use this command to access it through the local psql using the below command:
+## Access the database (psql)
 
-    `psql -d fmtm -U fmtm -h localhost`
+### Option 1
 
-**Option 2** (when running the database in Docker) use this command to access the
-PostgreSQL shell inside the fmtm-db-1 container and interact with the fmtm database
-using the psql command-line interface:
+Access the database container using psql on your local machine:
 
-    `docker exec -it fmtm-db-1 psql -U fmtm fmtm`
+```bash
+psql -d fmtm -U fmtm -h localhost
+```
 
-And then connect to the database using this command :
+### Option 2
 
-    `\c fmtm`
+Access a PostgreSQL shell inside the fmtm_db container:
 
-## To access the fmtm database using psql, follow the instructions below
+```bash
+docker exec -it fmtm_db psql -U fmtm fmtm
+```
 
-### A few helpful psql commands
+And then connect to the database using this command:
 
-- Open a terminal window and run the following command:
+```bash
+\c fmtm
+```
 
-        docker exec -it fmtm-db-1 psql -U fmtm fmtm
-
-  This will open the psql command-line interface and connect you to the fmtm database.
-
-- Once connected to the fmtm database, you can switch to a different database using the command:
-
-        \c dbname
-
-  Replace "dbname" with the name of the database you want to switch to. forexample `\c fmtm`
+## A few helpful psql commands
 
 - You can list all the databases using the command:
 
-        \l
+```bash
+\l
+```
 
 - To list all the schemas of the currently connected database, use the command:
 
-        \dn
+```bash
+\dn
+```
 
 - To list all the functions in the current database, use the command:
 
-        \df
+```bash
+\df
+```
 
 - To list all the views in the current database, use the command:
 
-        \dv
+```bash
+\dv
+```
 
 - To list all the users and roles, use the command:
 
-        \du
+```bash
+\du
+```
 
 - To list all the tables in the current database, use the command:
 
-        \dt
+```bash
+\dt
+```
 
 - To describe a table, use the command:
 
-        \d table_name
+```bash
+\d table_name
+```
 
-  Replace "table_name" with the name of the table you want to describe.
+Replace "table_name" with the name of the table you want to describe.
 
 - To execute the last command again, use the command:
 
-        \g
+```bash
+\g
+```
 
 - To view your command history, use the command:
 
-        \s
+```bash
+\s
+```
 
 - To save your command history to a file, use the command:
 
-        \s filename
+```bash
+\s filename
+```
 
-  Replace "filename" with the name of the file you want to save the command history to.
+Replace "filename" with the name of the file you
+want to save the command history to.
 
 - To execute commands from a file, use the command:
 
-        \i filename
+```bash
+\i filename
+```
 
-  Replace "filename" with the name of the file containing the commands you want to execute.
+Replace "filename" with the name of the file
+containing the commands you want to execute.
 
 - To view a list of all psql commands, use the command:
 
-        \?
+```bash
+\?
+```
 
 - To view help for a specific command, use the command:
 
-        \h command_name
+```bash
+\h command_name
+```
 
-  Replace "command_name" with the name of the command you want help with.
+Replace "command_name" with the name of the command you want help with.
 
 - To exit psql, use the command:
 
-        \q
+```bash
+\q
+```
 
 **Note:** If you make a change, don't forget to commit the change!
 
-# Migrations
-
-Migrations are a way to manage changes to the database schema over time. We haven't yet implemented migrations in fmtm, but if you need to drop all tables, you can use the following commands while connected to the fmtm database:
-
-If you need to drop all tables, connect to fmtm and...
-
-    drop table mapping_issue_categories cascade;
-    drop table organisation_managers cascade;
-    drop table organisations  cascade;
-    drop table project_allowed_users cascade;
-    drop table project_chat cascade;
-    drop table project_info cascade;
-    drop table project_teams cascade;
-    drop table projects cascade;
-    drop table task_history cascade;
-    drop table task_invalidation_history cascade;
-    drop table task_mapping_issues cascade;
-    drop table tasks cascade;
-    drop table teams cascade;
-    drop table user_licenses cascade;
-    drop table users cascade;
-    drop table x_form cascade;
-
-**Note:** Remember to use caution when dropping tables, as this will permanently delete all data in those tables. If you make any changes to the database, be sure to commit them to ensure that they are saved.
+## Migrations
+
+- Migrations are a way to manage changes to the database schema over time.
+- They are handled automatically by a management script when FMTM starts up.
+- Individual SQL migration scripts are placed in the `src/backend/migrations` dir.
+  - These should be idempotent, i.e. can run over and over without causing errors.
+  - There should also be a commented out SQL script for how to revert the migration.
+  - Scripts should be named sequentially,
+    i.e. the first is 001-some-migration.sql,
+    then they increment by one.
+  - Example `000-remove-user-password.sql`:
+
+```bash
+-- ## Migration to remove password field from public.users (replaced with OSM OAuth)
+
+
+-- ## Apply Migration
+-- Start a transaction
+BEGIN;
+-- Drop the 'password' column if it exists
+ALTER TABLE IF EXISTS public.users
+DROP COLUMN IF EXISTS password;
+-- Commit the transaction
+COMMIT;
+
+
+-- ## Revert Migration (comment above, uncomment below)
+-- -- Start a transaction
+-- BEGIN;
+-- -- Add the 'password' column back if it doesn't exist
+-- ALTER TABLE public.users
+-- ADD COLUMN IF NOT EXISTS password character varying;
+-- -- Commit the transaction
+-- COMMIT;
+```
+
+- When the docker compose stack starts,
+  an additional container starts up and runs a bash script once.
+- The script generates a _table_ called `migrations`,
+  which simply tracks the script name and execution date.
+- The `migrations` _directory_ is scanned for new files,
+  and if there is no record in the database of being applied,
+  the migration is applied.
+
+### Running Migrations Manually
+
+If for any reason you need to run migrations manually,
+there are a few options:
+
+#### Restart the migrations container
+
+```bash
+docker compose restart migrations
+```
+
+#### Run the migration script in docker
+
+This runs inside the backend container:
+
+```bash
+docker compose exec api bash /migrate-entrypoint.sh`
+```
+
+#### Run the migration script directly
+
+Make sure you have the 4 env vars for the database
+connection set on your machine,
+then run the migration script directly:
+
+```bash
+bash src/backend/migrate-entrypoint.sh
+```

src/backend/.dockerignore

@@ -10,4 +10,3 @@
 !pyproject.toml
 !pdm.lock
 !migrations
-!alembic.ini

src/backend/Dockerfile

@@ -108,7 +108,6 @@ WORKDIR /opt
 # Add app code
 COPY app/ /opt/app/
 COPY migrations/ /opt/migrations/
-COPY alembic.ini /opt/
 # Add non-root user, permissions
 RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \
     && mkdir -p /opt/logs /opt/tiles \
@@ -150,23 +149,24 @@ USER appuser
 FROM runtime as ci
 # Run all ci as root
 USER root
-ENV PATH="/root/.local/bin:$PATH"
-RUN set -ex \
+ARG PYTHON_IMG_TAG
+COPY --from=extract-deps \
+    /opt/python/requirements-ci.txt /opt/python/
+RUN mv /home/appuser/.local/bin/* /usr/local/bin/ \
+    && mv /home/appuser/.local/lib/python${PYTHON_IMG_TAG}/site-packages/* \
+    /usr/local/lib/python${PYTHON_IMG_TAG}/site-packages/ \
+    && set -ex \
     && apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install \
     -y --no-install-recommends \
-        "gosu" \
         "git" \
-    && rm -rf /var/lib/apt/lists/*
-RUN mv /home/appuser/.local /root/.local
-COPY --from=extract-deps \
-    /opt/python/requirements-ci.txt /opt/python/
-RUN pip install --user --upgrade --no-warn-script-location \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip install --upgrade --no-warn-script-location \
     --no-cache-dir -r \
     /opt/python/requirements-ci.txt \
-    && rm -r /opt/python
-# Pre-compile packages to .pyc (init speed gains)
-RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
+    && rm -r /opt/python \
+    # Pre-compile packages to .pyc (init speed gains)
+    && python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
 # Override entrypoint, as not possible in Github action
 ENTRYPOINT [""]
 CMD [""]