Rework QWC Config DB setup:

* Move migrations to qwc-base-db
* Introduce qwc-base-db-migrate image to migrate dockerized or external config DB
* Demo data will be initialized by an optional setup script in qwc-docker

.github/workflows/qwc-base-db.yml

@@ -1,29 +1,40 @@
-name: Publish qwc-base-db docker image
+name: Publish qwc-base-db docker images
 
 on: [push]
 
 jobs:
 
     build:
       runs-on: ubuntu-latest
+      if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
       steps:
 
       - uses: actions/checkout@master
 
       - name: Get version tag
         id: get_tag
         run: |
-          if [ ${{ startsWith(github.ref, 'refs/tags/') }} = true ]; then
-            echo ::set-output name=tag::${GITHUB_REF:10}
+          if [ ${{ endsWith(github.ref, '-lts') }} = true ]; then
+            echo "tag=latest-lts,${GITHUB_REF:10}" >>$GITHUB_OUTPUT
           else
-            echo ::set-output name=tag::
+            echo "tag=latest,${GITHUB_REF:10}" >>$GITHUB_OUTPUT
           fi
 
-      - name: Build and publish docker image
+      - name: Build and publish qwc-base-db docker image
         uses: elgohr/Publish-Docker-Github-Action@v5
         with:
           name: sourcepole/qwc-base-db
           username: ${{ secrets.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-          tags: "latest,${{ steps.get_tag.outputs.tag }}"
+          tags: "${{ steps.get_tag.outputs.tag }}"
           workdir: .
+
+      - name: Build and publish qwc-base-db-migrate docker image
+        uses: elgohr/Publish-Docker-Github-Action@v5
+        with:
+          name: sourcepole/qwc-base-db-migrate
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          tags: "${{ steps.get_tag.outputs.tag }}"
+          workdir: .
+          dockerfile: Dockerfile.migrate

Dockerfile

@@ -1,59 +1,21 @@
-# QWC Services base DB
-#
-# This is mostly the same as setup-external-db.sh
-#
-#                     PLEASE KEEP IN SYNC
-#
-# container ready to serve data from /data volume:
-#
-# - includes postgres server with postgis
-# - *on start* will check whether migration is needed.
-#   If migration is needed:
-#   - will set up config-db
-#
-# Please set ALEMBIC_VERSION to a specific commit hash
-# in the docker image runtime environment, if you want
-# to check out and run a different version of
-# qwc-config-db migrations than those from `head`.
-# See the `run-migrations.sh`.
+# A postgres DB with the minimal QWC config DB setup
 
-FROM postgres:13
+FROM postgres:15
 
-ENV DEBIAN_FRONTEND=noninteractive
+ENV PGDATA=/var/lib/postgresql/docker
+ENV POSTGRES_PASSWORD=
 
-ARG PG_MAJOR=13
+ARG PG_MAJOR=15
 ARG POSTGIS_VERSION=3
 
-ARG GIT_REPO=https://github.com/qwc-services/qwc-config-db.git
-
-ENV PGSERVICEFILE=/tmp/.pg_service.conf
-
-COPY install-postgis.sh install-alembic-and-clone-qwc-config-db.sh /usr/local/bin/
-RUN  cd /usr/local/bin && \
-     chmod +x install-postgis.sh install-alembic-and-clone-qwc-config-db.sh
-
+# Install postgis
 RUN \
-    export PATH=/usr/local/bin:/usr/bin:/bin && \
     apt-get update && \
-    apt-get upgrade -y && \
-    /usr/local/bin/install-postgis.sh $PG_MAJOR $POSTGIS_VERSION && \
-    /usr/local/bin/install-alembic-and-clone-qwc-config-db.sh $GIT_REPO && \
+    apt-get install -y curl postgresql-$PG_MAJOR-postgis-$POSTGIS_VERSION postgresql-$PG_MAJOR-postgis-$POSTGIS_VERSION-scripts && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-#RUN localedef -i de_CH -c -f UTF-8 -A /usr/share/locale/locale.alias de_CH.UTF-8
-#ENV LANG de_CH.utf8
-
-
-# setup database
+# Setup database
 # script to create DB, roles, grants etc
 COPY setup-roles-and-db.sh /docker-entrypoint-initdb.d/0_setup-db.sh
-
-# script to create tables
-COPY run-migrations.sh /docker-entrypoint-initdb.d/1_run-migrations.sh
-
-RUN chmod +x /docker-entrypoint-initdb.d/*.sh
-RUN cp -a /usr/local/bin/docker-entrypoint.sh /tmp/docker-entrypoint.sh
-
-ENV PGDATA /var/lib/postgresql/docker
-ENV POSTGRES_PASSWORD U6ZqsEdBmrER
+RUN chmod +x /docker-entrypoint-initdb.d/0_setup-db.sh

Dockerfile.migrate

@@ -0,0 +1,14 @@
+# Helper image to run qwc-config-db migrations
+FROM alpine:3.18
+
+ENV PGSERVICEFILE=/tmp/pg_service.conf
+ENV ALEMBIC_VERSION=head
+
+RUN apk add --no-cache --update py3-alembic py3-psycopg2
+
+COPY pg_service.conf /tmp/pg_service.conf
+COPY alembic.ini /tmp/alembic.ini
+COPY alembic /tmp/alembic
+
+WORKDIR /tmp
+ENTRYPOINT ["/bin/sh", "-c", "PGSERVICE=qwc_configdb alembic upgrade ${ALEMBIC_VERSION}"]

README.md

@@ -1,76 +1,71 @@
 [![CI](https://github.com/qwc-services/qwc-base-db/actions/workflows/qwc-base-db.yml/badge.svg)](https://github.com/qwc-services/qwc-base-db/actions)
 [![docker](https://img.shields.io/docker/v/sourcepole/qwc-base-db?label=qwc-base-db%20image&sort=semver)](https://hub.docker.com/r/sourcepole/qwc-base-db)
 
-QWC base DB
-============
-
-This repository creates a Docker image with a postgres server
-that can be used by QWC.
-
-The image contains the postgres server, the postgis extension,
-and scripts to set up the `qwc_configdb` configuration database
-and scripts to insert the demo data.
+QWC Base DB
+===========
+
+This repository creates two Docker images:
+
+* `qwc-base-db`: Image with a postgis server and the minimal schemas and roles for the QWC Config DB
+* `qwc-base-db-migrate`: Image containing `alembic` and the migrations to update the QWC Config DB to the latest schema
+
+These images are designed to be configured in a `docker-compose.yml` as follows (minimal configuration):
+
+```yml
+  qwc-postgis:
+    image: sourcepole/qwc-base-db:<version>
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 10s
+    environment:
+      POSTGRES_PASSWORD: '' # TODO: Set your postgres password here!
+    volumes:
+     - ./volumes/db:/var/lib/postgresql/docker
+
+  qwc-config-db-migrate:
+    image: sourcepole/qwc-base-db-migrate:<version>
+    depends_on:
+      qwc-postgis:
+        condition: service_healthy
+```
 
-When the image is run, then it checks whether the directory
-referenced by the `$PGDATA` ENV variable is empty. If that's
-the case then it will proceed with setting up the
-`qwc_configdb` DB and adding demo data to the `qwc_demo` DB.
+Note:
 
-The default value for `$PGDATA` ENV is `/var/lib/postgresql/docker`.
+* **You need to set a non-empty `POSTGRES_PASSWORD` ENV variable**.
+* For persistent storage, mount folder volume to `/var/lib/postgresql/docker`.
 
-The https://github.com/qwc-services/qwc-demo-db repository
-uses this image to create another container image with a
-ready to use database filled with demo data for easy trying
-out QWC2.
+When the `qwc-postgis` image is run, then it checks whether `/var/lib/postgresql/docker` is empty.
+If that's the case then it will proceed with setting up the
+`qwc_configdb` DB.
 
-Usage
------
+The `qwc-config-db-migrate` image will run `qwc-postgis` is up, and will apply all available migrations to the Config DB.
 
-### The Docker container image
+# Keeping the Config DB up-to-date
 
-The qwc-base-db Docker image is based on the
-[official Postgres Docker container image](https://hub.docker.com/_/postgres/).
-The mechanisms discussed below are based on those
-provided by the Postgres Docker image. Please
-see at the mentioned link.
+To keep the Config DB up to date, it is sufficient to update the `qwc-base-db-migrate` image version to the latest available version.
 
-The qwc-base-db Docker container image meant to be used like this:
+Migrations will be applied automatically, if necessary, whenever the Docker application is started.
 
-    $ cat docker-compose.yml
-    version: '2.1'
-    services:
-      qwc-postgis:
-        image: sourcepole/qwc-base-db
-        environment:
-          - ALEMBIC_VERSION=head
-          - PGDATA=/var/lib/postgresql/data/pgdata
-        healthcheck:
-          test: ["CMD-SHELL", "pg_isready -U postgres"]
-          interval: 10s
-        ports:
-         - "0.0.0.0:5432:5432"
-        volumes:
-          - ./volumes/postgres-data:/var/lib/postgresql/data
-    [...]
+To upgrade to a migration different than `head`, set the `ALEMBIC_VERSION` ENV variable.
 
-Upon start, the container will check whether `/var/lib/postgresql/data`
-is empty.
+# Managing an external Config DB
 
-#### starting with an empty `/var/lib/postgresql/data` volume
+You can use an external DB instead of the `qwc-base-db` dockerized DB.
 
-If `/var/lib/postgresql/data` is empty, then postgres will initialize it
-and execute scripts under `/docker-entrypoint-initdb.d` that will set up
-a `qwc_demo` DB and fill it with demo data.
+To set up the external DB, run the SQL commands in `setup-roles-and-db.sh` on your external DB.
 
-#### starting with an non-empty `/var/lib/postgresql/data` volume
+To apply the migrations, both as part as the initial setup and subsequently to keep the Config DB up-to-date:
 
-If `/var/lib/postgresql/data` is *NOT* empty, such as when:
+* Modify the `qwc_configdb` connection in `pg_service.conf` with the connection information to your external DB
+* Configure the `qwc-config-db-migrate` image mounting the modified `pg_service.conf`:
 
-* the container had already been started in the past and had already
-  initialized the the DB inside the provided volume or
+```yml
+  qwc-config-db-migrate:
+    image: sourcepole/qwc-base-db-migrate:<version>
+  volumes:
+    ./pg_service.conf:/tmp/pg_service.conf:ro
+```
 
-* the admin has attached a volume to the container that had some
-  other postgres database in it
+# DB Schema overview
 
-then postgres will just start and try to use the data that is already
-under `/var/lib/postgresql/data`.
+![er-diagram](er-diagram.png)

alembic.ini

@@ -0,0 +1,75 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = postgresql:///?service=qwc_configdb
+version_table_schema = qwc_config
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S