From a3ced1ed5917402c4487e8961fb6245b988c8774 Mon Sep 17 00:00:00 2001 From: Francesco Ballarin Date: Mon, 12 Feb 2024 10:18:52 +0100 Subject: [PATCH] Database creation now happens inside docker entrypoint --- .dockerignore | 2 - docker/.gitignore | 3 +- docker/Dockerfile | 23 +++++--- docker/README.md | 21 +++----- docker/docker_create_container.sh | 5 +- docker/docker_create_database.sh | 54 ------------------- docker/docker_create_image.sh | 8 +-- docker/docker_create_secrets.sh | 12 ++--- docker/docker_start.sh | 3 ++ docker/entrypoint.sh | 87 +++++++++++++++++++++++++++++++ patches/turing/apply_patches.sh | 2 +- 11 files changed, 127 insertions(+), 93 deletions(-) delete mode 100644 .dockerignore delete mode 100644 docker/docker_create_database.sh create mode 100755 docker/entrypoint.sh diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 771eaaa..0000000 --- a/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -.git -docker diff --git a/docker/.gitignore b/docker/.gitignore index 10e6187..4e5b556 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1,5 +1,4 @@ .docker_container_id -.docker_postgres_data_directory +.docker_django_secret_key .docker_postgres_password -.docker_secret_key .docker_volume_id diff --git a/docker/Dockerfile b/docker/Dockerfile index 63bfa6a..4115664 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ FROM debian:12 # also in the shell scripts stored in this directory. MAINTAINER Francesco Ballarin -ARG SECRET_KEY +ARG DJANGO_SECRET_KEY ARG POSTGRES_PASSWORD WORKDIR /root @@ -17,18 +17,19 @@ WORKDIR /root RUN apt update -y -q && \ apt install -y -qq curl net-tools python3-pip sudo unzip vim wget -COPY . . - # Part 1: turing and its runtime dependencies +COPY patches patches +COPY turing turing + RUN apt install -y -qq postgresql postgresql-client postgresql-contrib && \ echo "$POSTGRES_PASSWORD\n$POSTGRES_PASSWORD" | passwd postgres RUN bash patches/turing/apply_patches.sh && \ python3 -m pip install --break-system-packages -r turing/requirements.txt -RUN cat <> /root/turing/Turing/settings.ini +RUN cat < /root/turing/Turing/settings.ini [settings] -SECRET_KEY=$SECRET_KEY +SECRET_KEY=$DJANGO_SECRET_KEY DEBUG=False DEV_MODE=False ALLOWED_HOSTS=* @@ -42,7 +43,7 @@ EOF RUN cd turing && \ python3 manage.py collectstatic -# Part 2: test dependencies +# Part 2: turing and its test dependencies RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list' && \ apt update -y -q && \ @@ -56,5 +57,11 @@ RUN LATEST_STABLE_RELEASE=$(curl -sS https://googlechromelabs.github.io/chrome-f RUN python3 -m pip install --break-system-packages -r turing/requirements-dev.txt -# The end: run server on docker container start -CMD service postgresql start && cd turing && python3 manage.py runserver 0.0.0.0:8080 +# Part 3: mathrace-interaction +COPY mathrace_interaction mathrace_interaction + +# Part 4: set up entrypoint that: +# * creates and initialize databases on first run, +# * starts the server +COPY docker/entrypoint.sh /usr/local/bin/docker-entrypoint.sh +ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/README.md b/docker/README.md index 95525dd..9149bf0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -16,23 +16,19 @@ git clone --recurse-submodules https://github.com/dmf-unicatt/turing-dmf.git ``` cd turing-dmf/docker ``` -2. Create a docker volume that will contain the database: +2. Create a secret key for `django` and a password for `postgresql`: ``` -bash docker_create_volume.sh +bash docker_create_secrets.sh ``` -3. Create a secret key for `django` and a password for `postgresql`: +3. Create a docker volume that will contain the database: ``` -bash docker_create_secrets.sh +bash docker_create_volume.sh ``` 4. Create a `turing-dmf:latest` docker image based on the current **Turing @ DMF** repository: ``` bash docker_create_image.sh ``` -5. Create a database for **Turing**: -``` -bash docker_create_database.sh -``` -6. Create a docker container based on the current `turing-dmf:latest` docker image: +5. Create a docker container based on the current `turing-dmf:latest` docker image: ``` bash docker_create_container.sh ``` @@ -47,7 +43,7 @@ cd turing-dmf/docker ``` bash docker_start.sh ``` -**Turing** will be available at `https://host-server:8080`. +**Turing** will be available at `https://host-server:8080`. The database will be initialized upon the first run. 3. Attach a terminal to the running docker container ``` bash docker_terminal.sh @@ -75,7 +71,8 @@ bash docker_destroy_container.sh ``` Note that: - the container must not be running in order to destroy it. -- the database will be preserved upon destroying the container. +- secrets will be preserved upon destroying the container, and must not be regenarated. +- database volume (including the database itself) will be preserved upon the destroying the container, and must not be regenerated. 3. Refresh the `turing-dmf:latest` docker image based on the updated **Turing @ DMF** repository: ``` bash docker_create_image.sh @@ -84,5 +81,3 @@ bash docker_create_image.sh ``` bash docker_create_container.sh ``` - -Note that one must not run the command to create the docker volume containing the database, nor the command to create the database itself, since they are preserved upon destroying the former container. diff --git a/docker/docker_create_container.sh b/docker/docker_create_container.sh index 74c124d..b918152 100644 --- a/docker/docker_create_container.sh +++ b/docker/docker_create_container.sh @@ -10,9 +10,6 @@ set -e VOLUME_ID_FILE=".docker_volume_id" VOLUME_ID=$(cat "${VOLUME_ID_FILE}") -POSTGRES_DATA_DIRECTORY_FILE=".docker_postgres_data_directory" -POSTGRES_DATA_DIRECTORY=$(cat "${POSTGRES_DATA_DIRECTORY_FILE}") - CONTAINER_ID_FILE=".docker_container_id" if [[ -f "${CONTAINER_ID_FILE}" ]]; then echo "A container already exists!" @@ -25,6 +22,6 @@ else docker network create --subnet=10.200.1.0/24 turing-dmf-network # Start docker container on a fixed IP address. The corresponding mac address is generated following # https://maclookup.app/faq/how-do-i-identify-the-mac-address-of-a-docker-container-interface - CONTAINER_ID=$(docker create --net turing-dmf-network --ip 10.200.1.23 --mac-address 02:42:0a:c8:01:17 -p 8080:8080 -v /tmp/shared-turing-dmf:/tmp/shared-turing-dmf -v ${VOLUME_ID}:${POSTGRES_DATA_DIRECTORY} -e DOCKERHOSTNAME=$(cat /etc/hostname) turing-dmf:latest) + CONTAINER_ID=$(docker create --net turing-dmf-network --ip 10.200.1.23 --mac-address 02:42:0a:c8:01:17 -p 8080:8080 -v /tmp/shared-turing-dmf:/tmp/shared-turing-dmf -v ${VOLUME_ID}:/mnt/postgres_data_directory -e DOCKERHOSTNAME=$(cat /etc/hostname) turing-dmf:latest) echo ${CONTAINER_ID} > ${CONTAINER_ID_FILE} fi diff --git a/docker/docker_create_database.sh b/docker/docker_create_database.sh deleted file mode 100644 index af8e0a9..0000000 --- a/docker/docker_create_database.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 by the Turing @ DMF authors -# -# This file is part of Turing @ DMF. -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -set -e - -# Outputs of previous steeps -VOLUME_ID_FILE=".docker_volume_id" -VOLUME_ID=$(cat "${VOLUME_ID_FILE}") - -POSTGRES_PASSWORD_FILE=".docker_postgres_password" -POSTGRES_PASSWORD=$(cat "${POSTGRES_PASSWORD_FILE}") - -# Determine if the script was already run -POSTGRES_INITIALIZED_FILE=".docker_postgres_initialized" -if [[ -f "${POSTGRES_INITIALIZED_FILE}" ]]; then - POSTGRES_INITIALIZED=$(cat "${POSTGRES_INITIALIZED_FILE}") - if [[ ${POSTGRES_INITIALIZED} == ${VOLUME_ID} ]]; then - echo "The database in volume ${VOLUME_ID} was already initialized!" - exit 1 - fi -fi - -# Write out the postgresql data as of debian:12 -POSTGRES_DATA_DIRECTORY_FILE=".docker_postgres_data_directory" -POSTGRES_DATA_DIRECTORY="/var/lib/postgresql/15/main" -echo ${POSTGRES_DATA_DIRECTORY} > ${POSTGRES_DATA_DIRECTORY_FILE} - -# Use a clean container to create the default postgresql data, since turing-dmf:latest cannot be used -# because it already expects the database to exist -docker run --rm -v ${VOLUME_ID}:${POSTGRES_DATA_DIRECTORY} -e POSTGRES_PASSWORD=${POSTGRES_PASSWORD} debian:12 /bin/bash -c "\ - apt update -y -q && \ - apt install -y -qq postgresql sudo && \ - service postgresql start && \ - echo \$(sudo -u postgres psql -c \"SHOW data_directory;\") && \ - sudo -u postgres psql -c \"ALTER USER postgres WITH PASSWORD '${POSTGRES_PASSWORD}';\" && \ - sudo -u postgres createdb turing-dmf-db -" - -# Once the database has been created, ask turing to initialize it -docker run --rm -v ${VOLUME_ID}:${POSTGRES_DATA_DIRECTORY} turing-dmf:latest /bin/bash -c "\ - service postgresql start && \ - cd turing && \ - python3 manage.py makemigrations && \ - python3 manage.py makemigrations engine && \ - python3 manage.py migrate && \ - DJANGO_SUPERUSER_EMAIL=admin@admin.it DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_PASSWORD=admin python3 manage.py createsuperuser --no-input; -" - -# Mark the database in this volume as initialized -echo ${VOLUME_ID} > ${POSTGRES_INITIALIZED_FILE} diff --git a/docker/docker_create_image.sh b/docker/docker_create_image.sh index 347999a..2ec40f5 100644 --- a/docker/docker_create_image.sh +++ b/docker/docker_create_image.sh @@ -10,10 +10,12 @@ set -e # Do not run any further if we are not connected to the internet wget -q --spider https://www.google.com -SECRET_KEY_FILE=".docker_secret_key" -SECRET_KEY=$(cat "${SECRET_KEY_FILE}") +# Read in secrets +DJANGO_SECRET_KEY_FILE=".docker_django_secret_key" +DJANGO_SECRET_KEY=$(cat "${DJANGO_SECRET_KEY_FILE}") POSTGRES_PASSWORD_FILE=".docker_postgres_password" POSTGRES_PASSWORD=$(cat "${POSTGRES_PASSWORD_FILE}") -docker build --pull --build-arg SECRET_KEY=${SECRET_KEY} --build-arg POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -t turing-dmf:latest -f Dockerfile .. +# Build image +docker build --pull --build-arg DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY} --build-arg POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -t turing-dmf:latest -f Dockerfile .. diff --git a/docker/docker_create_secrets.sh b/docker/docker_create_secrets.sh index 4cd0d8b..3e53222 100644 --- a/docker/docker_create_secrets.sh +++ b/docker/docker_create_secrets.sh @@ -7,14 +7,14 @@ set -e -SECRET_KEY_FILE=".docker_secret_key" -if [[ -f "${SECRET_KEY_FILE}" ]]; then - echo "A secret key already exists!" - echo "If you want to destroy it and create a new one, please remove the ${SECRET_KEY_FILE} file" +DJANGO_SECRET_KEY_FILE=".docker_django_secret_key" +if [[ -f "${DJANGO_SECRET_KEY_FILE}" ]]; then + echo "A django secret key already exists!" + echo "If you want to destroy it and create a new one, please remove the ${DJANGO_SECRET_KEY_FILE} file" exit 1 else - SECRET_KEY=$(cat /dev/urandom | tr -dc 'abcdefghijklmnopqrstuvwxyz0123456789!@#$^&*-_=+' | head -c 50; echo) - echo ${SECRET_KEY} > ${SECRET_KEY_FILE} + DJANGO_SECRET_KEY=$(cat /dev/urandom | tr -dc 'abcdefghijklmnopqrstuvwxyz0123456789!@#$^&*-_=+' | head -c 50; echo) + echo ${DJANGO_SECRET_KEY} > ${DJANGO_SECRET_KEY_FILE} fi POSTGRES_PASSWORD_FILE=".docker_postgres_password" diff --git a/docker/docker_start.sh b/docker/docker_start.sh index db2e0c9..e20204e 100644 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -15,4 +15,7 @@ if [ "$( docker container inspect -f '{{.State.Running}}' ${CONTAINER_ID} )" == exit 1 else docker start ${CONTAINER_ID} + # Entrypoint outputs are not reported to stdout, but logged: fetch them and print them to stdout, after waiting + # a reasonable amount of time for the entrypoint to be done + sleep 5 && docker logs --since "6s" --timestamps ${CONTAINER_ID} fi diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..964d1fa --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Copyright (C) 2024 by the Turing @ DMF authors +# +# This file is part of Turing @ DMF. +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -e + +# Installed postgres version +POSTGRES_VERSION=15 + +# Get the value of docker build args from the settings file +POSTGRES_DATABASE_NAME=$(sed -n -e "s/^RDS_DB_NAME=//p" /root/turing/Turing/settings.ini) +if [[ "${POSTGRES_DATABASE_NAME}" != *"-db" ]]; then + echo "Expected database name ${POSTGRES_DATABASE_NAME} to end with -db" + exit 1 +fi +POSTGRES_PASSWORD=$(sed -n -e "s/^RDS_PASSWORD=//p" /root/turing/Turing/settings.ini) + +# Hardcode values of docker create args +POSTGRES_CLUSTER_NAME=${POSTGRES_DATABASE_NAME/-db/-cluster} +if [[ "${POSTGRES_CLUSTER_NAME}" != *"-cluster" ]]; then + echo "Expected cluster name ${POSTGRES_CLUSTER_NAME} to end with -cluster" + exit 1 +fi +POSTGRES_CLUSTER_DATA_DIRECTORY=/mnt/postgres_data_directory + +# Create a new postgres cluster with data directory that matches the volume mounted in docker_create_container.sh, +# if not already done previously +# Note that the marker file .postgres_cluster_created cannot be put in ${POSTGRES_CLUSTER_DATA_DIRECTORY}, +# because the cluster needs to be re-created in every container. This is safe upon container destruction because +# postgres data direcory will not be cleared out when creating the cluster in a new container. +POSTGRES_CLUSTER_CREATED_FILE=/root/turing/.postgres_cluster_created +if [[ ! -f ${POSTGRES_CLUSTER_CREATED_FILE} ]]; then + echo "Creating a new postgres cluster" + pg_dropcluster ${POSTGRES_VERSION} main + pg_createcluster ${POSTGRES_VERSION} --datadir=${POSTGRES_CLUSTER_DATA_DIRECTORY} ${POSTGRES_CLUSTER_NAME} -- -E UTF8 --locale=C.utf8 --lc-messages=C + cp /etc/postgresql/${POSTGRES_VERSION}/${POSTGRES_CLUSTER_NAME}/*.conf ${POSTGRES_CLUSTER_DATA_DIRECTORY}/ + touch ${POSTGRES_CLUSTER_CREATED_FILE} +else + echo "Reusing existing postgres cluster" +fi + +# Start postgresql service +echo "Starting postgresql service" +service postgresql start + +# Initialize an empty postgres database, if not already done previously +POSTGRES_DATABASE_INITIALIZED_FILE=${POSTGRES_CLUSTER_DATA_DIRECTORY}/.postgres_database_initialized +if [[ ! -f ${POSTGRES_DATABASE_INITIALIZED_FILE} ]]; then + echo "Initializing an empty postgres database" + sudo -u postgres psql -c "ALTER USER postgres WITH PASSWORD '${POSTGRES_PASSWORD}';" + sudo -u postgres createdb ${POSTGRES_DATABASE_NAME} + touch ${POSTGRES_DATABASE_INITIALIZED_FILE} +else + echo "Reusing existing postgres database" +fi + +# Ask turing to initialize the django database, if not already done previously +DJANGO_DATABASE_MIGRATED_FILE=${POSTGRES_CLUSTER_DATA_DIRECTORY}/.django_database_migrated +if [[ ! -f ${DJANGO_DATABASE_MIGRATED_FILE} ]]; then + echo "Initializing django database" + cd /root/turing + python3 manage.py makemigrations + python3 manage.py makemigrations engine + python3 manage.py migrate + touch ${DJANGO_DATABASE_MIGRATED_FILE} +else + echo "Not initializing again django database" +fi + +# Add a default administration user to the django database, if not already done previously +DJANGO_ADMIN_INITIALIZED_FILE=${POSTGRES_CLUSTER_DATA_DIRECTORY}/.django_admin_initialized +if [[ ! -f ${DJANGO_ADMIN_INITIALIZED_FILE} ]]; then + echo "Initialize the default django administrator user with username admin and password admin. Make sure to change both the username and the password as soon as possible!" + cd /root/turing + DJANGO_SUPERUSER_EMAIL=admin@admin.it DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_PASSWORD=admin python3 manage.py createsuperuser --no-input + touch ${DJANGO_ADMIN_INITIALIZED_FILE} +else + echo "Not initializing again default django administrator" +fi + +# Start the server +echo "Starting the server" +cd /root/turing +python3 manage.py runserver 0.0.0.0:8080 diff --git a/patches/turing/apply_patches.sh b/patches/turing/apply_patches.sh index 9f92c5b..af4a707 100644 --- a/patches/turing/apply_patches.sh +++ b/patches/turing/apply_patches.sh @@ -7,7 +7,7 @@ set -e -if [[ ! -e "mathrace_interaction" || ! -e "patches" || ! -e "turing" ]]; then +if [[ ! -e "patches" || ! -e "turing" ]]; then echo "This script must be run as 'bash patches/turing/apply_patches.sh' from the top level directory of the repository" exit 1 fi