Move from sqlite3 to postgres

docker/.gitignore

@@ -1,3 +1,5 @@
 .docker_container_id
+.docker_postgres_data_directory
+.docker_postgres_password
 .docker_secret_key
 .docker_volume_id

docker/Dockerfile

@@ -5,18 +5,24 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 FROM debian:12
+# If upgrading the version number from 12 to the next debian release, make sure to do the same
+# also in the shell scripts stored in this directory.
 MAINTAINER Francesco Ballarin <[email protected]>
 
 ARG SECRET_KEY
+ARG POSTGRES_PASSWORD
 
 WORKDIR /root
 
 RUN apt update -y -q && \
-    apt install -y -qq curl net-tools python3-pip unzip vim wget
+    apt install -y -qq curl net-tools python3-pip sudo unzip vim wget
 
 COPY . .
 
 # Part 1: turing and its runtime dependencies
+RUN apt install -y -qq postgresql postgresql-client postgresql-contrib && \
+    echo "$POSTGRES_PASSWORD\n$POSTGRES_PASSWORD" | passwd postgres
+
 RUN bash patches/turing/apply_patches.sh && \
     python3 -m pip install --break-system-packages -r turing/requirements.txt
 
@@ -27,14 +33,13 @@ DEBUG=False
 DEV_MODE=False
 ALLOWED_HOSTS=*
 INTERNAL_IPS=127.0.0.1
-EMAIL_HOST=
-EMAIL_HOST_USER=
-EMAIL_HOST_PASSWORD=
-REGISTRATION_OPEN=
+RDS_DB_NAME=turing-dmf-db
+RDS_USERNAME=postgres
+RDS_PASSWORD=$POSTGRES_PASSWORD
+RDS_HOSTNAME=localhost
 EOF
 
 RUN cd turing && \
-    ln -s /mnt/database/db.sqlite3 . && \
     python3 manage.py collectstatic
 
 # Part 2: test dependencies
@@ -52,4 +57,4 @@ RUN LATEST_STABLE_RELEASE=$(curl -sS https://googlechromelabs.github.io/chrome-f
 RUN python3 -m pip install --break-system-packages -r turing/requirements-dev.txt
 
 # The end: run server on docker container start
-CMD cd turing && python3 manage.py runserver 0.0.0.0:8080
+CMD service postgresql start && cd turing && python3 manage.py runserver 0.0.0.0:8080

docker/README.md

@@ -20,21 +20,21 @@ cd turing-dmf/docker
 ```
 bash docker_create_volume.sh
 ```
-3. Create a secret key for `django`:
+3. Create a secret key for `django` and a password for `postgresql`:
 ```
-bash docker_create_secret_key.sh
+bash docker_create_secrets.sh
 ```
 4. Create a `turing-dmf:latest` docker image based on the current **Turing @ DMF** repository:
 ```
 bash docker_create_image.sh
 ```
-5. Create a docker container based on the current `turing-dmf:latest` docker image:
+5. Create a database for **Turing**:
 ```
-bash docker_create_container.sh
+bash docker_create_database.sh
 ```
-6. Create a database for **Turing**:
+6. Create a docker container based on the current `turing-dmf:latest` docker image:
 ```
-bash docker_create_database.sh
+bash docker_create_container.sh
 ```
 
 ## 3. Run the docker container
@@ -52,7 +52,11 @@ bash docker_start.sh
 ```
 bash docker_terminal.sh
 ```
-4. Stop the running docker container
+4. Explore the database volume with
+```
+bash docker_explore_volume.sh
+```
+5. Stop the running docker container
 ```
 bash docker_stop.sh
 ```

docker/docker_create_container.sh

@@ -8,13 +8,10 @@
 set -e
 
 VOLUME_ID_FILE=".docker_volume_id"
-if [[ ! -f "${VOLUME_ID_FILE}" ]]; then
-    echo "The database volume does not exist!"
-    echo "Please create it with docker_create_volume.sh"
-    exit 1
-else
-    VOLUME_ID=$(cat "${VOLUME_ID_FILE}")
-fi
+VOLUME_ID=$(cat "${VOLUME_ID_FILE}")
+
+POSTGRES_DATA_DIRECTORY_FILE=".docker_postgres_data_directory"
+POSTGRES_DATA_DIRECTORY=$(cat "${POSTGRES_DATA_DIRECTORY_FILE}")
 
 CONTAINER_ID_FILE=".docker_container_id"
 if [[ -f "${CONTAINER_ID_FILE}" ]]; then
@@ -28,6 +25,6 @@ else
     docker network create --subnet=10.200.1.0/24 turing-dmf-network
     # Start docker container on a fixed IP address. The corresponding mac address is generated following
     # https://maclookup.app/faq/how-do-i-identify-the-mac-address-of-a-docker-container-interface
-    CONTAINER_ID=$(docker create --net turing-dmf-network --ip 10.200.1.23 --mac-address 02:42:0a:c8:01:17 -p 8080:8080 -v /tmp/shared-turing-dmf:/tmp/shared-turing-dmf -v ${VOLUME_ID}:/mnt/database -e DOCKERHOSTNAME=$(cat /etc/hostname) turing-dmf:latest)
+    CONTAINER_ID=$(docker create --net turing-dmf-network --ip 10.200.1.23 --mac-address 02:42:0a:c8:01:17 -p 8080:8080 -v /tmp/shared-turing-dmf:/tmp/shared-turing-dmf -v ${VOLUME_ID}:${POSTGRES_DATA_DIRECTORY} -e DOCKERHOSTNAME=$(cat /etc/hostname) turing-dmf:latest)
     echo ${CONTAINER_ID} > ${CONTAINER_ID_FILE}
 fi

docker/docker_create_database.sh

@@ -7,25 +7,48 @@
 
 set -e
 
+# Outputs of previous steeps
 VOLUME_ID_FILE=".docker_volume_id"
-if [[ ! -f "${VOLUME_ID_FILE}" ]]; then
-    echo "The database volume does not exist!"
-    echo "Please create it with docker_create_volume.sh"
-    exit 1
-else
-    VOLUME_ID=$(cat "${VOLUME_ID_FILE}")
+VOLUME_ID=$(cat "${VOLUME_ID_FILE}")
+
+POSTGRES_PASSWORD_FILE=".docker_postgres_password"
+POSTGRES_PASSWORD=$(cat "${POSTGRES_PASSWORD_FILE}")
+
+# Determine if the script was already run
+POSTGRES_INITIALIZED_FILE=".docker_postgres_initialized"
+if [[ -f "${POSTGRES_INITIALIZED_FILE}" ]]; then
+    POSTGRES_INITIALIZED=$(cat "${POSTGRES_INITIALIZED_FILE}")
+    if [[ ${POSTGRES_INITIALIZED} == ${VOLUME_ID} ]]; then
+        echo "The database in volume ${VOLUME_ID} was already initialized!"
+        exit 1
+    fi
 fi
 
-docker run --rm -v ${VOLUME_ID}:/mnt/database turing-dmf:latest /bin/bash -c '\
-    DATABASE=/mnt/database/db.sqlite3 && \
-    if [[ -f "$DATABASE" ]]; then \
-        echo "Database already exists. Not overwriting"; \
-    else \
-        touch $DATABASE && \
-        cd turing && \
-        python3 manage.py makemigrations && \
-        python3 manage.py makemigrations engine && \
-        python3 manage.py migrate && \
-        [email protected] DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_PASSWORD=admin python3 manage.py createsuperuser --no-input;
-    fi \
-'
+# Write out the postgresql data as of debian:12
+POSTGRES_DATA_DIRECTORY_FILE=".docker_postgres_data_directory"
+POSTGRES_DATA_DIRECTORY="/var/lib/postgresql/15/main"
+echo ${POSTGRES_DATA_DIRECTORY} > ${POSTGRES_DATA_DIRECTORY_FILE}
+
+# Use a clean container to create the default postgresql data, since turing-dmf:latest cannot be used
+# because it already expects the database to exist
+docker run --rm -v ${VOLUME_ID}:${POSTGRES_DATA_DIRECTORY} -e POSTGRES_PASSWORD=${POSTGRES_PASSWORD} debian:12 /bin/bash -c "\
+    apt update -y -q && \
+    apt install -y -qq postgresql sudo && \
+    service postgresql start && \
+    echo \$(sudo -u postgres psql -c \"SHOW data_directory;\") && \
+    sudo -u postgres psql -c \"ALTER USER postgres WITH PASSWORD '${POSTGRES_PASSWORD}';\" && \
+    sudo -u postgres createdb turing-dmf-db
+"
+
+# Once the database has been created, ask turing to initialize it
+docker run --rm -v ${VOLUME_ID}:${POSTGRES_DATA_DIRECTORY} turing-dmf:latest /bin/bash -c "\
+    service postgresql start && \
+    cd turing && \
+    python3 manage.py makemigrations && \
+    python3 manage.py makemigrations engine && \
+    python3 manage.py migrate && \
+    [email protected] DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_PASSWORD=admin python3 manage.py createsuperuser --no-input;
+"
+
+# Mark the database in this volume as initialized
+echo ${VOLUME_ID} > ${POSTGRES_INITIALIZED_FILE}

docker/docker_create_image.sh

@@ -13,4 +13,7 @@ wget -q --spider https://www.google.com
 SECRET_KEY_FILE=".docker_secret_key"
 SECRET_KEY=$(cat "${SECRET_KEY_FILE}")
 
-docker build --pull --build-arg SECRET_KEY=${SECRET_KEY} -t turing-dmf:latest -f Dockerfile ..
+POSTGRES_PASSWORD_FILE=".docker_postgres_password"
+POSTGRES_PASSWORD=$(cat "${POSTGRES_PASSWORD_FILE}")
+
+docker build --pull --build-arg SECRET_KEY=${SECRET_KEY} --build-arg POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -t turing-dmf:latest -f Dockerfile ..

docker/docker_create_secrets.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright (C) 2024 by the Turing @ DMF authors
+#
+# This file is part of Turing @ DMF.
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+set -e
+
+SECRET_KEY_FILE=".docker_secret_key"
+if [[ -f "${SECRET_KEY_FILE}" ]]; then
+    echo "A secret key already exists!"
+    echo "If you want to destroy it and create a new one, please remove the ${SECRET_KEY_FILE} file"
+    exit 1
+else
+    SECRET_KEY=$(cat /dev/urandom | tr -dc 'abcdefghijklmnopqrstuvwxyz0123456789!@#$^&*-_=+' | head -c 50; echo)
+    echo ${SECRET_KEY} > ${SECRET_KEY_FILE}
+fi
+
+POSTGRES_PASSWORD_FILE=".docker_postgres_password"
+if [[ -f "${POSTGRES_PASSWORD_FILE}" ]]; then
+    echo "A postgres password already exists!"
+    echo "If you want to destroy it and create a new one, please remove the ${POSTGRES_PASSWORD_FILE} file"
+    exit 1
+else
+    POSTGRES_PASSWORD=$(cat /dev/urandom | tr -dc 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' | head -c 50; echo)
+    echo ${POSTGRES_PASSWORD} > ${POSTGRES_PASSWORD_FILE}
+fi

docker/docker_explore_volume.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright (C) 2024 by the Turing @ DMF authors
+#
+# This file is part of Turing @ DMF.
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+set -e
+
+VOLUME_ID_FILE=".docker_volume_id"
+VOLUME_ID=$(cat "${VOLUME_ID_FILE}")
+
+docker run -it --rm -v ${VOLUME_ID}:/mnt/docker-volume --workdir=/mnt/docker-volume debian:12