From d5d1742b70580d78ff985ad17bde8df4a99d4270 Mon Sep 17 00:00:00 2001 From: Francesco Ballarin Date: Mon, 12 Feb 2024 06:22:41 +0100 Subject: [PATCH] Move from sqlite3 to postgres --- docker/.gitignore | 2 + docker/Dockerfile | 19 ++++++---- docker/README.md | 18 +++++---- docker/docker_create_container.sh | 13 +++---- docker/docker_create_database.sh | 61 ++++++++++++++++++++---------- docker/docker_create_image.sh | 5 ++- docker/docker_create_secret_key.sh | 18 --------- docker/docker_create_secrets.sh | 28 ++++++++++++++ docker/docker_explore_volume.sh | 13 +++++++ 9 files changed, 117 insertions(+), 60 deletions(-) delete mode 100644 docker/docker_create_secret_key.sh create mode 100644 docker/docker_create_secrets.sh create mode 100644 docker/docker_explore_volume.sh diff --git a/docker/.gitignore b/docker/.gitignore index 1693c69..10e6187 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1,3 +1,5 @@ .docker_container_id +.docker_postgres_data_directory +.docker_postgres_password .docker_secret_key .docker_volume_id diff --git a/docker/Dockerfile b/docker/Dockerfile index 802e96f..63bfa6a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,18 +5,24 @@ # SPDX-License-Identifier: AGPL-3.0-or-later FROM debian:12 +# If upgrading the version number from 12 to the next debian release, make sure to do the same +# also in the shell scripts stored in this directory. MAINTAINER Francesco Ballarin ARG SECRET_KEY +ARG POSTGRES_PASSWORD WORKDIR /root RUN apt update -y -q && \ - apt install -y -qq curl net-tools python3-pip unzip vim wget + apt install -y -qq curl net-tools python3-pip sudo unzip vim wget COPY . . # Part 1: turing and its runtime dependencies +RUN apt install -y -qq postgresql postgresql-client postgresql-contrib && \ + echo "$POSTGRES_PASSWORD\n$POSTGRES_PASSWORD" | passwd postgres + RUN bash patches/turing/apply_patches.sh && \ python3 -m pip install --break-system-packages -r turing/requirements.txt @@ -27,14 +33,13 @@ DEBUG=False DEV_MODE=False ALLOWED_HOSTS=* INTERNAL_IPS=127.0.0.1 -EMAIL_HOST= -EMAIL_HOST_USER= -EMAIL_HOST_PASSWORD= -REGISTRATION_OPEN= +RDS_DB_NAME=turing-dmf-db +RDS_USERNAME=postgres +RDS_PASSWORD=$POSTGRES_PASSWORD +RDS_HOSTNAME=localhost EOF RUN cd turing && \ - ln -s /mnt/database/db.sqlite3 . && \ python3 manage.py collectstatic # Part 2: test dependencies @@ -52,4 +57,4 @@ RUN LATEST_STABLE_RELEASE=$(curl -sS https://googlechromelabs.github.io/chrome-f RUN python3 -m pip install --break-system-packages -r turing/requirements-dev.txt # The end: run server on docker container start -CMD cd turing && python3 manage.py runserver 0.0.0.0:8080 +CMD service postgresql start && cd turing && python3 manage.py runserver 0.0.0.0:8080 diff --git a/docker/README.md b/docker/README.md index 60c3fe5..95525dd 100644 --- a/docker/README.md +++ b/docker/README.md @@ -20,21 +20,21 @@ cd turing-dmf/docker ``` bash docker_create_volume.sh ``` -3. Create a secret key for `django`: +3. Create a secret key for `django` and a password for `postgresql`: ``` -bash docker_create_secret_key.sh +bash docker_create_secrets.sh ``` 4. Create a `turing-dmf:latest` docker image based on the current **Turing @ DMF** repository: ``` bash docker_create_image.sh ``` -5. Create a docker container based on the current `turing-dmf:latest` docker image: +5. Create a database for **Turing**: ``` -bash docker_create_container.sh +bash docker_create_database.sh ``` -6. Create a database for **Turing**: +6. Create a docker container based on the current `turing-dmf:latest` docker image: ``` -bash docker_create_database.sh +bash docker_create_container.sh ``` ## 3. Run the docker container @@ -52,7 +52,11 @@ bash docker_start.sh ``` bash docker_terminal.sh ``` -4. Stop the running docker container +4. Explore the database volume with +``` +bash docker_explore_volume.sh +``` +5. Stop the running docker container ``` bash docker_stop.sh ``` diff --git a/docker/docker_create_container.sh b/docker/docker_create_container.sh index c7c1a2f..74c124d 100644 --- a/docker/docker_create_container.sh +++ b/docker/docker_create_container.sh @@ -8,13 +8,10 @@ set -e VOLUME_ID_FILE=".docker_volume_id" -if [[ ! -f "${VOLUME_ID_FILE}" ]]; then - echo "The database volume does not exist!" - echo "Please create it with docker_create_volume.sh" - exit 1 -else - VOLUME_ID=$(cat "${VOLUME_ID_FILE}") -fi +VOLUME_ID=$(cat "${VOLUME_ID_FILE}") + +POSTGRES_DATA_DIRECTORY_FILE=".docker_postgres_data_directory" +POSTGRES_DATA_DIRECTORY=$(cat "${POSTGRES_DATA_DIRECTORY_FILE}") CONTAINER_ID_FILE=".docker_container_id" if [[ -f "${CONTAINER_ID_FILE}" ]]; then @@ -28,6 +25,6 @@ else docker network create --subnet=10.200.1.0/24 turing-dmf-network # Start docker container on a fixed IP address. The corresponding mac address is generated following # https://maclookup.app/faq/how-do-i-identify-the-mac-address-of-a-docker-container-interface - CONTAINER_ID=$(docker create --net turing-dmf-network --ip 10.200.1.23 --mac-address 02:42:0a:c8:01:17 -p 8080:8080 -v /tmp/shared-turing-dmf:/tmp/shared-turing-dmf -v ${VOLUME_ID}:/mnt/database -e DOCKERHOSTNAME=$(cat /etc/hostname) turing-dmf:latest) + CONTAINER_ID=$(docker create --net turing-dmf-network --ip 10.200.1.23 --mac-address 02:42:0a:c8:01:17 -p 8080:8080 -v /tmp/shared-turing-dmf:/tmp/shared-turing-dmf -v ${VOLUME_ID}:${POSTGRES_DATA_DIRECTORY} -e DOCKERHOSTNAME=$(cat /etc/hostname) turing-dmf:latest) echo ${CONTAINER_ID} > ${CONTAINER_ID_FILE} fi diff --git a/docker/docker_create_database.sh b/docker/docker_create_database.sh index 577b00a..af8e0a9 100644 --- a/docker/docker_create_database.sh +++ b/docker/docker_create_database.sh @@ -7,25 +7,48 @@ set -e +# Outputs of previous steeps VOLUME_ID_FILE=".docker_volume_id" -if [[ ! -f "${VOLUME_ID_FILE}" ]]; then - echo "The database volume does not exist!" - echo "Please create it with docker_create_volume.sh" - exit 1 -else - VOLUME_ID=$(cat "${VOLUME_ID_FILE}") +VOLUME_ID=$(cat "${VOLUME_ID_FILE}") + +POSTGRES_PASSWORD_FILE=".docker_postgres_password" +POSTGRES_PASSWORD=$(cat "${POSTGRES_PASSWORD_FILE}") + +# Determine if the script was already run +POSTGRES_INITIALIZED_FILE=".docker_postgres_initialized" +if [[ -f "${POSTGRES_INITIALIZED_FILE}" ]]; then + POSTGRES_INITIALIZED=$(cat "${POSTGRES_INITIALIZED_FILE}") + if [[ ${POSTGRES_INITIALIZED} == ${VOLUME_ID} ]]; then + echo "The database in volume ${VOLUME_ID} was already initialized!" + exit 1 + fi fi -docker run --rm -v ${VOLUME_ID}:/mnt/database turing-dmf:latest /bin/bash -c '\ - DATABASE=/mnt/database/db.sqlite3 && \ - if [[ -f "$DATABASE" ]]; then \ - echo "Database already exists. Not overwriting"; \ - else \ - touch $DATABASE && \ - cd turing && \ - python3 manage.py makemigrations && \ - python3 manage.py makemigrations engine && \ - python3 manage.py migrate && \ - DJANGO_SUPERUSER_EMAIL=admin@admin.it DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_PASSWORD=admin python3 manage.py createsuperuser --no-input; - fi \ -' +# Write out the postgresql data as of debian:12 +POSTGRES_DATA_DIRECTORY_FILE=".docker_postgres_data_directory" +POSTGRES_DATA_DIRECTORY="/var/lib/postgresql/15/main" +echo ${POSTGRES_DATA_DIRECTORY} > ${POSTGRES_DATA_DIRECTORY_FILE} + +# Use a clean container to create the default postgresql data, since turing-dmf:latest cannot be used +# because it already expects the database to exist +docker run --rm -v ${VOLUME_ID}:${POSTGRES_DATA_DIRECTORY} -e POSTGRES_PASSWORD=${POSTGRES_PASSWORD} debian:12 /bin/bash -c "\ + apt update -y -q && \ + apt install -y -qq postgresql sudo && \ + service postgresql start && \ + echo \$(sudo -u postgres psql -c \"SHOW data_directory;\") && \ + sudo -u postgres psql -c \"ALTER USER postgres WITH PASSWORD '${POSTGRES_PASSWORD}';\" && \ + sudo -u postgres createdb turing-dmf-db +" + +# Once the database has been created, ask turing to initialize it +docker run --rm -v ${VOLUME_ID}:${POSTGRES_DATA_DIRECTORY} turing-dmf:latest /bin/bash -c "\ + service postgresql start && \ + cd turing && \ + python3 manage.py makemigrations && \ + python3 manage.py makemigrations engine && \ + python3 manage.py migrate && \ + DJANGO_SUPERUSER_EMAIL=admin@admin.it DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_PASSWORD=admin python3 manage.py createsuperuser --no-input; +" + +# Mark the database in this volume as initialized +echo ${VOLUME_ID} > ${POSTGRES_INITIALIZED_FILE} diff --git a/docker/docker_create_image.sh b/docker/docker_create_image.sh index fe064a2..347999a 100644 --- a/docker/docker_create_image.sh +++ b/docker/docker_create_image.sh @@ -13,4 +13,7 @@ wget -q --spider https://www.google.com SECRET_KEY_FILE=".docker_secret_key" SECRET_KEY=$(cat "${SECRET_KEY_FILE}") -docker build --pull --build-arg SECRET_KEY=${SECRET_KEY} -t turing-dmf:latest -f Dockerfile .. +POSTGRES_PASSWORD_FILE=".docker_postgres_password" +POSTGRES_PASSWORD=$(cat "${POSTGRES_PASSWORD_FILE}") + +docker build --pull --build-arg SECRET_KEY=${SECRET_KEY} --build-arg POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -t turing-dmf:latest -f Dockerfile .. diff --git a/docker/docker_create_secret_key.sh b/docker/docker_create_secret_key.sh deleted file mode 100644 index bdf4149..0000000 --- a/docker/docker_create_secret_key.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 by the Turing @ DMF authors -# -# This file is part of Turing @ DMF. -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -set -e - -SECRET_KEY_FILE=".docker_secret_key" -if [[ -f "${SECRET_KEY_FILE}" ]]; then - echo "A secret key already exists!" - echo "If you want to destroy it and create a new one, please remove the ${SECRET_KEY_FILE} file" - exit 1 -else - SECRET_KEY=$(cat /dev/urandom | tr -dc 'abcdefghijklmnopqrstuvwxyz0123456789!@#$^&*(-_=+)' | head -c 50; echo) - echo ${SECRET_KEY} > ${SECRET_KEY_FILE} -fi diff --git a/docker/docker_create_secrets.sh b/docker/docker_create_secrets.sh new file mode 100644 index 0000000..4cd0d8b --- /dev/null +++ b/docker/docker_create_secrets.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2024 by the Turing @ DMF authors +# +# This file is part of Turing @ DMF. +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -e + +SECRET_KEY_FILE=".docker_secret_key" +if [[ -f "${SECRET_KEY_FILE}" ]]; then + echo "A secret key already exists!" + echo "If you want to destroy it and create a new one, please remove the ${SECRET_KEY_FILE} file" + exit 1 +else + SECRET_KEY=$(cat /dev/urandom | tr -dc 'abcdefghijklmnopqrstuvwxyz0123456789!@#$^&*-_=+' | head -c 50; echo) + echo ${SECRET_KEY} > ${SECRET_KEY_FILE} +fi + +POSTGRES_PASSWORD_FILE=".docker_postgres_password" +if [[ -f "${POSTGRES_PASSWORD_FILE}" ]]; then + echo "A postgres password already exists!" + echo "If you want to destroy it and create a new one, please remove the ${POSTGRES_PASSWORD_FILE} file" + exit 1 +else + POSTGRES_PASSWORD=$(cat /dev/urandom | tr -dc 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' | head -c 50; echo) + echo ${POSTGRES_PASSWORD} > ${POSTGRES_PASSWORD_FILE} +fi diff --git a/docker/docker_explore_volume.sh b/docker/docker_explore_volume.sh new file mode 100644 index 0000000..75679bb --- /dev/null +++ b/docker/docker_explore_volume.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright (C) 2024 by the Turing @ DMF authors +# +# This file is part of Turing @ DMF. +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -e + +VOLUME_ID_FILE=".docker_volume_id" +VOLUME_ID=$(cat "${VOLUME_ID_FILE}") + +docker run -it --rm -v ${VOLUME_ID}:/mnt/docker-volume --workdir=/mnt/docker-volume debian:12