From 099a4932b8f565faf4cc44b279665a8297dd58c8 Mon Sep 17 00:00:00 2001 From: Simon Guilbault Date: Thu, 17 Oct 2024 15:54:37 -0400 Subject: [PATCH] Updating to Django 5 and Ubuntu 24.04 (#65) * Updating to Django 5 and Ubuntu 24.04 * Temporarily remove db version check to support mariadb server on EL8 without appstream enabled * Configurable workers and threads * Configuring Content Security Policy * Fix IDP-less debugging on django5 * Switching charset to utf8mb4 --- Containerfile | 17 ++++-- dbcheck.patch | 11 ++++ docs/install.md | 13 +++- jobstats/migrations/0002_utf8.py | 15 +++++ notes/migrations/0002_utf8.py | 16 +++++ podman/deploy.sh | 12 ++++ requirements.txt | 95 ++++++++++++++--------------- run-django-production.sh | 2 +- userportal/authentication.py | 2 +- userportal/settings/10-base.py | 16 ++--- userportal/settings/20-databases.py | 4 +- 11 files changed, 137 insertions(+), 66 deletions(-) create mode 100644 dbcheck.patch create mode 100644 jobstats/migrations/0002_utf8.py create mode 100644 notes/migrations/0002_utf8.py diff --git a/Containerfile b/Containerfile index 906b163..d6c93b1 100644 --- a/Containerfile +++ b/Containerfile @@ -1,21 +1,26 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 -RUN apt-get update && apt-get install -y tzdata && apt install -y python3.10 python3-pip python3-dev python3-numpy libpq-dev nginx libmysqlclient-dev build-essential libsasl2-dev libldap2-dev libssl-dev xmlsec1 gettext +RUN apt-get update && apt-get install -y tzdata && apt install -y python3.12 python3-pip python3-dev python3.12-venv libpq-dev nginx libmysqlclient-dev pkg-config build-essential libsasl2-dev libldap2-dev libssl-dev xmlsec1 gettext git WORKDIR /opt/userportal COPY requirements.txt ./ -RUN pip3 install --upgrade pip && \ - pip3 install --no-cache-dir -r requirements.txt +RUN python3 -m venv /opt/userportal-env && \ + /opt/userportal-env/bin/pip install --upgrade pip && \ + /opt/userportal-env/bin/pip install --no-cache-dir -r requirements.txt COPY . . -RUN patch /usr/local/lib/python3.10/dist-packages/ldapdb/backends/ldap/base.py < /opt/userportal/ldapdb.patch -RUN python3 manage.py collectstatic --noinput && python3 manage.py compilemessages +RUN patch /opt/userportal-env/lib/python3.12/site-packages/ldapdb/backends/ldap/base.py < /opt/userportal/ldapdb.patch + +# Temporarily remove db version check to support mariadb server on EL8 without appstream enabled +RUN patch /opt/userportal-env/lib/python3.12/site-packages/django/db/backends/base/base.py < /opt/userportal/dbcheck.patch + +RUN /opt/userportal-env/bin/python manage.py collectstatic --noinput && /opt/userportal-env/bin/python manage.py compilemessages EXPOSE 8000 CMD ["./run-django-production.sh"] diff --git a/dbcheck.patch b/dbcheck.patch new file mode 100644 index 0000000..b61e43a --- /dev/null +++ b/dbcheck.patch @@ -0,0 +1,11 @@ +--- /opt/userportal-env/lib/python3.12/site-packages/django/db/backends/base/base.py 2024-07-17 17:45:44.000000000 +0000 ++++ /opt/userportal-env/lib/python3.12/site-packages/django/db/backends/base/base.py.modified 2024-07-17 19:21:39.250937980 +0000 +@@ -222,7 +222,7 @@ + """Initialize the database connection settings.""" + global RAN_DB_VERSION_CHECK + if self.alias not in RAN_DB_VERSION_CHECK: +- self.check_database_version_supported() ++ #self.check_database_version_supported() + RAN_DB_VERSION_CHECK.add(self.alias) + + def create_cursor(self, name=None): diff --git a/docs/install.md b/docs/install.md index 4363a9c..999cf16 100644 --- a/docs/install.md +++ b/docs/install.md @@ -5,7 +5,18 @@ Before installing in production, [a test environment should be set up to test th The portal can be installed directly on a Rocky8 Apache web server or with Nginx and Gunicorn. The portal can also be deployed as a container with Podman or Kubernetes. Some scripts used to deploy both Nginx and Django containers inside the same pod are provided in the `podman` directory. The various recommendations for any normal Django production deployment can be followed. -[Deploying Django](https://docs.djangoproject.com/en/3.2/howto/deployment/) +[Deploying Django](https://docs.djangoproject.com/en/5.0/howto/deployment/) + +The database should support UTF8. With MariaDB, the default collation can be changed with the following command: + +``` +ALTER DATABASE userportal CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +``` + +Migration scripts will also ensure that the tables and columns are converted to the correct collation. + +# Production with containers +Using containers is the recommended way to deploy the portal. The container is automatically built in the CI pipeline and pushed to the Github registry. This container is handling the django application. The static files are served by a standard Nginx container. Both containers are deployed in the same pod with a shared volume containing the static files. # Production without containers on Rocky Linux 8 diff --git a/jobstats/migrations/0002_utf8.py b/jobstats/migrations/0002_utf8.py new file mode 100644 index 0000000..fcbf62b --- /dev/null +++ b/jobstats/migrations/0002_utf8.py @@ -0,0 +1,15 @@ +# Generated by Django 5.0.7 on 2024-10-16 18:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('jobstats', '0001_initial'), + ] + + operations = [ + migrations.RunSQL('ALTER TABLE jobstats_jobscript CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'), + migrations.RunSQL('ALTER TABLE jobstats_jobscript MODIFY submit_script LONGTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'), + ] diff --git a/notes/migrations/0002_utf8.py b/notes/migrations/0002_utf8.py new file mode 100644 index 0000000..0382500 --- /dev/null +++ b/notes/migrations/0002_utf8.py @@ -0,0 +1,16 @@ +# Generated by Django 5.0.7 on 2024-10-16 18:03 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('notes', '0001_initial'), + ] + + operations = [ + migrations.RunSQL('ALTER TABLE notes_note CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'), + migrations.RunSQL('ALTER TABLE notes_note MODIFY title VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'), + migrations.RunSQL('ALTER TABLE notes_note MODIFY notes LONGTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'), + ] diff --git a/podman/deploy.sh b/podman/deploy.sh index 8cfa8c6..74bcca1 100755 --- a/podman/deploy.sh +++ b/podman/deploy.sh @@ -20,6 +20,16 @@ if [ -z "$VERSION" ]; then VERSION=latest fi +if [ -z "$WORKERS" ]; then + echo "WORKERS is not set in the config file, using 4" + WORKERS=4 +fi + +if [ -z "$THREADS" ]; then + echo "THREADS is not set in the config file, using 4" + THREADS=4 +fi + podman pod stop TT-$TT_ENV podman pod rm -f TT-$TT_ENV podman volume rm -f TT-$TT_ENV-static @@ -28,6 +38,8 @@ podman pod create -p $PORT:80 --name TT-$TT_ENV podman volume create TT-$TT_ENV-static podman run --detach --pod TT-$TT_ENV --name=TT-$TT_ENV-django \ + --env WORKERS=${WORKERS} \ + --env THREADS=${THREADS} \ -v $TT_ENV_PATH/99-local.py:/secrets/settings/99-local.py:Z \ -v $TT_ENV_PATH/private.key:/opt/userportal/private.key:Z \ -v $TT_ENV_PATH/public.cert:/opt/userportal/public.cert:Z \ diff --git a/requirements.txt b/requirements.txt index d880486..6b3d021 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,56 +1,55 @@ -asgiref==3.4.1 -certifi==2024.07.04 -cffi==1.15.1 -chardet==4.0.0 -charset-normalizer==3.3.1 -cryptography==42.0.4 -dateparser==1.0.0 +asgiref==3.8.1 +certifi==2024.7.4 +cffi==1.16.0 +charset-normalizer==3.3.2 +contourpy==1.2.1 +cryptography==42.0.8 +cycler==0.12.1 +dateparser==1.2.0 defusedxml==0.7.1 -Django==3.2.25 -django-auth-ldap==2.3.0 -django-bootstrap-pagination==1.7.1 -django-debug-toolbar==3.2.4 -django-ldapdb==1.5.1 +Django==5.0.9 +django-bootstrap-pagination @ git+https://github.com/teejaydub/django-bootstrap-pagination@82ea7a213860a741890f324e025f6ec215f24a32 +django-crispy-forms==2.2 +django-debug-toolbar==4.4.6 +django-ldapdb @ git+https://github.com/nikolaik/django-ldapdb@381e4f40032c96ea6ae6623a5a0339b3ea526ae5 django-settings-export==1.2.1 django-watchman==1.3.0 -djangorestframework==3.13.1 -djangorestframework-api-key==2.0.0 -djangorestframework-datatables==0.7.0 -djangosaml2==1.5.1 -elementpath==2.4.0 -flake8==3.9.0 +django_csp==3.8 +djangorestframework==3.15.2 +djangorestframework-datatables==0.7.2 +djangosaml2==1.9.3 +elementpath==4.4.0 +flake8==7.1.0 +fonttools==4.53.1 gunicorn==22.0.0 +httmock==1.4.0 idna==3.7 -importlib-metadata==3.7.3 -importlib-resources==5.4.0 -install==1.3.5 -IPy==1.1 -mccabe==0.6.1 -mysqlclient==2.0.3 -numpy==1.23.4 -pandas==1.1.5 -plotly==4.14.3 -prometheus-api-client==0.4.2 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pycodestyle==2.7.0 -pycparser==2.21 -pyflakes==2.3.0 -PyJWT==2.7.0 -Pyment==0.3.3 +kiwisolver==1.4.5 +matplotlib==3.9.2 +mccabe==0.7.0 +mysqlclient==2.2.4 +numpy==2.0.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +prometheus-api-client==0.5.5 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pycodestyle==2.12.0 +pycparser==2.22 +pyflakes==3.2.0 pyOpenSSL==24.1.0 -pysaml2==7.1.2 -python-dateutil==2.8.1 -python-ldap==3.4.3 -pytz==2021.1 -PyYAML==6.0 -regex==2020.11.13 +pyparsing==3.1.2 +pysaml2==7.5.0 +python-dateutil==2.9.0.post0 +python-ldap==3.4.4 +pytz==2024.1 +PyYAML==6.0.1 +regex==2024.5.15 requests==2.32.3 -retrying==1.3.3 -six==1.15.0 +six==1.16.0 sqlparse==0.5.0 -typing-extensions==3.7.4.3 -tzlocal==2.1 -urllib3==1.26.19 -xmlschema==1.9.2 -zipp==3.19.1 +tzdata==2024.1 +tzlocal==5.2 +urllib3==2.2.2 +xmlschema==2.5.1 diff --git a/run-django-production.sh b/run-django-production.sh index 4e89584..5ad380a 100755 --- a/run-django-production.sh +++ b/run-django-production.sh @@ -7,4 +7,4 @@ set -o nounset cp /secrets/settings/99-local.py /opt/userportal/userportal/settings/99-local.py mkdir -p /var/www/api/static cp -r /opt/userportal/collected-static/* /var/www/api/static/ -gunicorn --bind :8000 --workers 1 --timeout 90 userportal.wsgi \ No newline at end of file +/opt/userportal-env/bin/gunicorn --bind :8000 --workers $WORKERS --threads $THREADS --timeout 90 userportal.wsgi diff --git a/userportal/authentication.py b/userportal/authentication.py index 385dc00..9bc37cb 100644 --- a/userportal/authentication.py +++ b/userportal/authentication.py @@ -23,7 +23,7 @@ def clean_username(self, username): username = username.split('@')[0] return username - def configure_user(self, request, user): + def configure_user(self, request, user, created=True): if 'staff@computecanada.ca' in request.META['affiliation'] \ or 'staff@alliancecan.ca' in request.META['affiliation']: user.is_staff = True diff --git a/userportal/settings/10-base.py b/userportal/settings/10-base.py index 2d66303..c469dce 100644 --- a/userportal/settings/10-base.py +++ b/userportal/settings/10-base.py @@ -1,13 +1,5 @@ """ Django settings for userportal project. - -Generated by 'django-admin startproject' using Django 3.1.7. - -For more information on this file, see -https://docs.djangoproject.com/en/3.1/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/3.1/ref/settings/ """ from pathlib import Path @@ -43,6 +35,7 @@ 'django.contrib.staticfiles', 'django.contrib.humanize', 'djangosaml2', + 'csp', 'watchman', 'pages', @@ -78,6 +71,7 @@ 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'djangosaml2.middleware.SamlSessionMiddleware', + 'csp.middleware.CSPMiddleware', ] ROOT_URLCONF = 'userportal.urls' @@ -204,3 +198,9 @@ 'DEFAULT_PAGINATION_CLASS': 'rest_framework_datatables.pagination.DatatablesPageNumberPagination', 'PAGE_SIZE': 100, } + +# Content Security Policy +CSP_DEFAULT_SRC = ("'self'") +CSP_IMG_SRC = ("'self'", "data:", 'object-arbutus.cloud.computecanada.ca') +CSP_STYLE_SRC = ("'self'", "'unsafe-inline'", 'cdn.jsdelivr.net', 'cdnjs.cloudflare.com', 'cdn.datatables.net') +CSP_SCRIPT_SRC = ("'self'", "'unsafe-inline'", 'cdn.jsdelivr.net', 'cdnjs.cloudflare.com', 'cdn.datatables.net', 'code.jquery.com', 'cdn.plot.ly') diff --git a/userportal/settings/20-databases.py b/userportal/settings/20-databases.py index bd25836..8bcac64 100644 --- a/userportal/settings/20-databases.py +++ b/userportal/settings/20-databases.py @@ -10,7 +10,9 @@ 'HOST': 'dbserver', 'PORT': '3128', 'OPTIONS': { - } + 'charset': 'utf8mb4', + 'use_unicode': True, + }, }, 'slurm': { 'ENGINE': 'django.db.backends.mysql',