Merge pull request #99 from maykinmedia/feature/update-libraries

⬆️ upgraded all libraries to latest version

.github/workflows/ci.yml

@@ -61,7 +61,7 @@ jobs:
 
     strategy:
       matrix:
-        postgres: ['10', '11', '12']
+        postgres: ['11', '12', '13']
 
     name: Tests (PG ${{ matrix.postgres }})
 
@@ -79,17 +79,15 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.11'
+          python-version: '3.10'
       - uses: actions/setup-node@v2-beta
         with:
           node-version: '17'
 
       - name: Install system packages
         run: |
           sudo apt-get update \
-          && sudo apt-get install -y --no-install-recommends \
-            libgdal-dev \
-            gdal-bin
+          && sudo apt-get install -y --no-install-recommends
       - name: Install dependencies
         run: pip install -r requirements/dev.txt codecov
       - name: Build frontend

Dockerfile

@@ -1,31 +1,21 @@
-# This is a multi-stage build file, which means a stage is used to build
-# the backend (dependencies), the frontend stack and a final production
-# stage re-using assets from the build stages. This keeps the final production
-# image minimal in size.
+# Stage 1 - Compile needed python dependencies
+FROM python:3.10-slim-bookworm AS build
 
-# Stage 1 - Backend build environment
-# includes compilers and build tooling to create the environment
-FROM python:3.11-buster AS backend-build
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
+        pkg-config \
+        build-essential \
         libpq-dev \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
-RUN mkdir /app/src
 
-# Ensure we use the latest version of pip
-RUN pip install pip setuptools -U
 COPY ./requirements /app/requirements
+RUN pip install pip -U
 RUN pip install -r requirements/production.txt
 
 
-# Stage 2 - Install frontend deps and build assets
-FROM node:17-buster AS frontend-build
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        git \
-    && rm -rf /var/lib/apt/lists/*
+# Stage 2 - build frontend
+FROM node:16-bookworm-slim AS frontend-build
 
 WORKDIR /app
 
@@ -43,54 +33,76 @@ COPY ./src /app/src
 RUN npm run build
 
 
-# Stage 3 - Build docker image suitable for production
-FROM python:3.11-buster
+# Stage 3 - Build docker image suitable for execution and deployment
+FROM python:3.10-slim-bookworm AS production
 
 # Stage 3.1 - Set up the needed production dependencies
 # install all the dependencies for GeoDjango
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
+        # bare minimum to debug live containers
         procps \
-        vim \
+        nano \
+        # serve correct Content-Type headers
+        mime-support \
+        # (geo) django dependencies
         postgresql-client \
-        # lxml deps
-        # libxslt \
+        gettext \
+        binutils \
     && rm -rf /var/lib/apt/lists/*
 
+RUN pip install pip -U
+
 WORKDIR /app
+# COPY ./cache /app/cache
 COPY ./bin/docker_start.sh /start.sh
+COPY ./bin/wait_for_db.sh /wait_for_db.sh
 COPY ./bin/celery_worker.sh /celery_worker.sh
-COPY ./bin/celery_beat.sh /celery_beat.sh
 COPY ./bin/celery_flower.sh /celery_flower.sh
-RUN mkdir /app/log
-RUN mkdir /app/media
 
-# copy backend build deps
-COPY --from=backend-build /usr/local/lib/python3.11 /usr/local/lib/python3.11
-COPY --from=backend-build /usr/local/bin/uwsgi /usr/local/bin/uwsgi
-COPY --from=backend-build /app/src/ /app/src/
-COPY --from=backend-build /usr/local/bin/celery /usr/local/bin/celery
+RUN mkdir /app/log /app/config /app/media /app/private-media
+# prevent writing to the container layer, which would degrade performance.
+# This also serves as a hint for the intended volumes.
+VOLUME ["/app/log", "/app/media", "/app/private-media"]
 
+# copy backend build deps
+COPY --from=build /usr/local/lib/python3.10 /usr/local/lib/python3.10
+COPY --from=build /usr/local/bin/uwsgi /usr/local/bin/uwsgi
+COPY --from=build /usr/local/bin/celery /usr/local/bin/celery
 
-# copy frontend build statics
 COPY --from=frontend-build /app/src/openklant/static /app/src/openklant/static
 
-# copy source code
+# Stage 3.2 - Copy source code
 COPY ./src /app/src
 
-RUN useradd -M -u 1000 maykin
-RUN chown -R maykin /app
+RUN groupadd -g 1000 openklant \
+    && useradd -M -u 1000 -g 1000 openklant \
+    && chown -R openklant:openklant /app
 
 # drop privileges
-USER maykin
+USER openklant
 
 ARG COMMIT_HASH
+ARG RELEASE
 ENV GIT_SHA=${COMMIT_HASH}
+ENV RELEASE=${RELEASE}
+
 ENV DJANGO_SETTINGS_MODULE=openklant.conf.docker
 
 ARG SECRET_KEY=dummy
 
-# Run collectstatic, so the result is already included in the image
-RUN python src/manage.py collectstatic --noinput
+LABEL org.label-schema.vcs-ref=$COMMIT_HASH \
+      org.label-schema.vcs-url="https://github.com/maykinmedia/open-klant" \
+      org.label-schema.version=$RELEASE \
+      org.label-schema.name="Open Klant"
+
+# Run management commands:
+# * collectstatic -> bake the static assets into the image
+# * compilemessages -> ensure the translation catalog binaries are present
+# * warm_cache -> writes to the filesystem cache so that orgs don't need to open the
+#   firewall to github
+RUN python src/manage.py collectstatic --noinput \
+    && python src/manage.py compilemessages
+    # && python src/manage.py warm_cache
 
 EXPOSE 8000
-CMD ["/start.sh"]
+CMD ["/start.sh"]

INSTALL.rst

@@ -18,7 +18,7 @@ Prerequisites
 
 You need the following libraries and/or programs:
 
-* `Python`_ 3.11 or above
+* `Python`_ 3.10
 * Python `Virtualenv`_ and `Pip`_
 * `PostgreSQL`_ 11 or above
 * `Node.js`_

bin/compile_dependencies.sh

@@ -23,25 +23,22 @@ export CUSTOM_COMPILE_COMMAND="./bin/compile_dependencies.sh"
 # Base (& prod) deps
 pip-compile \
     --no-emit-index-url \
-    --allow-unsafe \
     "$@" \
     requirements/base.in
 
 # Dependencies for testing
 pip-compile \
     --no-emit-index-url \
     --output-file requirements/ci.txt \
-    --allow-unsafe \
     "$@" \
     requirements/base.txt \
     requirements/test-tools.in \
     requirements/docs.in
 
-# Dev dependencies - exact same set as CI + some extra tooling
+# Dev depedencies - exact same set as CI + some extra tooling
 pip-compile \
     --no-emit-index-url \
     --output-file requirements/dev.txt \
-    --allow-unsafe \
     "$@" \
     requirements/ci.txt \
-    requirements/dev.in
+    requirements/dev.in

bin/wait_for_db.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+
+# Wait for the database container
+# See: https://docs.docker.com/compose/startup-order/
+export PGHOST=${DB_HOST:-db}
+export PGPORT=${DB_PORT:-5432}
+
+until pg_isready; do
+  >&2 echo "Waiting for database connection..."
+  sleep 1
+done
+
+>&2 echo "Database is up."

requirements/base.in

@@ -1,38 +1,53 @@
 # Core python libraries
-psycopg2  # database driver
-pytz  # handle timezones
+celery
+click<8.1.0  # click>=8.1.0 causes issues with black https://github.com/psf/black/issues/2964
+jq
+jsonschema
+dictdiffer # Used to show diffs for audittrails in admin
+markdown  # used to render some markdown in code to html
+psycopg2
+python-dateutil
 python-dotenv  # environment variables for secrets
 python-decouple  # processing of envvar configs
+requests
+requests-cache
+zgw-consumers
+self-certifi
+bleach
 
 # Framework libraries
-django~=3.2.0
-django-admin-index
+django~=3.2.23
 django-axes
-django-better-admin-arrayfield
-django-choices
-django-cors-middleware
-django-filter==2.4.0
+django-cors-headers
+django-db-logger
+django-extra-views
+django-log-outgoing-requests
+django-loose-fk
+django-markup
 django-redis
-django-rosetta
-django-sniplates
+django-privates
+django-relativedelta
 mozilla-django-oidc-db
-# maykin-django-two-factor-auth
-# phonenumbers
-
-# API libraries
-djangorestframework
-# django-extra-fields
-# django-filter
-# drf-yasg  # api documentation
 
-vng-api-common[markdown_docs]==1.9.0  # pinned here before the move from drf-yasg to drf-spectacular
-gemma-zds-client<2.0.0  # lots of breaking changes in 2.0.0
-zgw-consumers>=0.25.0  # newest with simple cert manager
+# Admin and UI libraries
+django-admin-index
+django-sniplates
+django-better-admin-arrayfield
 
-# task queue
-celery
+# API libraries
+djangorestframework~=3.12.0
+drf-extra-fields
+django-filter
+djangorestframework-camel-case~=1.2.0
+drf-yasg
+drf-writable-nested
+commonground-api-common
+notifications-api-common
+humanize
+djangorestframework-inclusions
 
 # WSGI servers & monitoring - production oriented
 uwsgi
-sentry-sdk  # error monitoring
-elastic-apm  # Elastic APM integration
+sentry_sdk # error monitoring sentry
+flower # task monitoring
+elastic-apm  # Elastic APM integration