New docker image based on pure ubuntu and gracefully handle services

.docker/README.md

@@ -0,0 +1,11 @@
+# AiiDA docker stacks
+
+### Build images locally
+
+To build the images, run `doit build` (tested with *docker buildx* version v0.8.2).
+
+The build system will attempt to detect the local architecture and automatically build images for it (tested with amd64 and arm64).
+All commands `build`, `tests`, and `up` will use the locally detected platform and use a version tag based on the state of the local git repository.
+However, you can also specify a custom platform or version with the `--platform` and `--version` parameters, example: `doit build --arch=amd64 --version=my-version`.
+
+You can specify target stacks to build with `--target`, example: `doit build --target base --target base`.

.docker/base-with-services/Dockerfile

@@ -0,0 +1,45 @@
+# syntax=docker/dockerfile:1
+FROM base
+
+LABEL maintainer="AiiDAlab Team <[email protected]>"
+
+USER root
+WORKDIR /opt/
+
+ARG PGSQL_VERSION
+ARG RMQ_VERSION
+
+ENV PGSQL_VERSION=${PGSQL_VERSION}
+ENV RMQ_VERSION=${RMQ_VERSION}
+
+RUN mamba create -p /opt/conda/envs/aiida-core-services --yes \
+     --channel conda-forge \
+     postgresql=${PGSQL_VERSION} && \
+     mamba clean --all -f -y && \
+     fix-permissions "${CONDA_DIR}" && \
+     fix-permissions "/home/${SYSTEM_USER}"
+
+ENV MAMBA_RUN="mamba run -n aiida-core-services"
+
+# Install erlang.
+RUN apt-get update --yes && \
+     apt-get install --yes --no-install-recommends \
+     erlang \
+     xz-utils && \
+     apt-get clean && rm -rf /var/lib/apt/lists/* && \
+     # Install rabbitmq.
+     wget -c --no-check-certificate https://github.com/rabbitmq/rabbitmq-server/releases/download/v${RMQ_VERSION}/rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \
+     tar -xf rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \
+     rm rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \
+     mv rabbitmq_server-${RMQ_VERSION} /opt/conda/envs/aiida-core-services/ && \
+     ln -sf /opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/sbin/* /opt/conda/envs/aiida-core-services/bin/ && \
+     fix-permissions "${CONDA_DIR}"
+
+# s6-overlay to start services
+COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" s6-assets/config-quick-setup.yaml "/aiida/assets/config-quick-setup.yaml"
+COPY s6-assets/s6-rc.d /etc/s6-overlay/s6-rc.d
+COPY s6-assets/init /etc/init
+
+USER ${SYSTEM_UID}
+
+WORKDIR "/home/${SYSTEM_USER}"

.docker/base-with-services/s6-assets/config-quick-setup.yaml

@@ -0,0 +1,3 @@
+---
+db_name: aiida_db
+db_username: aiida

.docker/base-with-services/s6-assets/init/postgresql-init.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# make DB directory, if not existent
+if [ ! -d /home/${SYSTEM_USER}/.postgresql ]; then
+   mkdir /home/${SYSTEM_USER}/.postgresql
+   ${MAMBA_RUN} initdb -D /home/${SYSTEM_USER}/.postgresql
+   echo "unix_socket_directories = '/tmp'" >> /home/${SYSTEM_USER}/.postgresql/postgresql.conf
+fi
+
+PSQL_STATUS_CMD="pg_ctl -D /home/${SYSTEM_USER}/.postgresql status"
+
+# Fix problem with kubernetes cluster that adds rws permissions to the group
+# for more details see: https://github.com/materialscloud-org/aiidalab-z2jh-eosc/issues/5
+chmod g-rwxs /home/${SYSTEM_USER}/.postgresql -R
+
+# stores return value in $?
+running=true
+${MAMBA_RUN} ${PSQL_STATUS_CMD} || running=false
+
+# Postgresql was probably not shutdown properly. Cleaning up the mess...
+if ! $running ; then
+   echo "" > /home/${SYSTEM_USER}/.postgresql/logfile # empty log files
+   rm -vf /home/${SYSTEM_USER}/.postgresql/postmaster.pid
+fi

.docker/base-with-services/s6-assets/init/postgresql-prepare.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+PG_ISREADY=1
+while [ "$PG_ISREADY" != "0"  ]; do
+  sleep 1
+  ${MAMBA_RUN} pg_isready --quiet
+  PG_ISREADY=$?
+done

.docker/base-with-services/s6-assets/init/rabbitmq-init.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+RABBITMQ_DATA_DIR="/home/${SYSTEM_USER}/.rabbitmq"
+
+mkdir -p "${RABBITMQ_DATA_DIR}"
+fix-permissions "${RABBITMQ_DATA_DIR}"
+
+# Fix issue where the erlang cookie permissions are corrupted.
+chmod 400 "/home/${SYSTEM_USER}/.erlang.cookie" || echo "erlang cookie not created yet."
+
+# Set base directory for RabbitMQ to persist its data. This needs to be set to a folder in the system user's home
+# directory as that is the only folder that is persisted outside of the container.
+RMQ_ETC_DIR="/opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/etc/rabbitmq"
+echo MNESIA_BASE="${RABBITMQ_DATA_DIR}" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf"
+echo LOG_BASE="${RABBITMQ_DATA_DIR}/log" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf"
+
+# using workaround from https://github.com/aiidateam/aiida-core/wiki/RabbitMQ-version-to-use
+# set timeout to 100 hours
+echo "consumer_timeout=3600000" >> "${RMQ_ETC_DIR}/rabbitmq.conf"
+
+# Explicitly define the node name. This is necessary because the mnesia subdirectory contains the hostname, which by
+# default is set to the value of $(hostname -s), which for docker containers, will be a random hexadecimal string. Upon
+# restart, this will be different and so the original mnesia folder with the persisted data will not be found. The
+# reason RabbitMQ is built this way is through this way it allows to run multiple nodes on a single machine each with
+# isolated mnesia directories. Since in the AiiDA setup we only need and run a single node, we can simply use localhost.
+echo NODENAME=rabbit@localhost >> "${RMQ_ETC_DIR}/rabbitmq-env.conf"

.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/timeout-up

@@ -0,0 +1 @@
+0

.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/type

@@ -0,0 +1 @@
+oneshot

.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/up

@@ -0,0 +1,6 @@
+#!/command/execlineb -S0
+
+with-contenv
+
+foreground { s6-echo "Calling /etc/init/postgresql-init" }
+/etc/init/postgresql-init.sh

.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/timeout-up

@@ -0,0 +1 @@
+0

.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/type

@@ -0,0 +1 @@
+oneshot

.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/up

@@ -0,0 +1,6 @@
+#!/command/execlineb -S0
+
+with-contenv
+
+foreground { s6-echo "Calling /etc/init/postgresql-prepare" }
+/etc/init/postgresql-prepare.sh

.docker/base-with-services/s6-assets/s6-rc.d/postgresql/down-signal

@@ -0,0 +1 @@
+SIGINT

.docker/base-with-services/s6-assets/s6-rc.d/postgresql/run

@@ -0,0 +1,29 @@
+#!/command/execlineb -P
+
+with-contenv
+
+# # -w waits until server is up
+# echo "DEBUG"
+# echo "${SYSTEM_USER}"
+#
+# PSQL_START_CMD="pg_ctl --timeout=180 -w -D /home/${SYSTEM_USER}/.postgresql -l /home/${SYSTEM_USER}/.postgresql/logfile start"
+# PSQL_STATUS_CMD="pg_ctl -D /home/${SYSTEM_USER}/.postgresql status"
+#
+# MAMBA_RUN="mamba run -n aiida-core-services"
+#
+# # Fix problem with kubernetes cluster that adds rws permissions to the group
+# # for more details see: https://github.com/materialscloud-org/aiidalab-z2jh-eosc/issues/5
+# chmod g-rwxs /home/${SYSTEM_USER}/.postgresql -R
+#
+# # stores return value in $?
+# running=true
+# ${MAMBA_RUN} ${PSQL_STATUS_CMD} || running=false
+#
+# # Postgresql was probably not shutdown properly. Cleaning up the mess...
+# if ! $running ; then
+#    echo "" > /home/${SYSTEM_USER}/.postgresql/logfile # empty log files
+#    rm -vf /home/${SYSTEM_USER}/.postgresql/postmaster.pid
+#    ${MAMBA_RUN} ${PSQL_START_CMD}
+# fi
+
+mamba run -n aiida-core-services postgres -D /home/aiida/.postgresql

.docker/base-with-services/s6-assets/s6-rc.d/postgresql/type

@@ -0,0 +1 @@
+longrun

.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/timeout-up

@@ -0,0 +1 @@
+0

.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/type

@@ -0,0 +1 @@
+oneshot

.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/up

@@ -0,0 +1,5 @@
+#!/command/execlineb -S0
+with-contenv
+
+foreground { s6-echo "Calling /etc/init/rabbitmq-init.sh" }
+/etc/init/rabbitmq-init.sh

.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/down-signal

@@ -0,0 +1 @@
+SIGINT

.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/run

@@ -0,0 +1,6 @@
+#!/command/execlineb -P
+
+with-contenv
+
+foreground { s6-echo "Calling /etc/init/rabbitmq.sh" }
+mamba run -n aiida-core-services rabbitmq-server

.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/type

@@ -0,0 +1 @@
+longrun

.docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/timeout-up

@@ -0,0 +1 @@
+0

.docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/type

@@ -0,0 +1 @@
+oneshot

.docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/up

@@ -0,0 +1,10 @@
+#!/command/execlineb -P
+
+# Supress rabbitmq version warning
+# it is built using RMQ version > 3.8.15 which has the issue as described in
+# https://github.com/aiidateam/aiida-core/wiki/RabbitMQ-version-to-use
+# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf
+verdi config set warnings.rabbitmq_version False
+
+# Supress verdi version warning
+verdi config set warnings.development_version False

.docker/base/Dockerfile

@@ -0,0 +1,166 @@
+# syntax=docker/dockerfile:1
+
+# Inspired by jupyter's docker-stacks-fundation image:
+# https://github.com/jupyter/docker-stacks/blob/main/docker-stacks-foundation/Dockerfile
+
+ARG BASE=ubuntu:22.04
+
+FROM $BASE
+
+LABEL maintainer="AiiDAlab Team <[email protected]>"
+
+ARG SYSTEM_USER="aiida"
+ARG SYSTEM_UID="1000"
+ARG SYSTEM_GID="100"
+
+
+# Fix: https://github.com/hadolint/hadolint/wiki/DL4006
+# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+USER root
+
+ENV SYSTEM_USER="${SYSTEM_USER}"
+
+# Install all OS dependencies for notebook server that starts but lacks all
+# features (e.g., download as all possible file formats)
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update --yes && \
+    # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as
+    #   the ubuntu base image is rebuilt too seldom sometimes (less than once a month)
+    apt-get upgrade --yes && \
+    apt-get install --yes --no-install-recommends \
+    # - bzip2 is necessary to extract the micromamba executable.
+    bzip2 \
+    # - xz-utils is necessary to extract the s6-overlay.
+    xz-utils \
+    ca-certificates \
+    locales \
+    sudo \
+    # the gcc compiler need to build some python packages e.g. psutil and pymatgen
+    build-essential \
+    wget && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+    locale-gen
+
+# Install s6-overlay to handle startup and shutdown of services
+ARG S6_OVERLAY_VERSION=3.1.5.0
+RUN wget --progress=dot:giga -O /tmp/s6-overlay-noarch.tar.xz \
+        "https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz" && \
+    tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && \
+    rm /tmp/s6-overlay-noarch.tar.xz
+
+RUN set -x && \
+    arch=$(uname -m) && \
+    wget --progress=dot:giga -O /tmp/s6-overlay-binary.tar.xz \
+        "https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-${arch}.tar.xz" && \
+    tar -C / -Jxpf /tmp/s6-overlay-binary.tar.xz && \
+    rm /tmp/s6-overlay-binary.tar.xz
+
+# Configure environment
+ENV CONDA_DIR=/opt/conda \
+    SHELL=/bin/bash \
+    SYSTEM_USER="${SYSTEM_USER}" \
+    SYSTEM_UID=${SYSTEM_UID} \
+    SYSTEM_GID=${SYSTEM_GID} \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8
+ENV PATH="${CONDA_DIR}/bin:${PATH}" \
+    HOME="/home/${SYSTEM_USER}"
+
+
+# Copy a script that we will use to correct permissions after running certain commands
+COPY fix-permissions /usr/local/bin/fix-permissions
+RUN chmod a+rx /usr/local/bin/fix-permissions
+
+# Enable prompt color in the skeleton .bashrc before creating the default SYSTEM_USER
+# hadolint ignore=SC2016
+RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \
+   # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446
+   echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc
+
+# Create SYSTEM_USER with name jovyan user with UID=1000 and in the 'users' group
+# and make sure these dirs are writable by the `users` group.
+RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \
+    sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \
+    sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \
+    useradd -l -m -s /bin/bash -N -u "${SYSTEM_UID}" "${SYSTEM_USER}" && \
+    mkdir -p "${CONDA_DIR}" && \
+    chown "${SYSTEM_USER}:${SYSTEM_GID}" "${CONDA_DIR}" && \
+    chmod g+w /etc/passwd && \
+    fix-permissions "${HOME}" && \
+    fix-permissions "${CONDA_DIR}"
+
+USER ${SYSTEM_UID}
+
+# Pin python version here
+ARG PYTHON_VERSION
+
+# Download and install Micromamba, and initialize Conda prefix.
+#   <https://github.com/mamba-org/mamba#micromamba>
+#   Similar projects using Micromamba:
+#     - Micromamba-Docker: <https://github.com/mamba-org/micromamba-docker>
+#     - repo2docker: <https://github.com/jupyterhub/repo2docker>
+# Install Python, Mamba and jupyter_core
+# Cleanup temporary files and remove Micromamba
+# Correct permissions
+# Do all this in a single RUN command to avoid duplicating all of the
+# files across image layers when the permissions change
+COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" initial-condarc "${CONDA_DIR}/.condarc"
+WORKDIR /tmp
+RUN set -x && \
+    arch=$(uname -m) && \
+    if [ "${arch}" = "x86_64" ]; then \
+        # Should be simpler, see <https://github.com/mamba-org/mamba/issues/1437>
+        arch="64"; \
+    fi && \
+    wget --progress=dot:giga -O /tmp/micromamba.tar.bz2 \
+        "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \
+    tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \
+    rm /tmp/micromamba.tar.bz2 && \
+    PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \
+    if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \
+    # Install the packages
+    ./micromamba install \
+        --root-prefix="${CONDA_DIR}" \
+        --prefix="${CONDA_DIR}" \
+        --yes \
+        "${PYTHON_SPECIFIER}" \
+        'mamba' && \
+    rm micromamba && \
+    # Pin major.minor version of python
+    mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \
+    mamba clean --all -f -y && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${SYSTEM_USER}"
+
+# Switch to root to install AiiDA and set AiiDA as service
+# Install AiiDA from source code
+USER root
+COPY --from=src . /tmp/aiida-core
+RUN pip install /tmp/aiida-core --no-cache-dir && \
+    rm -rf /tmp/aiida-core
+
+# Enable verdi autocompletion.
+RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \
+     echo 'eval "$(_VERDI_COMPLETE=bash_source verdi)"' >> "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \
+     chmod +x "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \
+     fix-permissions "${CONDA_DIR}"
+
+# COPY AiiDA profile configuration for profile setup init script
+COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" s6-assets/config-quick-setup.yaml "/aiida/assets/config-quick-setup.yaml"
+COPY s6-assets/s6-rc.d /etc/s6-overlay/s6-rc.d
+COPY s6-assets/init /etc/init
+
+# Otherwise will stuck on oneshot services
+# https://github.com/just-containers/s6-overlay/issues/467
+ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0
+
+# Switch back to USER aiida to avoid accidental container runs as root
+USER ${SYSTEM_UID}
+
+ENTRYPOINT ["/init"]
+
+WORKDIR "${HOME}"