From 5dd41a21f175bca0933b757d5bd0dee20e967d31 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Tue, 25 Apr 2023 15:41:27 +0200 Subject: [PATCH 01/19] dependencies for image build --- .gitignore | 3 +++ dodo.py | 0 pyproject.toml | 5 ++++- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 dodo.py diff --git a/.gitignore b/.gitignore index 3cf188d3f3..67fb72c24c 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ docs/source/reference/apidoc _sandbox pplot_out/ + +# docker +.dodo.* \ No newline at end of file diff --git a/dodo.py b/dodo.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pyproject.toml b/pyproject.toml index aebcba57a3..3f03dfc831 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,13 +101,16 @@ notebook = [ ] pre-commit = [ "mypy==0.991", - "packaging==20.3", + "packaging~=20.9", "pre-commit~=2.2", "pylint~=2.17.4", "pylint-aiida~=0.1.1", "sqlalchemy[mypy]~=1.4.29", "tomli", "types-PyYAML", + # XXX: pre-commit -> dev, below are for docker image + "doit~=0.36", + "dunamai~=1.13", ] tests = [ "aiida-export-migration-tests==0.9.0", From 15bddbcbfecbf222a21e706e97e7bd2bec9171a1 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Tue, 25 Apr 2023 21:48:25 +0200 Subject: [PATCH 02/19] WIP: basic baker --- .docker/build.json | 11 ++++ .docker/docker-bake.hcl | 69 +++++++++++++++++++++++ .gitignore | 3 +- dodo.py | 118 ++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + 5 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 .docker/build.json create mode 100644 .docker/docker-bake.hcl diff --git a/.docker/build.json b/.docker/build.json new file mode 100644 index 0000000000..3bfe39ce05 --- /dev/null +++ b/.docker/build.json @@ -0,0 +1,11 @@ +{ + "variable": { + "PYTHON_VERSION": { + "default": "3.9.13" + }, + "PGSQL_VERSION": { + "default": "15" + } + } + } + \ No newline at end of file diff --git a/.docker/docker-bake.hcl b/.docker/docker-bake.hcl new file mode 100644 index 0000000000..0da2c5c17a --- /dev/null +++ b/.docker/docker-bake.hcl @@ -0,0 +1,69 @@ +# docker-bake.hcl +variable "VERSION" { +} + +variable "PYTHON_VERSION" { +} + +variable "PGSQL_VERSION" { +} + +variable "BASE_IMAGE" { + default = "jupyter/minimal-notebook:python-${PYTHON_VERSION}" +} + +variable "ORGANIZATION" { + default = "aiidateam" +} + +variable "REGISTRY" { + default = "docker.io/" +} + +variable "PLATFORMS" { + default = ["linux/amd64", "linux/arm64"] +} + +variable "TARGETS" { + default = ["base", "base-with-services"] +} + +function "tags" { + params = [image] + result = [ + "${REGISTRY}${ORGANIZATION}/${image}:${VERSION}", + "${REGISTRY}${ORGANIZATION}/${image}:python-${PYTHON_VERSION}", + "${REGISTRY}${ORGANIZATION}/${image}:postgresql-${PGSQL_VERSION}", + ] +} + +group "default" { + targets = "${TARGETS}" +} + +target "base-meta" { + tags = tags("base") +} +target "base-with-services-meta" { + tags = tags("base-with-services") +} + +target "base" { + inherits = ["base-meta"] + context = "stack/base" + platforms = "${PLATFORMS}" + args = { + "BASE" = "${BASE_IMAGE}" + } +} +target "base-with-services" { + inherits = ["base-with-services-meta"] + context = "stack/base-with-services" + contexts = { + base = "target:base" + } + platforms = "${PLATFORMS}" + args = { + "PGSQL_VERSION" = "${PGSQL_VERSION}" + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 67fb72c24c..d41253317a 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,5 @@ _sandbox pplot_out/ # docker -.dodo.* \ No newline at end of file +.doit.* +docker-bake.override.json \ No newline at end of file diff --git a/dodo.py b/dodo.py index e69de29bb2..502fe18a76 100644 --- a/dodo.py +++ b/dodo.py @@ -0,0 +1,118 @@ +import json +import platform +from pathlib import Path + +from doit.tools import title_with_actions + +import docker +from dunamai import Version + +_DOCKER_CLIENT = docker.from_env() +_DOCKER_ARCHITECTURE = _DOCKER_CLIENT.info()["Architecture"] + +DOIT_CONFIG = {"default_tasks": ["build"]} + +_ARCH_PLATFORM_MAPPING = { + "aarch64": "linux/arm64", + "x86_64": "linux/amd64", +} + +VERSION = Version.from_git().serialize().replace("+", "_") +PLATFORM = _ARCH_PLATFORM_MAPPING.get(_DOCKER_ARCHITECTURE) + +if PLATFORM is None: + raise RuntimeError( + f"Unsupported architecture {_DOCKER_ARCHITECTURE} on platform {platform.system()}." + ) + +_REGISTRY_PARAM = { + "name": "registry", + "short": "r", + "long": "registry", + "type": str, + "default": "", + "help": "Specify the docker image registry.", +} + +_VERSION_PARAM = { + "name": "version", + "long": "version", + "type": "str", + "default": VERSION, + "help": ( + "Specify the version of the stack for building / testing. Defaults to a " + "version determined from the state of the local git repository." + ), +} + +_PLATFORM_PARAM = { + "name": "platform", + "long": "platform", + "type": str, + "default": PLATFORM, + "help": "Specify the platform to build for. Examples: linux/amd64, linux/arm64", +} + + +def task_build(): + """Build all docker images.""" + + def generate_version_override(version, registry): + Path(".docker/docker-bake.override.json").write_text( + json.dumps(dict(VERSION=version, REGISTRY=registry)) + ) + + return { + "actions": [ + generate_version_override, + "docker buildx bake -f .docker/docker-bake.hcl -f .docker/build.json " + "-f .docker/docker-bake.override.json " + "--set '*.platform=%(platform)s' " + "--load", + ], + "title": title_with_actions, + "params": [ + _REGISTRY_PARAM, + _VERSION_PARAM, + _PLATFORM_PARAM, + ], + "verbosity": 2, + } + + +def task_tests(): + """Run tests with pytest.""" + + return { + "actions": ["REGISTRY=%(registry)s VERSION=:%(version)s pytest -v"], + "params": [_REGISTRY_PARAM, _VERSION_PARAM], + "verbosity": 2, + } + + +def task_up(): + """Start AiiDAlab server for testing.""" + return { + "actions": [ + "AIIDALAB_PORT=%(port)i REGISTRY=%(registry)s VERSION=:%(version)s " + "docker-compose up --detach" + ], + "params": [ + { + "name": "port", + "short": "p", + "long": "port", + "type": int, + "default": 8888, + "help": "Specify the AiiDAlab host port.", + }, + _REGISTRY_PARAM, + _VERSION_PARAM, + ], + "verbosity": 2, + } + + +def task_down(): + """Stop AiiDAlab server.""" + return {"actions": ["docker-compose down"], "verbosity": 2} diff --git a/pyproject.toml b/pyproject.toml index 3f03dfc831..6c41c7935e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,7 @@ pre-commit = [ # XXX: pre-commit -> dev, below are for docker image "doit~=0.36", "dunamai~=1.13", + "docker~=5.0", ] tests = [ "aiida-export-migration-tests==0.9.0", From 09d541af7e460b0c1eb3de5131811b035e6db910 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Wed, 26 Apr 2023 01:59:43 +0200 Subject: [PATCH 03/19] base image can run --- .docker/docker-bake.hcl | 14 +- .docker/docker-compose.yml | 44 ++++++ .docker/docker-rabbitmq.yml | 34 ----- .docker/dodo.py | 131 ++++++++++++++++++ .docker/my_init.d/configure-aiida.sh | 4 - .docker/stack/base-with-services/Dockerfile | 63 +++++++++ .../before-notebook.d/20_start-postgresql.sh | 36 +++++ .../30_start-rabbitmq-amd64.sh | 25 ++++ .../30_start-rabbitmq-arm64.sh | 32 +++++ .../41_suppress-rabbitmq-version-warning.sh | 7 + .../config-quick-setup.yaml | 3 + .docker/stack/base/Dockerfile | 92 ++++++++++++ .docker/stack/base/config-quick-setup.yaml | 15 ++ .docker/stack/base/init.d/_init.sh | 11 ++ .../base/init.d/aiida_profile_preparation.sh} | 55 +++----- .pre-commit-config.yaml | 1 + dodo.py | 118 ---------------- 17 files changed, 493 insertions(+), 192 deletions(-) create mode 100644 .docker/docker-compose.yml delete mode 100644 .docker/docker-rabbitmq.yml create mode 100644 .docker/dodo.py delete mode 100755 .docker/my_init.d/configure-aiida.sh create mode 100644 .docker/stack/base-with-services/Dockerfile create mode 100644 .docker/stack/base-with-services/before-notebook.d/20_start-postgresql.sh create mode 100644 .docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-amd64.sh create mode 100644 .docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-arm64.sh create mode 100644 .docker/stack/base-with-services/before-notebook.d/41_suppress-rabbitmq-version-warning.sh create mode 100644 .docker/stack/base-with-services/config-quick-setup.yaml create mode 100644 .docker/stack/base/Dockerfile create mode 100644 .docker/stack/base/config-quick-setup.yaml create mode 100644 .docker/stack/base/init.d/_init.sh rename .docker/{opt/configure-aiida.sh => stack/base/init.d/aiida_profile_preparation.sh} (66%) delete mode 100644 dodo.py diff --git a/.docker/docker-bake.hcl b/.docker/docker-bake.hcl index 0da2c5c17a..08accbb33c 100644 --- a/.docker/docker-bake.hcl +++ b/.docker/docker-bake.hcl @@ -8,8 +8,15 @@ variable "PYTHON_VERSION" { variable "PGSQL_VERSION" { } +variable "AIIDA_VERSION" { +} + +variable "PIP_VERSION" { + default = "22.0.4" +} + variable "BASE_IMAGE" { - default = "jupyter/minimal-notebook:python-${PYTHON_VERSION}" + default = "mambaorg/micromamba:jammy" } variable "ORGANIZATION" { @@ -54,6 +61,9 @@ target "base" { platforms = "${PLATFORMS}" args = { "BASE" = "${BASE_IMAGE}" + "PYTHON_VERSION" = "${PYTHON_VERSION}" + "PIP_VERSION" = "${PIP_VERSION}" + "AIIDA_VERSION" = "${AIIDA_VERSION}" } } target "base-with-services" { @@ -66,4 +76,4 @@ target "base-with-services" { args = { "PGSQL_VERSION" = "${PGSQL_VERSION}" } -} \ No newline at end of file +} diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml new file mode 100644 index 0000000000..37ae6f0793 --- /dev/null +++ b/.docker/docker-compose.yml @@ -0,0 +1,44 @@ +--- +version: '3.4' + +services: + + database: + image: postgres:15 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + volumes: + - aiida-postgres-db:/var/lib/postgresql/data + healthcheck: + interval: 5s + timeout: 5s + retries: 10 + + messaging: + image: rabbitmq:3.8.14-management + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + volumes: + - aiida-rmq-data:/var/lib/rabbitmq/ + + aiida: + image: ${REGISTRY:-}${BASE_IMAGE:-aiidateam/base}${VERSION:-} + environment: + RMQHOST: messaging + TZ: Europe/Zurich + SETUP_DEFAULT_AIIDA_PROFILE: 'true' + volumes: + - aiida-home-folder:/home/aiida + depends_on: + database: + condition: service_started + messaging: + condition: service_started + command: tail -f /dev/null + +volumes: + aiida-postgres-db: + aiida-rmq-data: + aiida-home-folder: diff --git a/.docker/docker-rabbitmq.yml b/.docker/docker-rabbitmq.yml deleted file mode 100644 index da266790ff..0000000000 --- a/.docker/docker-rabbitmq.yml +++ /dev/null @@ -1,34 +0,0 @@ -# A small configuration for use in local CI testing, -# if you wish to control the rabbitmq used. - -# Simply install docker, then run: -# $ docker-compose -f .docker/docker-rabbitmq.yml up -d - -# and to power down, after testing: -# $ docker-compose -f .docker/docker-rabbitmq.yml down - -# you can monitor rabbitmq use at: http://localhost:15672 - -version: '3.4' - -services: - - rabbit: - image: rabbitmq:3.8.3-management - container_name: aiida-rmq - environment: - RABBITMQ_DEFAULT_USER: guest - RABBITMQ_DEFAULT_PASS: guest - ports: - - '5672:5672' - - '15672:15672' - healthcheck: - test: rabbitmq-diagnostics -q ping - interval: 30s - timeout: 30s - retries: 5 - networks: - - aiida-rmq - -networks: - aiida-rmq: diff --git a/.docker/dodo.py b/.docker/dodo.py new file mode 100644 index 0000000000..3056970536 --- /dev/null +++ b/.docker/dodo.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +"""Doit tasks for building and testing the docker images. +The idea is borrowed from AiiDAlab docker stack. +""" +import json +from pathlib import Path +import platform + +import docker # pylint: disable=import-error +from doit.tools import title_with_actions # pylint: disable=import-error +from dunamai import Version # pylint: disable=import-error + +_DOCKER_CLIENT = docker.from_env() +_DOCKER_ARCHITECTURE = _DOCKER_CLIENT.info()['Architecture'] + +DOIT_CONFIG = {'default_tasks': ['build']} + +_ARCH_PLATFORM_MAPPING = { + 'aarch64': 'linux/arm64', + 'x86_64': 'linux/amd64', +} + +VERSION = Version.from_git().serialize().replace('+', '_') +PLATFORM = _ARCH_PLATFORM_MAPPING.get(_DOCKER_ARCHITECTURE) + +if PLATFORM is None: + raise RuntimeError(f'Unsupported architecture {_DOCKER_ARCHITECTURE} on platform {platform.system()}.') + +_REGISTRY_PARAM = { + 'name': 'registry', + 'short': 'r', + 'long': 'registry', + 'type': str, + 'default': '', + 'help': 'Specify the docker image registry.', +} + +_VERSION_PARAM = { + 'name': 'version', + 'long': 'version', + 'type': 'str', + 'default': VERSION, + 'help': ('Specify the AiiDA version to build for.',), +} + +_AIIDA_VERSION_PARAM = { + 'name': 'aiida_version', + 'long': 'aiida-version', + 'type': 'str', + 'default': '', + 'help': ( + 'Specify the version of the stack for building / testing. Defaults to a ' + 'version determined from the state of the local git repository.' + ), +} + +_PLATFORM_PARAM = { + 'name': 'platform', + 'long': 'platform', + 'type': str, + 'default': PLATFORM, + 'help': 'Specify the platform to build for. Examples: linux/amd64, linux/arm64', +} + + +def task_build(): + """Build all docker images.""" + + def generate_version_override(version, registry, targets): + if len(targets) > 2: + # Workaround of issue of doit, which rather than override the default value, it appends + # https://github.com/pydoit/doit/issues/436 + targets = targets[2:] + Path('docker-bake.override.json').write_text( + json.dumps(dict(VERSION=version, REGISTRY=registry, TARGETS=targets)), + encoding='utf-8', + ) + + return { + 'actions': [ + generate_version_override, + 'docker buildx bake -f docker-bake.hcl -f build.json ' + '-f docker-bake.override.json ' + "--set '*.platform=%(platform)s' " + '--load', + ], + 'title': title_with_actions, + 'params': [ + _REGISTRY_PARAM, + _VERSION_PARAM, + _PLATFORM_PARAM, + _AIIDA_VERSION_PARAM, + { + 'name': 'targets', + 'long': 'targets', + 'short': 't', + 'type': list, + 'default': ['base', 'base-with-services'], + 'help': 'Specify the target to build.', + }, + ], + 'verbosity': 2, + } + + +def task_tests(): + """Run tests with pytest.""" + + return { + 'actions': ['REGISTRY=%(registry)s VERSION=:%(version)s pytest -v'], + 'params': [_REGISTRY_PARAM, _VERSION_PARAM], + 'verbosity': 2, + } + + +def task_up(): + """Start AiiDAlab server for testing.""" + return { + 'actions': ['REGISTRY=%(registry)s VERSION=:%(version)s ' + 'docker-compose up --detach'], + 'params': [ + _REGISTRY_PARAM, + _VERSION_PARAM, + ], + 'verbosity': 2, + } + + +def task_down(): + """Stop AiiDAlab server.""" + return {'actions': ['docker-compose down'], 'verbosity': 2} diff --git a/.docker/my_init.d/configure-aiida.sh b/.docker/my_init.d/configure-aiida.sh deleted file mode 100755 index 7ac4476b07..0000000000 --- a/.docker/my_init.d/configure-aiida.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -em - -su -c /opt/configure-aiida.sh ${SYSTEM_USER} diff --git a/.docker/stack/base-with-services/Dockerfile b/.docker/stack/base-with-services/Dockerfile new file mode 100644 index 0000000000..3e2d30ab9c --- /dev/null +++ b/.docker/stack/base-with-services/Dockerfile @@ -0,0 +1,63 @@ +# syntax=docker/dockerfile:1 +FROM base + +LABEL maintainer="AiiDAlab Team " + +USER root +WORKDIR /opt/ + +ARG AIIDA_VERSION +ARG PGSQL_VERSION +ARG TARGETARCH + +RUN mamba create -p /opt/conda/envs/aiida-core-services --yes \ + postgresql=${PGSQL_VERSION} \ + && mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Install RabbitMQ in a dedicated conda environment. +# If the architecture is arm64, we install the default version of rabbitmq provided by the generic binary, +# # https://www.rabbitmq.com/install-generic-unix.html the version needs to be compatible with system's erlang version. +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + mamba install -p /opt/conda/envs/aiida-core-services --yes \ + rabbitmq-server=3.8.14 && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}"; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + # Install erlang. + apt-get update --yes && \ + apt-get install --yes --no-install-recommends erlang && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + # Install rabbitmq. + export RMQ_VERSION=3.9.13 && \ + wget -c https://github.com/rabbitmq/rabbitmq-server/releases/download/v${RMQ_VERSION}/rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ + tar -xf rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ + rm rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ + mv rabbitmq_server-${RMQ_VERSION} /opt/conda/envs/aiida-core-services/ && \ + fix-permissions "/opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}" && \ + ln -sf /opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/sbin/* /opt/conda/envs/aiida-core-services/bin/; \ +else \ + echo "Unknown architecture: ${TARGETARCH}."; \ +fi + +# Configure AiiDA profile. +COPY config-quick-setup.yaml . +COPY before-notebook.d/20_start-postgresql.sh /usr/local/bin/before-notebook.d/ +COPY before-notebook.d/30_start-rabbitmq-${TARGETARCH}.sh /usr/local/bin/before-notebook.d/ + +# Supress rabbitmq version warning for arm64 since +# it is built using latest version rabbitmq from apt install. +# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf +COPY before-notebook.d/41_suppress-rabbitmq-version-warning.sh /usr/local/bin/before-notebook.d/ +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + rm /usr/local/bin/before-notebook.d/41_suppress-rabbitmq-version-warning.sh; \ +fi + +USER ${NB_USER} + +WORKDIR "/home/${NB_USER}" + +# Initialize the database +RUN mamba run -n aiida-core-services initdb -D aiida_db -U aiida diff --git a/.docker/stack/base-with-services/before-notebook.d/20_start-postgresql.sh b/.docker/stack/base-with-services/before-notebook.d/20_start-postgresql.sh new file mode 100644 index 0000000000..0eb685e7d8 --- /dev/null +++ b/.docker/stack/base-with-services/before-notebook.d/20_start-postgresql.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Activate the conda environment with PostgreSQL installed in it. +# conda activate pgsql + +# -w waits until server is up +PSQL_START_CMD="pg_ctl --timeout=180 -w -D /home/${NB_USER}/.postgresql -l /home/${NB_USER}/.postgresql/logfile start" +PSQL_STOP_CMD="pg_ctl -w -D /home/${NB_USER}/.postgresql stop" +PSQL_STATUS_CMD="pg_ctl -D /home/${NB_USER}/.postgresql status" + +MAMBA_RUN="mamba run -n aiida-core-services" + +# make DB directory, if not existent +if [ ! -d /home/${NB_USER}/.postgresql ]; then + mkdir /home/${NB_USER}/.postgresql + ${MAMBA_RUN} initdb -D /home/${NB_USER}/.postgresql + echo "unix_socket_directories = '/tmp'" >> /home/${NB_USER}/.postgresql/postgresql.conf + ${MAMBA_RUN} ${PSQL_START_CMD} + +# else don't +else + # Fix problem with kubernetes cluster that adds rws permissions to the group + # for more details see: https://github.com/materialscloud-org/aiidalab-z2jh-eosc/issues/5 + chmod g-rwxs /home/${NB_USER}/.postgresql -R + + # stores return value in $? + running=true + ${MAMBA_RUN} ${PSQL_STATUS_CMD} || running=false + + # Postgresql was probably not shutdown properly. Cleaning up the mess... + if ! $running ; then + echo "" > /home/${NB_USER}/.postgresql/logfile # empty log files + rm -vf /home/${NB_USER}/.postgresql/postmaster.pid + ${MAMBA_RUN} ${PSQL_START_CMD} + fi +fi diff --git a/.docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-amd64.sh b/.docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-amd64.sh new file mode 100644 index 0000000000..5c55fff3fc --- /dev/null +++ b/.docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-amd64.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -em + +RABBITMQ_DATA_DIR="/home/${NB_USER}/.rabbitmq" + +mkdir -p "${RABBITMQ_DATA_DIR}" +fix-permissions "${RABBITMQ_DATA_DIR}" + +# Fix issue where the erlang cookie permissions are corrupted. +chmod 400 "/home/${NB_USER}/.erlang.cookie" || echo "erlang cookie not created yet." + +# Set base directory for RabbitMQ to persist its data. This needs to be set to a folder in the system user's home +# directory as that is the only folder that is persisted outside of the container. +RMQ_ETC_DIR="/opt/conda/envs/aiida-core-services/etc/rabbitmq" +echo MNESIA_BASE="${RABBITMQ_DATA_DIR}" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" +echo LOG_BASE="${RABBITMQ_DATA_DIR}/log" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" + +# Explicitly define the node name. This is necessary because the mnesia subdirectory contains the hostname, which by +# default is set to the value of $(hostname -s), which for docker containers, will be a random hexadecimal string. Upon +# restart, this will be different and so the original mnesia folder with the persisted data will not be found. The +# reason RabbitMQ is built this way is through this way it allows to run multiple nodes on a single machine each with +# isolated mnesia directories. Since in the AiiDA setup we only need and run a single node, we can simply use localhost. +echo NODENAME=rabbit@localhost >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" + +mamba run -n aiida-core-services rabbitmq-server -detached diff --git a/.docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-arm64.sh b/.docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-arm64.sh new file mode 100644 index 0000000000..c19763a220 --- /dev/null +++ b/.docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-arm64.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -em + +RABBITMQ_DATA_DIR="/home/${NB_USER}/.rabbitmq" + +mkdir -p "${RABBITMQ_DATA_DIR}" +fix-permissions "${RABBITMQ_DATA_DIR}" + +# Fix issue where the erlang cookie permissions are corrupted. +chmod 400 "/home/${NB_USER}/.erlang.cookie" || echo "erlang cookie not created yet." + +# Set base directory for RabbitMQ to persist its data. This needs to be set to a folder in the system user's home +# directory as that is the only folder that is persisted outside of the container. +export RMQ_VERSION=3.9.13 +RMQ_ETC_DIR="/opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/etc/rabbitmq" +echo MNESIA_BASE="${RABBITMQ_DATA_DIR}" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" +echo LOG_BASE="${RABBITMQ_DATA_DIR}/log" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" + +# RabbitMQ with versions >= 3.8.15 have reduced some default timeouts +# baseimage phusion/baseimage:jammy-1.0.0 running ubuntu 22.04 will install higher version of rabbimq by apt. +# using workaround from https://github.com/aiidateam/aiida-core/wiki/RabbitMQ-version-to-use +# set timeout to 100 hours +echo "consumer_timeout=3600000" >> "${RMQ_ETC_DIR}/rabbitmq.conf" + +# Explicitly define the node name. This is necessary because the mnesia subdirectory contains the hostname, which by +# default is set to the value of $(hostname -s), which for docker containers, will be a random hexadecimal string. Upon +# restart, this will be different and so the original mnesia folder with the persisted data will not be found. The +# reason RabbitMQ is built this way is through this way it allows to run multiple nodes on a single machine each with +# isolated mnesia directories. Since in the AiiDA setup we only need and run a single node, we can simply use localhost. +echo NODENAME=rabbit@localhost >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" + +mamba run -n aiida-core-services rabbitmq-server -detached diff --git a/.docker/stack/base-with-services/before-notebook.d/41_suppress-rabbitmq-version-warning.sh b/.docker/stack/base-with-services/before-notebook.d/41_suppress-rabbitmq-version-warning.sh new file mode 100644 index 0000000000..f778bbb5db --- /dev/null +++ b/.docker/stack/base-with-services/before-notebook.d/41_suppress-rabbitmq-version-warning.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -em + +# Supress rabbitmq version warning for arm64 since +# it is built using latest version rabbitmq from apt install. +# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf +verdi config set warnings.rabbitmq_version False diff --git a/.docker/stack/base-with-services/config-quick-setup.yaml b/.docker/stack/base-with-services/config-quick-setup.yaml new file mode 100644 index 0000000000..24c516270d --- /dev/null +++ b/.docker/stack/base-with-services/config-quick-setup.yaml @@ -0,0 +1,3 @@ +--- +db_name: aiida_db +db_username: aiida diff --git a/.docker/stack/base/Dockerfile b/.docker/stack/base/Dockerfile new file mode 100644 index 0000000000..605504a0ed --- /dev/null +++ b/.docker/stack/base/Dockerfile @@ -0,0 +1,92 @@ +ARG BASE +FROM ${BASE} + +LABEL maintainer="AiiDAlab Team " + +ARG NEW_MAMBA_USER=aiida +ARG NEW_MAMBA_USER_ID=1000 +ARG NEW_MAMBA_USER_GID=1000 + +USER root +RUN usermod "--login=${NEW_MAMBA_USER}" "--home=/home/${NEW_MAMBA_USER}" \ + --move-home "-u ${NEW_MAMBA_USER_ID}" "${MAMBA_USER}" && \ + groupmod "--new-name=${NEW_MAMBA_USER}" \ + "-g ${NEW_MAMBA_USER_GID}" "${MAMBA_USER}" && \ + # Update the expected value of MAMBA_USER for the + # _entrypoint.sh consistency check. + echo "${NEW_MAMBA_USER}" > "/etc/arg_mamba_user" && \ + : + +ENV MAMBA_USER=$NEW_MAMBA_USER + +# Pin python and pip version +ARG PYTHON_VERSION +ARG PIP_VERSION + +# The docker only build after the release of conda-forge. +ARG AIIDA_VERSION + +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + wget && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Install the python base requirements to base environment. +# XXX: bring back mamba-bash-completion once it is available on conda-forge. +# https://github.com/conda-forge/mamba-bash-completion-feedstock/pull/2 +RUN micromamba install --yes \ + --channel conda-forge \ + -n base \ + openssh \ + python=${PYTHON_VERSION} \ + pip=${PIP_VERSION} \ + aiida-core=${AIIDA_VERSION} \ + && micromamba clean --all -f -y + +# Pin shared requirements in the base environemnt. +# XXX: ironup +RUN echo "pip==${PIP_VERSION}" >> /opt/requirements.txt +RUN echo "python==${PYTHON_VERSION}" >> /opt/requirements.txt +RUN echo "aiida-core==${AIIDA_VERSION}" >> /opt/requirements.txt + +# Configure pip to use requirements file as constraints file. +ENV PIP_CONSTRAINT=/opt/requirements.txt + +# copy the entrypoint script +COPY --chmod=755 init.d/* /usr/local/bin + +# Enable verdi autocompletion. +RUN mkdir -p "/opt/conda/etc/conda/activate.d" && \ + echo 'eval "$(_VERDI_COMPLETE=bash_source verdi)"' >> "/opt/conda/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ + chmod +x "/opt/conda/etc/conda/activate.d/activate_aiida_autocompletion.sh" + +# Install the load-singlesshagent.sh script as described here: +# https://aiida.readthedocs.io/projects/aiida-core/en/v2.0.0/howto/ssh.html#starting-the-ssh-agent +RUN wget --no-check-certificate --directory-prefix=/opt/conda/ \ + "https://aiida.readthedocs.io/projects/aiida-core/en/v2.0.0/_downloads/4265ec5a42c3a3dba586dd460c0db95e/load-singlesshagent.sh" + +# SSH agent configuration + +# Make sure that the known_hosts file is present inside the .ssh folder. +RUN mkdir -p --mode=0700 /home/${MAMBA_USER}/.ssh && \ + touch /home/${MAMBA_USER}/.ssh/known_hosts + +RUN /usr/local/bin/_entrypoint.sh ssh-keygen -f /home/${MAMBA_USER}/.ssh/id_rsa -t rsa -b 4096 -m PEM -N '' + +# Configure AiiDA profile. +COPY config-quick-setup.yaml /opt/config-quick-setup.yaml + +# Environment variables for AiiDA profile setup +ENV SETUP_DEFAULT_AIIDA_PROFILE true +ENV AIIDA_PROFILE_NAME default +ENV AIIDA_USER_EMAIL aiida@localhost +ENV AIIDA_USER_FIRST_NAME Giuseppe +ENV AIIDA_USER_LAST_NAME Verdi +ENV AIIDA_USER_INSTITUTION Khedivial + +# Switch to system user +USER $MAMBA_USER + +WORKDIR /home/${MAMBA_USER} + +ENTRYPOINT [ "/usr/local/bin/_init.sh" ] diff --git a/.docker/stack/base/config-quick-setup.yaml b/.docker/stack/base/config-quick-setup.yaml new file mode 100644 index 0000000000..f910069e1d --- /dev/null +++ b/.docker/stack/base/config-quick-setup.yaml @@ -0,0 +1,15 @@ +--- +db_engine: postgresql_psycopg2 +db_backend: core.psql_dos +db_host: database +db_port: 5432 +su_db_username: postgres +su_db_password: password +su_db_name: template1 +db_name: aiida_db +db_username: aiida +db_password: password +broker_host: messaging +broker_port: 5672 +broker_username: guest +broker_password: guest diff --git a/.docker/stack/base/init.d/_init.sh b/.docker/stack/base/init.d/_init.sh new file mode 100644 index 0000000000..5a60386273 --- /dev/null +++ b/.docker/stack/base/init.d/_init.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# This is the entrypoint script for the docker container. + +# Start the ssh-agent +/usr/local/bin/_entrypoint.sh ssh-agent + +# setup aiida must be run in the environment +/usr/local/bin/_entrypoint.sh /usr/local/bin/aiida_profile_preparation.sh + +exec "$@" diff --git a/.docker/opt/configure-aiida.sh b/.docker/stack/base/init.d/aiida_profile_preparation.sh similarity index 66% rename from .docker/opt/configure-aiida.sh rename to .docker/stack/base/init.d/aiida_profile_preparation.sh index 92aa3ab45d..869aa3d591 100755 --- a/.docker/opt/configure-aiida.sh +++ b/.docker/stack/base/init.d/aiida_profile_preparation.sh @@ -8,11 +8,8 @@ set -x # Environment. export SHELL=/bin/bash -# Setup AiiDA autocompletion. -grep _VERDI_COMPLETE /home/${SYSTEM_USER}/.bashrc &> /dev/null || echo 'eval "$(_VERDI_COMPLETE=source verdi)"' >> /home/${SYSTEM_USER}/.bashrc - # Check if user requested to set up AiiDA profile (and if it exists already) -if [[ ${SETUP_DEFAULT_PROFILE} == true ]] && ! verdi profile show ${PROFILE_NAME} &> /dev/null; then +if [[ ${SETUP_DEFAULT_AIIDA_PROFILE} == true ]] && ! verdi profile show ${AIIDA_PROFILE_NAME} &> /dev/null; then NEED_SETUP_PROFILE=true; else NEED_SETUP_PROFILE=false; @@ -22,15 +19,14 @@ fi if [[ ${NEED_SETUP_PROFILE} == true ]]; then # Create AiiDA profile. - verdi quicksetup \ - --non-interactive \ - --profile "${PROFILE_NAME}" \ - --email "${USER_EMAIL}" \ - --first-name "${USER_FIRST_NAME}" \ - --last-name "${USER_LAST_NAME}" \ - --institution "${USER_INSTITUTION}" \ - --db-host "${DB_HOST:localhost}" \ - --broker-host "${BROKER_HOST:localhost}" + verdi quicksetup \ + --non-interactive \ + --profile "${AIIDA_PROFILE_NAME}" \ + --email "${AIIDA_USER_EMAIL}" \ + --first-name "${AIIDA_USER_FIRST_NAME}" \ + --last-name "${AIIDA_USER_LAST_NAME}" \ + --institution "${AIIDA_USER_INSTITUTION}" \ + --config /opt/config-quick-setup.yaml # Setup and configure local computer. computer_name=localhost @@ -52,18 +48,18 @@ if [[ ${NEED_SETUP_PROFILE} == true ]]; then exit 1 fi - verdi computer show ${computer_name} || verdi computer setup \ - --non-interactive \ - --label "${computer_name}" \ - --description "this computer" \ - --hostname "${computer_name}" \ + verdi computer show ${computer_name} || verdi computer setup \ + --non-interactive \ + --label "${computer_name}" \ + --description "this computer" \ + --hostname "${computer_name}" \ --transport core.local \ - --scheduler core.direct \ - --work-dir /home/aiida/aiida_run/ \ - --mpirun-command "mpirun -np {tot_num_mpiprocs}" \ - --mpiprocs-per-machine ${LOCALHOST_MPI_PROCS_PER_MACHINE} && \ - verdi computer configure core.local "${computer_name}" \ - --non-interactive \ + --scheduler core.direct \ + --work-dir /home/${NB_USER}/aiida_run/ \ + --mpirun-command "mpirun -np {tot_num_mpiprocs}" \ + --mpiprocs-per-machine ${LOCALHOST_MPI_PROCS_PER_MACHINE} && \ + verdi computer configure core.local "${computer_name}" \ + --non-interactive \ --safe-interval 0.0 fi @@ -75,16 +71,7 @@ verdi profile show || echo "The default profile is not set." verdi daemon stop # Migration will run for the default profile. -verdi storage migrate --force || echo "Database migration failed." - -# Supress rabbitmq version warning for arm64 since -# the it build using latest version rabbitmq from apt install -# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf -export ARCH=`uname -m` -if [ "$ARCH" = "aarch64" ]; then \ - verdi config set warnings.rabbitmq_version False -fi - +verdi storage migrate --force # Daemon will start only if the database exists and is migrated to the latest version. verdi daemon start || echo "AiiDA daemon is not running." diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b21190151f..2da795fc31 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,6 +74,7 @@ repos: (?x)^( .github/.*| .molecule/.*| + .docker/.*| docs/.*| utils/.*| diff --git a/dodo.py b/dodo.py deleted file mode 100644 index 502fe18a76..0000000000 --- a/dodo.py +++ /dev/null @@ -1,118 +0,0 @@ -import json -import platform -from pathlib import Path - -from doit.tools import title_with_actions - -import docker -from dunamai import Version - -_DOCKER_CLIENT = docker.from_env() -_DOCKER_ARCHITECTURE = _DOCKER_CLIENT.info()["Architecture"] - -DOIT_CONFIG = {"default_tasks": ["build"]} - -_ARCH_PLATFORM_MAPPING = { - "aarch64": "linux/arm64", - "x86_64": "linux/amd64", -} - -VERSION = Version.from_git().serialize().replace("+", "_") -PLATFORM = _ARCH_PLATFORM_MAPPING.get(_DOCKER_ARCHITECTURE) - -if PLATFORM is None: - raise RuntimeError( - f"Unsupported architecture {_DOCKER_ARCHITECTURE} on platform {platform.system()}." - ) - -_REGISTRY_PARAM = { - "name": "registry", - "short": "r", - "long": "registry", - "type": str, - "default": "", - "help": "Specify the docker image registry.", -} - -_VERSION_PARAM = { - "name": "version", - "long": "version", - "type": "str", - "default": VERSION, - "help": ( - "Specify the version of the stack for building / testing. Defaults to a " - "version determined from the state of the local git repository." - ), -} - -_PLATFORM_PARAM = { - "name": "platform", - "long": "platform", - "type": str, - "default": PLATFORM, - "help": "Specify the platform to build for. Examples: linux/amd64, linux/arm64", -} - - -def task_build(): - """Build all docker images.""" - - def generate_version_override(version, registry): - Path(".docker/docker-bake.override.json").write_text( - json.dumps(dict(VERSION=version, REGISTRY=registry)) - ) - - return { - "actions": [ - generate_version_override, - "docker buildx bake -f .docker/docker-bake.hcl -f .docker/build.json " - "-f .docker/docker-bake.override.json " - "--set '*.platform=%(platform)s' " - "--load", - ], - "title": title_with_actions, - "params": [ - _REGISTRY_PARAM, - _VERSION_PARAM, - _PLATFORM_PARAM, - ], - "verbosity": 2, - } - - -def task_tests(): - """Run tests with pytest.""" - - return { - "actions": ["REGISTRY=%(registry)s VERSION=:%(version)s pytest -v"], - "params": [_REGISTRY_PARAM, _VERSION_PARAM], - "verbosity": 2, - } - - -def task_up(): - """Start AiiDAlab server for testing.""" - return { - "actions": [ - "AIIDALAB_PORT=%(port)i REGISTRY=%(registry)s VERSION=:%(version)s " - "docker-compose up --detach" - ], - "params": [ - { - "name": "port", - "short": "p", - "long": "port", - "type": int, - "default": 8888, - "help": "Specify the AiiDAlab host port.", - }, - _REGISTRY_PARAM, - _VERSION_PARAM, - ], - "verbosity": 2, - } - - -def task_down(): - """Stop AiiDAlab server.""" - return {"actions": ["docker-compose down"], "verbosity": 2} From 4dcc7625fda1aef7797fbe7bf9c606e9b6318782 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Thu, 27 Apr 2023 22:26:07 +0200 Subject: [PATCH 04/19] with-service stack --- .docker/stack/base-with-services/Dockerfile | 24 +++++++++---------- .../stack/base-with-services/init.d/_init.sh | 20 ++++++++++++++++ .../start-postgresql.sh} | 0 .../start-rabbitmq-amd64.sh} | 0 .../start-rabbitmq-arm64.sh} | 0 .../suppress-rabbitmq-version-warning.sh} | 0 .docker/stack/base/Dockerfile | 2 +- .docker/stack/base/init.d/_init.sh | 2 +- 8 files changed, 33 insertions(+), 15 deletions(-) create mode 100644 .docker/stack/base-with-services/init.d/_init.sh rename .docker/stack/base-with-services/{before-notebook.d/20_start-postgresql.sh => init.d/start-postgresql.sh} (100%) rename .docker/stack/base-with-services/{before-notebook.d/30_start-rabbitmq-amd64.sh => init.d/start-rabbitmq-amd64.sh} (100%) rename .docker/stack/base-with-services/{before-notebook.d/30_start-rabbitmq-arm64.sh => init.d/start-rabbitmq-arm64.sh} (100%) rename .docker/stack/base-with-services/{before-notebook.d/41_suppress-rabbitmq-version-warning.sh => init.d/suppress-rabbitmq-version-warning.sh} (100%) diff --git a/.docker/stack/base-with-services/Dockerfile b/.docker/stack/base-with-services/Dockerfile index 3e2d30ab9c..cbc08e932e 100644 --- a/.docker/stack/base-with-services/Dockerfile +++ b/.docker/stack/base-with-services/Dockerfile @@ -10,21 +10,19 @@ ARG AIIDA_VERSION ARG PGSQL_VERSION ARG TARGETARCH -RUN mamba create -p /opt/conda/envs/aiida-core-services --yes \ +RUN micromamba create -p /opt/conda/envs/aiida-core-services --yes \ + --channel conda-forge \ postgresql=${PGSQL_VERSION} \ - && mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" + && micromamba clean --all -f -y # Install RabbitMQ in a dedicated conda environment. # If the architecture is arm64, we install the default version of rabbitmq provided by the generic binary, # # https://www.rabbitmq.com/install-generic-unix.html the version needs to be compatible with system's erlang version. RUN if [ "$TARGETARCH" = "amd64" ]; then \ - mamba install -p /opt/conda/envs/aiida-core-services --yes \ + micromamba install -p /opt/conda/envs/aiida-core-services --yes \ + --channel conda-forge \ rabbitmq-server=3.8.14 && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}"; \ + micromamba clean --all -f -y && \ elif [ "$TARGETARCH" = "arm64" ]; then \ # Install erlang. apt-get update --yes && \ @@ -44,15 +42,15 @@ fi # Configure AiiDA profile. COPY config-quick-setup.yaml . -COPY before-notebook.d/20_start-postgresql.sh /usr/local/bin/before-notebook.d/ -COPY before-notebook.d/30_start-rabbitmq-${TARGETARCH}.sh /usr/local/bin/before-notebook.d/ +COPY init.d/start-postgresql.sh /usr/local/bin/init.d/ +COPY init.d/start-rabbitmq-${TARGETARCH}.sh /usr/local/bin/init.d/ # Supress rabbitmq version warning for arm64 since # it is built using latest version rabbitmq from apt install. # We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf -COPY before-notebook.d/41_suppress-rabbitmq-version-warning.sh /usr/local/bin/before-notebook.d/ +COPY init.d/suppress-rabbitmq-version-warning.sh /usr/local/bin/init.d/ RUN if [ "$TARGETARCH" = "amd64" ]; then \ - rm /usr/local/bin/before-notebook.d/41_suppress-rabbitmq-version-warning.sh; \ + rm /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh; \ fi USER ${NB_USER} @@ -60,4 +58,4 @@ USER ${NB_USER} WORKDIR "/home/${NB_USER}" # Initialize the database -RUN mamba run -n aiida-core-services initdb -D aiida_db -U aiida +RUN micromamba run -n aiida-core-services initdb -D aiida_db -U aiida diff --git a/.docker/stack/base-with-services/init.d/_init.sh b/.docker/stack/base-with-services/init.d/_init.sh new file mode 100644 index 0000000000..78ba5d1783 --- /dev/null +++ b/.docker/stack/base-with-services/init.d/_init.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# This is the entrypoint script for the docker container. + +# Start the ssh-agent +/usr/local/bin/_entrypoint.sh ssh-agent + +# Start postgresql +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start_postgresql.sh + +# Start rabbitmq +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start_rabbitmq.sh + +# setup aiida must be run in the environment +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/aiida_profile_preparation.sh + +# supress the warning of incorrect rabbitmq version +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh + +exec "$@" diff --git a/.docker/stack/base-with-services/before-notebook.d/20_start-postgresql.sh b/.docker/stack/base-with-services/init.d/start-postgresql.sh similarity index 100% rename from .docker/stack/base-with-services/before-notebook.d/20_start-postgresql.sh rename to .docker/stack/base-with-services/init.d/start-postgresql.sh diff --git a/.docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-amd64.sh b/.docker/stack/base-with-services/init.d/start-rabbitmq-amd64.sh similarity index 100% rename from .docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-amd64.sh rename to .docker/stack/base-with-services/init.d/start-rabbitmq-amd64.sh diff --git a/.docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-arm64.sh b/.docker/stack/base-with-services/init.d/start-rabbitmq-arm64.sh similarity index 100% rename from .docker/stack/base-with-services/before-notebook.d/30_start-rabbitmq-arm64.sh rename to .docker/stack/base-with-services/init.d/start-rabbitmq-arm64.sh diff --git a/.docker/stack/base-with-services/before-notebook.d/41_suppress-rabbitmq-version-warning.sh b/.docker/stack/base-with-services/init.d/suppress-rabbitmq-version-warning.sh similarity index 100% rename from .docker/stack/base-with-services/before-notebook.d/41_suppress-rabbitmq-version-warning.sh rename to .docker/stack/base-with-services/init.d/suppress-rabbitmq-version-warning.sh diff --git a/.docker/stack/base/Dockerfile b/.docker/stack/base/Dockerfile index 605504a0ed..8bf04121d3 100644 --- a/.docker/stack/base/Dockerfile +++ b/.docker/stack/base/Dockerfile @@ -53,7 +53,7 @@ RUN echo "aiida-core==${AIIDA_VERSION}" >> /opt/requirements.txt ENV PIP_CONSTRAINT=/opt/requirements.txt # copy the entrypoint script -COPY --chmod=755 init.d/* /usr/local/bin +COPY --chmod=755 init.d/* /usr/local/bin/init.d # Enable verdi autocompletion. RUN mkdir -p "/opt/conda/etc/conda/activate.d" && \ diff --git a/.docker/stack/base/init.d/_init.sh b/.docker/stack/base/init.d/_init.sh index 5a60386273..9d42d982d3 100644 --- a/.docker/stack/base/init.d/_init.sh +++ b/.docker/stack/base/init.d/_init.sh @@ -6,6 +6,6 @@ /usr/local/bin/_entrypoint.sh ssh-agent # setup aiida must be run in the environment -/usr/local/bin/_entrypoint.sh /usr/local/bin/aiida_profile_preparation.sh +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/aiida_profile_preparation.sh exec "$@" From 19134fee7c1d0685e8053b27195440128c117e29 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Fri, 28 Apr 2023 00:39:11 +0200 Subject: [PATCH 05/19] with-service stack more - need to handle conda env permission. --- .docker/stack/base-with-services/Dockerfile | 22 ++++++----- .../base-with-services/{init.d => }/_init.sh | 6 +-- .../init.d/start-postgres.sh | 37 +++++++++++++++++++ .../init.d/start-postgresql.sh | 36 ------------------ .../init.d/start-rabbitmq-amd64.sh | 8 ++-- .../init.d/start-rabbitmq-arm64.sh | 8 ++-- .docker/stack/base/Dockerfile | 4 +- .docker/stack/base/{init.d => }/_init.sh | 2 +- ...ration.sh => aiida-profile-preparation.sh} | 2 +- 9 files changed, 66 insertions(+), 59 deletions(-) rename .docker/stack/base-with-services/{init.d => }/_init.sh (63%) create mode 100644 .docker/stack/base-with-services/init.d/start-postgres.sh delete mode 100644 .docker/stack/base-with-services/init.d/start-postgresql.sh rename .docker/stack/base/{init.d => }/_init.sh (70%) rename .docker/stack/base/init.d/{aiida_profile_preparation.sh => aiida-profile-preparation.sh} (97%) diff --git a/.docker/stack/base-with-services/Dockerfile b/.docker/stack/base-with-services/Dockerfile index cbc08e932e..9927414049 100644 --- a/.docker/stack/base-with-services/Dockerfile +++ b/.docker/stack/base-with-services/Dockerfile @@ -22,19 +22,20 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ micromamba install -p /opt/conda/envs/aiida-core-services --yes \ --channel conda-forge \ rabbitmq-server=3.8.14 && \ - micromamba clean --all -f -y && \ + micromamba clean --all -f -y; \ elif [ "$TARGETARCH" = "arm64" ]; then \ # Install erlang. apt-get update --yes && \ - apt-get install --yes --no-install-recommends erlang && \ + apt-get install --yes --no-install-recommends \ + erlang \ + xz-utils && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ # Install rabbitmq. export RMQ_VERSION=3.9.13 && \ - wget -c https://github.com/rabbitmq/rabbitmq-server/releases/download/v${RMQ_VERSION}/rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ + wget -c --no-check-certificate https://github.com/rabbitmq/rabbitmq-server/releases/download/v${RMQ_VERSION}/rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ tar -xf rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ rm rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ mv rabbitmq_server-${RMQ_VERSION} /opt/conda/envs/aiida-core-services/ && \ - fix-permissions "/opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}" && \ ln -sf /opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/sbin/* /opt/conda/envs/aiida-core-services/bin/; \ else \ echo "Unknown architecture: ${TARGETARCH}."; \ @@ -42,20 +43,23 @@ fi # Configure AiiDA profile. COPY config-quick-setup.yaml . -COPY init.d/start-postgresql.sh /usr/local/bin/init.d/ -COPY init.d/start-rabbitmq-${TARGETARCH}.sh /usr/local/bin/init.d/ +COPY --chmod=755 init.d/start-postgres.sh /usr/local/bin/init.d/ +COPY --chmod=755 init.d/start-rabbitmq-${TARGETARCH}.sh /usr/local/bin/init.d/start-rabbitmq.sh +COPY --chmod=755 _init.sh /usr/local/bin/_init.sh # Supress rabbitmq version warning for arm64 since # it is built using latest version rabbitmq from apt install. # We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf -COPY init.d/suppress-rabbitmq-version-warning.sh /usr/local/bin/init.d/ RUN if [ "$TARGETARCH" = "amd64" ]; then \ rm /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh; \ fi -USER ${NB_USER} +USER $MAMBA_USER -WORKDIR "/home/${NB_USER}" +WORKDIR "/home/${MAMBA_USER}" # Initialize the database RUN micromamba run -n aiida-core-services initdb -D aiida_db -U aiida + +ENTRYPOINT [ "/usr/local/bin/_init.sh" ] +# ENTRYPOINT ["_entrypoint.sh"] diff --git a/.docker/stack/base-with-services/init.d/_init.sh b/.docker/stack/base-with-services/_init.sh similarity index 63% rename from .docker/stack/base-with-services/init.d/_init.sh rename to .docker/stack/base-with-services/_init.sh index 78ba5d1783..a7f78fd5a2 100644 --- a/.docker/stack/base-with-services/init.d/_init.sh +++ b/.docker/stack/base-with-services/_init.sh @@ -6,13 +6,13 @@ /usr/local/bin/_entrypoint.sh ssh-agent # Start postgresql -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start_postgresql.sh +# /usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start-postgres.sh # Start rabbitmq -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start_rabbitmq.sh +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start-rabbitmq.sh # setup aiida must be run in the environment -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/aiida_profile_preparation.sh +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/aiida-profile-preparation.sh # supress the warning of incorrect rabbitmq version /usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh diff --git a/.docker/stack/base-with-services/init.d/start-postgres.sh b/.docker/stack/base-with-services/init.d/start-postgres.sh new file mode 100644 index 0000000000..8cb18dbbaa --- /dev/null +++ b/.docker/stack/base-with-services/init.d/start-postgres.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Activate the conda environment with PostgreSQL installed in it. +# conda activate pgsql + +# -w waits until server is up +SYSTEM_USER=${MAMBA_USER} +PSQL_START_CMD="pg_ctl --timeout=180 -w -D /home/${SYSTEM_USER}/.postgresql -l /home/${SYSTEM_USER}/.postgresql/logfile start" +PSQL_STOP_CMD="pg_ctl -w -D /home/${SYSTEM_USER}/.postgresql stop" +PSQL_STATUS_CMD="pg_ctl -D /home/${SYSTEM_USER}/.postgresql status" + +MAMBA_RUN="micromamba run -n aiida-core-services" + +# make DB directory, if not existent +if [ ! -d /home/${SYSTEM_USER}/.postgresql ]; then + mkdir /home/${SYSTEM_USER}/.postgresql + ${MAMBA_RUN} initdb -D /home/${SYSTEM_USER}/.postgresql + echo "unix_socket_directories = '/tmp'" >> /home/${SYSTEM_USER}/.postgresql/postgresql.conf + ${MAMBA_RUN} ${PSQL_START_CMD} + +# else don't +else + # Fix problem with kubernetes cluster that adds rws permissions to the group + # for more details see: https://github.com/materialscloud-org/aiidalab-z2jh-eosc/issues/5 + chmod g-rwxs /home/${NBSYSTEM_USER_USER}/.postgresql -R + + # stores return value in $? + running=true + ${MAMBA_RUN} ${PSQL_STATUS_CMD} || running=false + + # Postgresql was probably not shutdown properly. Cleaning up the mess... + if ! $running ; then + echo "" > /home/${SYSTEM_USER}/.postgresql/logfile # empty log files + rm -vf /home/${SYSTEM_USER}/.postgresql/postmaster.pid + ${MAMBA_RUN} ${PSQL_START_CMD} + fi +fi diff --git a/.docker/stack/base-with-services/init.d/start-postgresql.sh b/.docker/stack/base-with-services/init.d/start-postgresql.sh deleted file mode 100644 index 0eb685e7d8..0000000000 --- a/.docker/stack/base-with-services/init.d/start-postgresql.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Activate the conda environment with PostgreSQL installed in it. -# conda activate pgsql - -# -w waits until server is up -PSQL_START_CMD="pg_ctl --timeout=180 -w -D /home/${NB_USER}/.postgresql -l /home/${NB_USER}/.postgresql/logfile start" -PSQL_STOP_CMD="pg_ctl -w -D /home/${NB_USER}/.postgresql stop" -PSQL_STATUS_CMD="pg_ctl -D /home/${NB_USER}/.postgresql status" - -MAMBA_RUN="mamba run -n aiida-core-services" - -# make DB directory, if not existent -if [ ! -d /home/${NB_USER}/.postgresql ]; then - mkdir /home/${NB_USER}/.postgresql - ${MAMBA_RUN} initdb -D /home/${NB_USER}/.postgresql - echo "unix_socket_directories = '/tmp'" >> /home/${NB_USER}/.postgresql/postgresql.conf - ${MAMBA_RUN} ${PSQL_START_CMD} - -# else don't -else - # Fix problem with kubernetes cluster that adds rws permissions to the group - # for more details see: https://github.com/materialscloud-org/aiidalab-z2jh-eosc/issues/5 - chmod g-rwxs /home/${NB_USER}/.postgresql -R - - # stores return value in $? - running=true - ${MAMBA_RUN} ${PSQL_STATUS_CMD} || running=false - - # Postgresql was probably not shutdown properly. Cleaning up the mess... - if ! $running ; then - echo "" > /home/${NB_USER}/.postgresql/logfile # empty log files - rm -vf /home/${NB_USER}/.postgresql/postmaster.pid - ${MAMBA_RUN} ${PSQL_START_CMD} - fi -fi diff --git a/.docker/stack/base-with-services/init.d/start-rabbitmq-amd64.sh b/.docker/stack/base-with-services/init.d/start-rabbitmq-amd64.sh index 5c55fff3fc..5b48aa0b9e 100644 --- a/.docker/stack/base-with-services/init.d/start-rabbitmq-amd64.sh +++ b/.docker/stack/base-with-services/init.d/start-rabbitmq-amd64.sh @@ -1,13 +1,13 @@ #!/bin/bash set -em -RABBITMQ_DATA_DIR="/home/${NB_USER}/.rabbitmq" +SYSTEM_USER=${MAMBA_USER} +RABBITMQ_DATA_DIR="/home/${SYSTEM_USER}/.rabbitmq" mkdir -p "${RABBITMQ_DATA_DIR}" -fix-permissions "${RABBITMQ_DATA_DIR}" # Fix issue where the erlang cookie permissions are corrupted. -chmod 400 "/home/${NB_USER}/.erlang.cookie" || echo "erlang cookie not created yet." +chmod 400 "/home/${SYSTEM_USER}/.erlang.cookie" || echo "erlang cookie not created yet." # Set base directory for RabbitMQ to persist its data. This needs to be set to a folder in the system user's home # directory as that is the only folder that is persisted outside of the container. @@ -22,4 +22,4 @@ echo LOG_BASE="${RABBITMQ_DATA_DIR}/log" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" # isolated mnesia directories. Since in the AiiDA setup we only need and run a single node, we can simply use localhost. echo NODENAME=rabbit@localhost >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" -mamba run -n aiida-core-services rabbitmq-server -detached +micromamba run -n aiida-core-services rabbitmq-server -detached diff --git a/.docker/stack/base-with-services/init.d/start-rabbitmq-arm64.sh b/.docker/stack/base-with-services/init.d/start-rabbitmq-arm64.sh index c19763a220..11d71a2b6f 100644 --- a/.docker/stack/base-with-services/init.d/start-rabbitmq-arm64.sh +++ b/.docker/stack/base-with-services/init.d/start-rabbitmq-arm64.sh @@ -1,13 +1,13 @@ #!/bin/bash set -em -RABBITMQ_DATA_DIR="/home/${NB_USER}/.rabbitmq" +SYSTEM_USER=${MAMBA_USER} +RABBITMQ_DATA_DIR="/home/${SYSTEM_USER}/.rabbitmq" mkdir -p "${RABBITMQ_DATA_DIR}" -fix-permissions "${RABBITMQ_DATA_DIR}" # Fix issue where the erlang cookie permissions are corrupted. -chmod 400 "/home/${NB_USER}/.erlang.cookie" || echo "erlang cookie not created yet." +chmod 400 "/home/${SYSTEM_USER}/.erlang.cookie" || echo "erlang cookie not created yet." # Set base directory for RabbitMQ to persist its data. This needs to be set to a folder in the system user's home # directory as that is the only folder that is persisted outside of the container. @@ -29,4 +29,4 @@ echo "consumer_timeout=3600000" >> "${RMQ_ETC_DIR}/rabbitmq.conf" # isolated mnesia directories. Since in the AiiDA setup we only need and run a single node, we can simply use localhost. echo NODENAME=rabbit@localhost >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" -mamba run -n aiida-core-services rabbitmq-server -detached +micromamba run -n aiida-core-services rabbitmq-server -detached diff --git a/.docker/stack/base/Dockerfile b/.docker/stack/base/Dockerfile index 8bf04121d3..108b992eb7 100644 --- a/.docker/stack/base/Dockerfile +++ b/.docker/stack/base/Dockerfile @@ -53,7 +53,9 @@ RUN echo "aiida-core==${AIIDA_VERSION}" >> /opt/requirements.txt ENV PIP_CONSTRAINT=/opt/requirements.txt # copy the entrypoint script -COPY --chmod=755 init.d/* /usr/local/bin/init.d +RUN mkdir -p /usr/local/bin/init.d +COPY --chmod=755 init.d/* /usr/local/bin/init.d/ +COPY --chmod=755 _init.sh /usr/local/bin/_init.sh # Enable verdi autocompletion. RUN mkdir -p "/opt/conda/etc/conda/activate.d" && \ diff --git a/.docker/stack/base/init.d/_init.sh b/.docker/stack/base/_init.sh similarity index 70% rename from .docker/stack/base/init.d/_init.sh rename to .docker/stack/base/_init.sh index 9d42d982d3..d29c3202a7 100644 --- a/.docker/stack/base/init.d/_init.sh +++ b/.docker/stack/base/_init.sh @@ -6,6 +6,6 @@ /usr/local/bin/_entrypoint.sh ssh-agent # setup aiida must be run in the environment -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/aiida_profile_preparation.sh +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/aiida-profile-preparation.sh exec "$@" diff --git a/.docker/stack/base/init.d/aiida_profile_preparation.sh b/.docker/stack/base/init.d/aiida-profile-preparation.sh similarity index 97% rename from .docker/stack/base/init.d/aiida_profile_preparation.sh rename to .docker/stack/base/init.d/aiida-profile-preparation.sh index 869aa3d591..05dde219a0 100755 --- a/.docker/stack/base/init.d/aiida_profile_preparation.sh +++ b/.docker/stack/base/init.d/aiida-profile-preparation.sh @@ -55,7 +55,7 @@ if [[ ${NEED_SETUP_PROFILE} == true ]]; then --hostname "${computer_name}" \ --transport core.local \ --scheduler core.direct \ - --work-dir /home/${NB_USER}/aiida_run/ \ + --work-dir /home/${MAMBA_USER}/aiida_run/ \ --mpirun-command "mpirun -np {tot_num_mpiprocs}" \ --mpiprocs-per-machine ${LOCALHOST_MPI_PROCS_PER_MACHINE} && \ verdi computer configure core.local "${computer_name}" \ From e09ae4b68a51c13383ec674e68e4d5b97edf4e2f Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Fri, 28 Apr 2023 23:31:28 +0200 Subject: [PATCH 06/19] WIP: stuck at init.sh --- .docker/stack/base-with-services/Dockerfile | 10 +++++++--- .docker/stack/base-with-services/_init.sh | 6 +++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.docker/stack/base-with-services/Dockerfile b/.docker/stack/base-with-services/Dockerfile index 9927414049..cc0b2057e3 100644 --- a/.docker/stack/base-with-services/Dockerfile +++ b/.docker/stack/base-with-services/Dockerfile @@ -15,6 +15,12 @@ RUN micromamba create -p /opt/conda/envs/aiida-core-services --yes \ postgresql=${PGSQL_VERSION} \ && micromamba clean --all -f -y +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + locales && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + locale-gen en_US.UTF-8 + # Install RabbitMQ in a dedicated conda environment. # If the architecture is arm64, we install the default version of rabbitmq provided by the generic binary, # # https://www.rabbitmq.com/install-generic-unix.html the version needs to be compatible with system's erlang version. @@ -50,6 +56,7 @@ COPY --chmod=755 _init.sh /usr/local/bin/_init.sh # Supress rabbitmq version warning for arm64 since # it is built using latest version rabbitmq from apt install. # We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf +COPY --chmod=755 init.d/suppress-rabbitmq-version-warning.sh /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh RUN if [ "$TARGETARCH" = "amd64" ]; then \ rm /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh; \ fi @@ -58,8 +65,5 @@ USER $MAMBA_USER WORKDIR "/home/${MAMBA_USER}" -# Initialize the database -RUN micromamba run -n aiida-core-services initdb -D aiida_db -U aiida - ENTRYPOINT [ "/usr/local/bin/_init.sh" ] # ENTRYPOINT ["_entrypoint.sh"] diff --git a/.docker/stack/base-with-services/_init.sh b/.docker/stack/base-with-services/_init.sh index a7f78fd5a2..f0fd0ffead 100644 --- a/.docker/stack/base-with-services/_init.sh +++ b/.docker/stack/base-with-services/_init.sh @@ -6,10 +6,10 @@ /usr/local/bin/_entrypoint.sh ssh-agent # Start postgresql -# /usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start-postgres.sh +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start-postgres.sh -# Start rabbitmq -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start-rabbitmq.sh +# Start rabbitmq, needs to bu run by root +/usr/local/bin/init.d/start-rabbitmq.sh # setup aiida must be run in the environment /usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/aiida-profile-preparation.sh From 1f0c5d937c1f9e9ebd3db2c42d892449a9f5defc Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Tue, 4 Jul 2023 00:49:59 +0200 Subject: [PATCH 07/19] adapt dodo script to build for correct arch --- .docker/README.md | 11 ++ .../{stack => }/base-with-services/Dockerfile | 0 .../{stack => }/base-with-services/_init.sh | 0 .../config-quick-setup.yaml | 0 .../init.d/start-postgres.sh | 0 .../init.d/start-rabbitmq-amd64.sh | 0 .../init.d/start-rabbitmq-arm64.sh | 0 .../suppress-rabbitmq-version-warning.sh | 0 .docker/{stack => }/base/Dockerfile | 0 .docker/{stack => }/base/_init.sh | 0 .../{stack => }/base/config-quick-setup.yaml | 0 .../base/init.d/aiida-profile-preparation.sh | 0 .docker/docker-bake.hcl | 12 +- .docker/dodo.py | 186 ++++++++++-------- .docker/requirements.txt | 10 + 15 files changed, 129 insertions(+), 90 deletions(-) create mode 100644 .docker/README.md rename .docker/{stack => }/base-with-services/Dockerfile (100%) rename .docker/{stack => }/base-with-services/_init.sh (100%) rename .docker/{stack => }/base-with-services/config-quick-setup.yaml (100%) rename .docker/{stack => }/base-with-services/init.d/start-postgres.sh (100%) rename .docker/{stack => }/base-with-services/init.d/start-rabbitmq-amd64.sh (100%) rename .docker/{stack => }/base-with-services/init.d/start-rabbitmq-arm64.sh (100%) rename .docker/{stack => }/base-with-services/init.d/suppress-rabbitmq-version-warning.sh (100%) rename .docker/{stack => }/base/Dockerfile (100%) rename .docker/{stack => }/base/_init.sh (100%) rename .docker/{stack => }/base/config-quick-setup.yaml (100%) rename .docker/{stack => }/base/init.d/aiida-profile-preparation.sh (100%) create mode 100644 .docker/requirements.txt diff --git a/.docker/README.md b/.docker/README.md new file mode 100644 index 0000000000..4df6dd7413 --- /dev/null +++ b/.docker/README.md @@ -0,0 +1,11 @@ +# AiiDA docker stacks + +### Build images locally + +To build the images, run `doit build` (tested with *docker buildx* version v0.8.2). + +The build system will attempt to detect the local architecture and automatically build images for it (tested with amd64 and arm64). +All commands `build`, `tests`, and `up` will use the locally detected platform and use a version tag based on the state of the local git repository. +However, you can also specify a custom platform or version with the `--platform` and `--version` parameters, example: `doit build --arch=amd64 --version=my-version`. + +You can specify target stacks to build with `--target`, example: `doit build --target base --target base`. \ No newline at end of file diff --git a/.docker/stack/base-with-services/Dockerfile b/.docker/base-with-services/Dockerfile similarity index 100% rename from .docker/stack/base-with-services/Dockerfile rename to .docker/base-with-services/Dockerfile diff --git a/.docker/stack/base-with-services/_init.sh b/.docker/base-with-services/_init.sh similarity index 100% rename from .docker/stack/base-with-services/_init.sh rename to .docker/base-with-services/_init.sh diff --git a/.docker/stack/base-with-services/config-quick-setup.yaml b/.docker/base-with-services/config-quick-setup.yaml similarity index 100% rename from .docker/stack/base-with-services/config-quick-setup.yaml rename to .docker/base-with-services/config-quick-setup.yaml diff --git a/.docker/stack/base-with-services/init.d/start-postgres.sh b/.docker/base-with-services/init.d/start-postgres.sh similarity index 100% rename from .docker/stack/base-with-services/init.d/start-postgres.sh rename to .docker/base-with-services/init.d/start-postgres.sh diff --git a/.docker/stack/base-with-services/init.d/start-rabbitmq-amd64.sh b/.docker/base-with-services/init.d/start-rabbitmq-amd64.sh similarity index 100% rename from .docker/stack/base-with-services/init.d/start-rabbitmq-amd64.sh rename to .docker/base-with-services/init.d/start-rabbitmq-amd64.sh diff --git a/.docker/stack/base-with-services/init.d/start-rabbitmq-arm64.sh b/.docker/base-with-services/init.d/start-rabbitmq-arm64.sh similarity index 100% rename from .docker/stack/base-with-services/init.d/start-rabbitmq-arm64.sh rename to .docker/base-with-services/init.d/start-rabbitmq-arm64.sh diff --git a/.docker/stack/base-with-services/init.d/suppress-rabbitmq-version-warning.sh b/.docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh similarity index 100% rename from .docker/stack/base-with-services/init.d/suppress-rabbitmq-version-warning.sh rename to .docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh diff --git a/.docker/stack/base/Dockerfile b/.docker/base/Dockerfile similarity index 100% rename from .docker/stack/base/Dockerfile rename to .docker/base/Dockerfile diff --git a/.docker/stack/base/_init.sh b/.docker/base/_init.sh similarity index 100% rename from .docker/stack/base/_init.sh rename to .docker/base/_init.sh diff --git a/.docker/stack/base/config-quick-setup.yaml b/.docker/base/config-quick-setup.yaml similarity index 100% rename from .docker/stack/base/config-quick-setup.yaml rename to .docker/base/config-quick-setup.yaml diff --git a/.docker/stack/base/init.d/aiida-profile-preparation.sh b/.docker/base/init.d/aiida-profile-preparation.sh similarity index 100% rename from .docker/stack/base/init.d/aiida-profile-preparation.sh rename to .docker/base/init.d/aiida-profile-preparation.sh diff --git a/.docker/docker-bake.hcl b/.docker/docker-bake.hcl index 08accbb33c..60d0de6175 100644 --- a/.docker/docker-bake.hcl +++ b/.docker/docker-bake.hcl @@ -11,10 +11,6 @@ variable "PGSQL_VERSION" { variable "AIIDA_VERSION" { } -variable "PIP_VERSION" { - default = "22.0.4" -} - variable "BASE_IMAGE" { default = "mambaorg/micromamba:jammy" } @@ -41,6 +37,7 @@ function "tags" { "${REGISTRY}${ORGANIZATION}/${image}:${VERSION}", "${REGISTRY}${ORGANIZATION}/${image}:python-${PYTHON_VERSION}", "${REGISTRY}${ORGANIZATION}/${image}:postgresql-${PGSQL_VERSION}", + "${REGISTRY}${ORGANIZATION}/${image}:aiida-${AIIDA_VERSION}", ] } @@ -52,23 +49,22 @@ target "base-meta" { tags = tags("base") } target "base-with-services-meta" { - tags = tags("base-with-services") + tags = tags("aiida-core") } target "base" { inherits = ["base-meta"] - context = "stack/base" + context = "base" platforms = "${PLATFORMS}" args = { "BASE" = "${BASE_IMAGE}" "PYTHON_VERSION" = "${PYTHON_VERSION}" - "PIP_VERSION" = "${PIP_VERSION}" "AIIDA_VERSION" = "${AIIDA_VERSION}" } } target "base-with-services" { inherits = ["base-with-services-meta"] - context = "stack/base-with-services" + context = "base-with-services" contexts = { base = "target:base" } diff --git a/.docker/dodo.py b/.docker/dodo.py index 3056970536..19eedda420 100644 --- a/.docker/dodo.py +++ b/.docker/dodo.py @@ -1,105 +1,117 @@ -# -*- coding: utf-8 -*- -"""Doit tasks for building and testing the docker images. -The idea is borrowed from AiiDAlab docker stack. -""" import json -from pathlib import Path import platform +from pathlib import Path -import docker # pylint: disable=import-error -from doit.tools import title_with_actions # pylint: disable=import-error -from dunamai import Version # pylint: disable=import-error +from doit.tools import title_with_actions + +import docker +from dunamai import Version _DOCKER_CLIENT = docker.from_env() -_DOCKER_ARCHITECTURE = _DOCKER_CLIENT.info()['Architecture'] +_DOCKER_ARCHITECTURE = _DOCKER_CLIENT.info()["Architecture"] + +DOIT_CONFIG = {"default_tasks": ["build"]} -DOIT_CONFIG = {'default_tasks': ['build']} +VERSION = Version.from_git().serialize().replace("+", "_") -_ARCH_PLATFORM_MAPPING = { - 'aarch64': 'linux/arm64', - 'x86_64': 'linux/amd64', +_ARCH_MAPPING = { + "x86_64": "amd64", + "aarch64": "arm64", } -VERSION = Version.from_git().serialize().replace('+', '_') -PLATFORM = _ARCH_PLATFORM_MAPPING.get(_DOCKER_ARCHITECTURE) +ARCH = _ARCH_MAPPING.get(_DOCKER_ARCHITECTURE) -if PLATFORM is None: - raise RuntimeError(f'Unsupported architecture {_DOCKER_ARCHITECTURE} on platform {platform.system()}.') +if ARCH is None: + raise RuntimeError( + f"Unsupported architecture {ARCH} on platform {platform.system()}." + ) _REGISTRY_PARAM = { - 'name': 'registry', - 'short': 'r', - 'long': 'registry', - 'type': str, - 'default': '', - 'help': 'Specify the docker image registry.', + "name": "registry", + "short": "r", + "long": "registry", + "type": str, + "default": "", + "help": "Specify the docker image registry.", } -_VERSION_PARAM = { - 'name': 'version', - 'long': 'version', - 'type': 'str', - 'default': VERSION, - 'help': ('Specify the AiiDA version to build for.',), +_ORGANIZATION_PARAM = { + "name": "organization", + "short": "o", + "long": "organization", + "type": str, + "default": "aiidalab", + "help": "Specify the docker image organization.", } -_AIIDA_VERSION_PARAM = { - 'name': 'aiida_version', - 'long': 'aiida-version', - 'type': 'str', - 'default': '', - 'help': ( - 'Specify the version of the stack for building / testing. Defaults to a ' - 'version determined from the state of the local git repository.' +_VERSION_PARAM = { + "name": "version", + "long": "version", + "type": "str", + "default": VERSION, + "help": ( + "Specify the version of the stack for building / testing. Defaults to a " + "version determined from the state of the local git repository." ), } -_PLATFORM_PARAM = { - 'name': 'platform', - 'long': 'platform', - 'type': str, - 'default': PLATFORM, - 'help': 'Specify the platform to build for. Examples: linux/amd64, linux/arm64', +_ARCH_PARAM = { + "name": "architecture", + "long": "arch", + "type": str, + "default": ARCH, + "help": "Specify the platform to build for. Examples: arm64, amd64.", } def task_build(): """Build all docker images.""" - def generate_version_override(version, registry, targets): + def generate_version_override( + version, registry, targets, architecture, organization + ): if len(targets) > 2: # Workaround of issue of doit, which rather than override the default value, it appends # https://github.com/pydoit/doit/issues/436 targets = targets[2:] - Path('docker-bake.override.json').write_text( - json.dumps(dict(VERSION=version, REGISTRY=registry, TARGETS=targets)), - encoding='utf-8', + + platforms = [f"linux/{architecture}"] + + Path("docker-bake.override.json").write_text( + json.dumps( + dict( + VERSION=version, + REGISTRY=registry, + TARGETS=targets, + ORGANIZATION=organization, + PLATFORMS=platforms, + ) + ) ) return { - 'actions': [ + "actions": [ generate_version_override, - 'docker buildx bake -f docker-bake.hcl -f build.json ' - '-f docker-bake.override.json ' - "--set '*.platform=%(platform)s' " - '--load', + "docker buildx bake -f docker-bake.hcl -f build.json " + "-f docker-bake.override.json " + "--load", ], - 'title': title_with_actions, - 'params': [ + "title": title_with_actions, + "params": [ + _ORGANIZATION_PARAM, _REGISTRY_PARAM, _VERSION_PARAM, - _PLATFORM_PARAM, - _AIIDA_VERSION_PARAM, + _ARCH_PARAM, { - 'name': 'targets', - 'long': 'targets', - 'short': 't', - 'type': list, - 'default': ['base', 'base-with-services'], - 'help': 'Specify the target to build.', + "name": "targets", + "long": "targets", + "short": "t", + "type": list, + "default": ["base", "base-with-services"], + "help": "Specify the target to build.", }, ], - 'verbosity': 2, + "verbosity": 2, } @@ -107,25 +119,35 @@ def task_tests(): """Run tests with pytest.""" return { - 'actions': ['REGISTRY=%(registry)s VERSION=:%(version)s pytest -v'], - 'params': [_REGISTRY_PARAM, _VERSION_PARAM], - 'verbosity': 2, - } - - -def task_up(): - """Start AiiDAlab server for testing.""" - return { - 'actions': ['REGISTRY=%(registry)s VERSION=:%(version)s ' - 'docker-compose up --detach'], - 'params': [ - _REGISTRY_PARAM, - _VERSION_PARAM, - ], - 'verbosity': 2, + "actions": ["REGISTRY=%(registry)s VERSION=:%(version)s pytest -v"], + "params": [_REGISTRY_PARAM, _VERSION_PARAM], + "verbosity": 2, } -def task_down(): - """Stop AiiDAlab server.""" - return {'actions': ['docker-compose down'], 'verbosity': 2} +# def task_up(): +# """Start AiiDAlab server for testing.""" +# return { +# "actions": [ +# "AIIDALAB_PORT=%(port)i REGISTRY=%(registry)s VERSION=:%(version)s " +# "docker-compose up --detach" +# ], +# "params": [ +# { +# "name": "port", +# "short": "p", +# "long": "port", +# "type": int, +# "default": 8888, +# "help": "Specify the AiiDAlab host port.", +# }, +# _REGISTRY_PARAM, +# _VERSION_PARAM, +# ], +# "verbosity": 2, +# } + + +# def task_down(): +# """Stop AiiDAlab server.""" +# return {"actions": ["docker-compose down"], "verbosity": 2} \ No newline at end of file diff --git a/.docker/requirements.txt b/.docker/requirements.txt new file mode 100644 index 0000000000..6763b51984 --- /dev/null +++ b/.docker/requirements.txt @@ -0,0 +1,10 @@ +docker +pre-commit +pytest +pytest-xdist +requests +tabulate +pytest-docker +docker-compose +doit +dunamai \ No newline at end of file From 8ef60fbde31c1f87f7fc7033f49e8a7b9e118bb3 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Tue, 4 Jul 2023 08:26:14 +0200 Subject: [PATCH 08/19] Use mamba image back again --- .docker/base-with-services/Dockerfile | 69 -------------- .docker/base-with-services/_init.sh | 20 ---- .../config-quick-setup.yaml | 3 - .../init.d/start-postgres.sh | 37 -------- .../init.d/start-rabbitmq-amd64.sh | 25 ----- .../init.d/start-rabbitmq-arm64.sh | 32 ------- .../suppress-rabbitmq-version-warning.sh | 7 -- .docker/base/Dockerfile | 94 ------------------- .docker/base/_init.sh | 11 --- .docker/base/config-quick-setup.yaml | 15 --- .../base/init.d/aiida-profile-preparation.sh | 77 --------------- .docker/docker-bake.hcl | 21 +---- 12 files changed, 5 insertions(+), 406 deletions(-) delete mode 100644 .docker/base-with-services/Dockerfile delete mode 100644 .docker/base-with-services/_init.sh delete mode 100644 .docker/base-with-services/config-quick-setup.yaml delete mode 100644 .docker/base-with-services/init.d/start-postgres.sh delete mode 100644 .docker/base-with-services/init.d/start-rabbitmq-amd64.sh delete mode 100644 .docker/base-with-services/init.d/start-rabbitmq-arm64.sh delete mode 100644 .docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh delete mode 100644 .docker/base/Dockerfile delete mode 100644 .docker/base/_init.sh delete mode 100644 .docker/base/config-quick-setup.yaml delete mode 100755 .docker/base/init.d/aiida-profile-preparation.sh diff --git a/.docker/base-with-services/Dockerfile b/.docker/base-with-services/Dockerfile deleted file mode 100644 index cc0b2057e3..0000000000 --- a/.docker/base-with-services/Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -# syntax=docker/dockerfile:1 -FROM base - -LABEL maintainer="AiiDAlab Team " - -USER root -WORKDIR /opt/ - -ARG AIIDA_VERSION -ARG PGSQL_VERSION -ARG TARGETARCH - -RUN micromamba create -p /opt/conda/envs/aiida-core-services --yes \ - --channel conda-forge \ - postgresql=${PGSQL_VERSION} \ - && micromamba clean --all -f -y - -RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends \ - locales && \ - apt-get clean && rm -rf /var/lib/apt/lists/* && \ - locale-gen en_US.UTF-8 - -# Install RabbitMQ in a dedicated conda environment. -# If the architecture is arm64, we install the default version of rabbitmq provided by the generic binary, -# # https://www.rabbitmq.com/install-generic-unix.html the version needs to be compatible with system's erlang version. -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - micromamba install -p /opt/conda/envs/aiida-core-services --yes \ - --channel conda-forge \ - rabbitmq-server=3.8.14 && \ - micromamba clean --all -f -y; \ -elif [ "$TARGETARCH" = "arm64" ]; then \ - # Install erlang. - apt-get update --yes && \ - apt-get install --yes --no-install-recommends \ - erlang \ - xz-utils && \ - apt-get clean && rm -rf /var/lib/apt/lists/* && \ - # Install rabbitmq. - export RMQ_VERSION=3.9.13 && \ - wget -c --no-check-certificate https://github.com/rabbitmq/rabbitmq-server/releases/download/v${RMQ_VERSION}/rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ - tar -xf rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ - rm rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ - mv rabbitmq_server-${RMQ_VERSION} /opt/conda/envs/aiida-core-services/ && \ - ln -sf /opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/sbin/* /opt/conda/envs/aiida-core-services/bin/; \ -else \ - echo "Unknown architecture: ${TARGETARCH}."; \ -fi - -# Configure AiiDA profile. -COPY config-quick-setup.yaml . -COPY --chmod=755 init.d/start-postgres.sh /usr/local/bin/init.d/ -COPY --chmod=755 init.d/start-rabbitmq-${TARGETARCH}.sh /usr/local/bin/init.d/start-rabbitmq.sh -COPY --chmod=755 _init.sh /usr/local/bin/_init.sh - -# Supress rabbitmq version warning for arm64 since -# it is built using latest version rabbitmq from apt install. -# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf -COPY --chmod=755 init.d/suppress-rabbitmq-version-warning.sh /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - rm /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh; \ -fi - -USER $MAMBA_USER - -WORKDIR "/home/${MAMBA_USER}" - -ENTRYPOINT [ "/usr/local/bin/_init.sh" ] -# ENTRYPOINT ["_entrypoint.sh"] diff --git a/.docker/base-with-services/_init.sh b/.docker/base-with-services/_init.sh deleted file mode 100644 index f0fd0ffead..0000000000 --- a/.docker/base-with-services/_init.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# This is the entrypoint script for the docker container. - -# Start the ssh-agent -/usr/local/bin/_entrypoint.sh ssh-agent - -# Start postgresql -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start-postgres.sh - -# Start rabbitmq, needs to bu run by root -/usr/local/bin/init.d/start-rabbitmq.sh - -# setup aiida must be run in the environment -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/aiida-profile-preparation.sh - -# supress the warning of incorrect rabbitmq version -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh - -exec "$@" diff --git a/.docker/base-with-services/config-quick-setup.yaml b/.docker/base-with-services/config-quick-setup.yaml deleted file mode 100644 index 24c516270d..0000000000 --- a/.docker/base-with-services/config-quick-setup.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -db_name: aiida_db -db_username: aiida diff --git a/.docker/base-with-services/init.d/start-postgres.sh b/.docker/base-with-services/init.d/start-postgres.sh deleted file mode 100644 index 8cb18dbbaa..0000000000 --- a/.docker/base-with-services/init.d/start-postgres.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Activate the conda environment with PostgreSQL installed in it. -# conda activate pgsql - -# -w waits until server is up -SYSTEM_USER=${MAMBA_USER} -PSQL_START_CMD="pg_ctl --timeout=180 -w -D /home/${SYSTEM_USER}/.postgresql -l /home/${SYSTEM_USER}/.postgresql/logfile start" -PSQL_STOP_CMD="pg_ctl -w -D /home/${SYSTEM_USER}/.postgresql stop" -PSQL_STATUS_CMD="pg_ctl -D /home/${SYSTEM_USER}/.postgresql status" - -MAMBA_RUN="micromamba run -n aiida-core-services" - -# make DB directory, if not existent -if [ ! -d /home/${SYSTEM_USER}/.postgresql ]; then - mkdir /home/${SYSTEM_USER}/.postgresql - ${MAMBA_RUN} initdb -D /home/${SYSTEM_USER}/.postgresql - echo "unix_socket_directories = '/tmp'" >> /home/${SYSTEM_USER}/.postgresql/postgresql.conf - ${MAMBA_RUN} ${PSQL_START_CMD} - -# else don't -else - # Fix problem with kubernetes cluster that adds rws permissions to the group - # for more details see: https://github.com/materialscloud-org/aiidalab-z2jh-eosc/issues/5 - chmod g-rwxs /home/${NBSYSTEM_USER_USER}/.postgresql -R - - # stores return value in $? - running=true - ${MAMBA_RUN} ${PSQL_STATUS_CMD} || running=false - - # Postgresql was probably not shutdown properly. Cleaning up the mess... - if ! $running ; then - echo "" > /home/${SYSTEM_USER}/.postgresql/logfile # empty log files - rm -vf /home/${SYSTEM_USER}/.postgresql/postmaster.pid - ${MAMBA_RUN} ${PSQL_START_CMD} - fi -fi diff --git a/.docker/base-with-services/init.d/start-rabbitmq-amd64.sh b/.docker/base-with-services/init.d/start-rabbitmq-amd64.sh deleted file mode 100644 index 5b48aa0b9e..0000000000 --- a/.docker/base-with-services/init.d/start-rabbitmq-amd64.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -em - -SYSTEM_USER=${MAMBA_USER} -RABBITMQ_DATA_DIR="/home/${SYSTEM_USER}/.rabbitmq" - -mkdir -p "${RABBITMQ_DATA_DIR}" - -# Fix issue where the erlang cookie permissions are corrupted. -chmod 400 "/home/${SYSTEM_USER}/.erlang.cookie" || echo "erlang cookie not created yet." - -# Set base directory for RabbitMQ to persist its data. This needs to be set to a folder in the system user's home -# directory as that is the only folder that is persisted outside of the container. -RMQ_ETC_DIR="/opt/conda/envs/aiida-core-services/etc/rabbitmq" -echo MNESIA_BASE="${RABBITMQ_DATA_DIR}" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" -echo LOG_BASE="${RABBITMQ_DATA_DIR}/log" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" - -# Explicitly define the node name. This is necessary because the mnesia subdirectory contains the hostname, which by -# default is set to the value of $(hostname -s), which for docker containers, will be a random hexadecimal string. Upon -# restart, this will be different and so the original mnesia folder with the persisted data will not be found. The -# reason RabbitMQ is built this way is through this way it allows to run multiple nodes on a single machine each with -# isolated mnesia directories. Since in the AiiDA setup we only need and run a single node, we can simply use localhost. -echo NODENAME=rabbit@localhost >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" - -micromamba run -n aiida-core-services rabbitmq-server -detached diff --git a/.docker/base-with-services/init.d/start-rabbitmq-arm64.sh b/.docker/base-with-services/init.d/start-rabbitmq-arm64.sh deleted file mode 100644 index 11d71a2b6f..0000000000 --- a/.docker/base-with-services/init.d/start-rabbitmq-arm64.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -set -em - -SYSTEM_USER=${MAMBA_USER} -RABBITMQ_DATA_DIR="/home/${SYSTEM_USER}/.rabbitmq" - -mkdir -p "${RABBITMQ_DATA_DIR}" - -# Fix issue where the erlang cookie permissions are corrupted. -chmod 400 "/home/${SYSTEM_USER}/.erlang.cookie" || echo "erlang cookie not created yet." - -# Set base directory for RabbitMQ to persist its data. This needs to be set to a folder in the system user's home -# directory as that is the only folder that is persisted outside of the container. -export RMQ_VERSION=3.9.13 -RMQ_ETC_DIR="/opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/etc/rabbitmq" -echo MNESIA_BASE="${RABBITMQ_DATA_DIR}" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" -echo LOG_BASE="${RABBITMQ_DATA_DIR}/log" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" - -# RabbitMQ with versions >= 3.8.15 have reduced some default timeouts -# baseimage phusion/baseimage:jammy-1.0.0 running ubuntu 22.04 will install higher version of rabbimq by apt. -# using workaround from https://github.com/aiidateam/aiida-core/wiki/RabbitMQ-version-to-use -# set timeout to 100 hours -echo "consumer_timeout=3600000" >> "${RMQ_ETC_DIR}/rabbitmq.conf" - -# Explicitly define the node name. This is necessary because the mnesia subdirectory contains the hostname, which by -# default is set to the value of $(hostname -s), which for docker containers, will be a random hexadecimal string. Upon -# restart, this will be different and so the original mnesia folder with the persisted data will not be found. The -# reason RabbitMQ is built this way is through this way it allows to run multiple nodes on a single machine each with -# isolated mnesia directories. Since in the AiiDA setup we only need and run a single node, we can simply use localhost. -echo NODENAME=rabbit@localhost >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" - -micromamba run -n aiida-core-services rabbitmq-server -detached diff --git a/.docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh b/.docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh deleted file mode 100644 index f778bbb5db..0000000000 --- a/.docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -em - -# Supress rabbitmq version warning for arm64 since -# it is built using latest version rabbitmq from apt install. -# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf -verdi config set warnings.rabbitmq_version False diff --git a/.docker/base/Dockerfile b/.docker/base/Dockerfile deleted file mode 100644 index 108b992eb7..0000000000 --- a/.docker/base/Dockerfile +++ /dev/null @@ -1,94 +0,0 @@ -ARG BASE -FROM ${BASE} - -LABEL maintainer="AiiDAlab Team " - -ARG NEW_MAMBA_USER=aiida -ARG NEW_MAMBA_USER_ID=1000 -ARG NEW_MAMBA_USER_GID=1000 - -USER root -RUN usermod "--login=${NEW_MAMBA_USER}" "--home=/home/${NEW_MAMBA_USER}" \ - --move-home "-u ${NEW_MAMBA_USER_ID}" "${MAMBA_USER}" && \ - groupmod "--new-name=${NEW_MAMBA_USER}" \ - "-g ${NEW_MAMBA_USER_GID}" "${MAMBA_USER}" && \ - # Update the expected value of MAMBA_USER for the - # _entrypoint.sh consistency check. - echo "${NEW_MAMBA_USER}" > "/etc/arg_mamba_user" && \ - : - -ENV MAMBA_USER=$NEW_MAMBA_USER - -# Pin python and pip version -ARG PYTHON_VERSION -ARG PIP_VERSION - -# The docker only build after the release of conda-forge. -ARG AIIDA_VERSION - -RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends \ - wget && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Install the python base requirements to base environment. -# XXX: bring back mamba-bash-completion once it is available on conda-forge. -# https://github.com/conda-forge/mamba-bash-completion-feedstock/pull/2 -RUN micromamba install --yes \ - --channel conda-forge \ - -n base \ - openssh \ - python=${PYTHON_VERSION} \ - pip=${PIP_VERSION} \ - aiida-core=${AIIDA_VERSION} \ - && micromamba clean --all -f -y - -# Pin shared requirements in the base environemnt. -# XXX: ironup -RUN echo "pip==${PIP_VERSION}" >> /opt/requirements.txt -RUN echo "python==${PYTHON_VERSION}" >> /opt/requirements.txt -RUN echo "aiida-core==${AIIDA_VERSION}" >> /opt/requirements.txt - -# Configure pip to use requirements file as constraints file. -ENV PIP_CONSTRAINT=/opt/requirements.txt - -# copy the entrypoint script -RUN mkdir -p /usr/local/bin/init.d -COPY --chmod=755 init.d/* /usr/local/bin/init.d/ -COPY --chmod=755 _init.sh /usr/local/bin/_init.sh - -# Enable verdi autocompletion. -RUN mkdir -p "/opt/conda/etc/conda/activate.d" && \ - echo 'eval "$(_VERDI_COMPLETE=bash_source verdi)"' >> "/opt/conda/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ - chmod +x "/opt/conda/etc/conda/activate.d/activate_aiida_autocompletion.sh" - -# Install the load-singlesshagent.sh script as described here: -# https://aiida.readthedocs.io/projects/aiida-core/en/v2.0.0/howto/ssh.html#starting-the-ssh-agent -RUN wget --no-check-certificate --directory-prefix=/opt/conda/ \ - "https://aiida.readthedocs.io/projects/aiida-core/en/v2.0.0/_downloads/4265ec5a42c3a3dba586dd460c0db95e/load-singlesshagent.sh" - -# SSH agent configuration - -# Make sure that the known_hosts file is present inside the .ssh folder. -RUN mkdir -p --mode=0700 /home/${MAMBA_USER}/.ssh && \ - touch /home/${MAMBA_USER}/.ssh/known_hosts - -RUN /usr/local/bin/_entrypoint.sh ssh-keygen -f /home/${MAMBA_USER}/.ssh/id_rsa -t rsa -b 4096 -m PEM -N '' - -# Configure AiiDA profile. -COPY config-quick-setup.yaml /opt/config-quick-setup.yaml - -# Environment variables for AiiDA profile setup -ENV SETUP_DEFAULT_AIIDA_PROFILE true -ENV AIIDA_PROFILE_NAME default -ENV AIIDA_USER_EMAIL aiida@localhost -ENV AIIDA_USER_FIRST_NAME Giuseppe -ENV AIIDA_USER_LAST_NAME Verdi -ENV AIIDA_USER_INSTITUTION Khedivial - -# Switch to system user -USER $MAMBA_USER - -WORKDIR /home/${MAMBA_USER} - -ENTRYPOINT [ "/usr/local/bin/_init.sh" ] diff --git a/.docker/base/_init.sh b/.docker/base/_init.sh deleted file mode 100644 index d29c3202a7..0000000000 --- a/.docker/base/_init.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# This is the entrypoint script for the docker container. - -# Start the ssh-agent -/usr/local/bin/_entrypoint.sh ssh-agent - -# setup aiida must be run in the environment -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/aiida-profile-preparation.sh - -exec "$@" diff --git a/.docker/base/config-quick-setup.yaml b/.docker/base/config-quick-setup.yaml deleted file mode 100644 index f910069e1d..0000000000 --- a/.docker/base/config-quick-setup.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -db_engine: postgresql_psycopg2 -db_backend: core.psql_dos -db_host: database -db_port: 5432 -su_db_username: postgres -su_db_password: password -su_db_name: template1 -db_name: aiida_db -db_username: aiida -db_password: password -broker_host: messaging -broker_port: 5672 -broker_username: guest -broker_password: guest diff --git a/.docker/base/init.d/aiida-profile-preparation.sh b/.docker/base/init.d/aiida-profile-preparation.sh deleted file mode 100755 index 05dde219a0..0000000000 --- a/.docker/base/init.d/aiida-profile-preparation.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -# This script is executed whenever the docker container is (re)started. - -# Debugging. -set -x - -# Environment. -export SHELL=/bin/bash - -# Check if user requested to set up AiiDA profile (and if it exists already) -if [[ ${SETUP_DEFAULT_AIIDA_PROFILE} == true ]] && ! verdi profile show ${AIIDA_PROFILE_NAME} &> /dev/null; then - NEED_SETUP_PROFILE=true; -else - NEED_SETUP_PROFILE=false; -fi - -# Setup AiiDA profile if needed. -if [[ ${NEED_SETUP_PROFILE} == true ]]; then - - # Create AiiDA profile. - verdi quicksetup \ - --non-interactive \ - --profile "${AIIDA_PROFILE_NAME}" \ - --email "${AIIDA_USER_EMAIL}" \ - --first-name "${AIIDA_USER_FIRST_NAME}" \ - --last-name "${AIIDA_USER_LAST_NAME}" \ - --institution "${AIIDA_USER_INSTITUTION}" \ - --config /opt/config-quick-setup.yaml - - # Setup and configure local computer. - computer_name=localhost - - # Determine the number of physical cores as a default for the number of - # available MPI ranks on the localhost. We do not count "logical" cores, - # since MPI parallelization over hyper-threaded cores is typically - # associated with a significant performance penalty. We use the - # `psutil.cpu_count(logical=False)` function as opposed to simply - # `os.cpu_count()` since the latter would include hyperthreaded (logical - # cores). - NUM_PHYSICAL_CORES=$(python -c 'import psutil; print(int(psutil.cpu_count(logical=False)))' 2>/dev/null) - LOCALHOST_MPI_PROCS_PER_MACHINE=${LOCALHOST_MPI_PROCS_PER_MACHINE:-${NUM_PHYSICAL_CORES}} - - if [ -z $LOCALHOST_MPI_PROCS_PER_MACHINE ]; then - echo "Unable to automatically determine the number of logical CPUs on this " - echo "machine. Please set the LOCALHOST_MPI_PROCS_PER_MACHINE variable to " - echo "explicitly set the number of available MPI ranks." - exit 1 - fi - - verdi computer show ${computer_name} || verdi computer setup \ - --non-interactive \ - --label "${computer_name}" \ - --description "this computer" \ - --hostname "${computer_name}" \ - --transport core.local \ - --scheduler core.direct \ - --work-dir /home/${MAMBA_USER}/aiida_run/ \ - --mpirun-command "mpirun -np {tot_num_mpiprocs}" \ - --mpiprocs-per-machine ${LOCALHOST_MPI_PROCS_PER_MACHINE} && \ - verdi computer configure core.local "${computer_name}" \ - --non-interactive \ - --safe-interval 0.0 -fi - - -# Show the default profile -verdi profile show || echo "The default profile is not set." - -# Make sure that the daemon is not running, otherwise the migration will abort. -verdi daemon stop - -# Migration will run for the default profile. -verdi storage migrate --force - -# Daemon will start only if the database exists and is migrated to the latest version. -verdi daemon start || echo "AiiDA daemon is not running." diff --git a/.docker/docker-bake.hcl b/.docker/docker-bake.hcl index 60d0de6175..88a5e766c4 100644 --- a/.docker/docker-bake.hcl +++ b/.docker/docker-bake.hcl @@ -8,13 +8,6 @@ variable "PYTHON_VERSION" { variable "PGSQL_VERSION" { } -variable "AIIDA_VERSION" { -} - -variable "BASE_IMAGE" { - default = "mambaorg/micromamba:jammy" -} - variable "ORGANIZATION" { default = "aiidateam" } @@ -24,7 +17,7 @@ variable "REGISTRY" { } variable "PLATFORMS" { - default = ["linux/amd64", "linux/arm64"] + default = ["linux/amd64"] } variable "TARGETS" { @@ -34,10 +27,8 @@ variable "TARGETS" { function "tags" { params = [image] result = [ - "${REGISTRY}${ORGANIZATION}/${image}:${VERSION}", - "${REGISTRY}${ORGANIZATION}/${image}:python-${PYTHON_VERSION}", - "${REGISTRY}${ORGANIZATION}/${image}:postgresql-${PGSQL_VERSION}", - "${REGISTRY}${ORGANIZATION}/${image}:aiida-${AIIDA_VERSION}", + "${REGISTRY}${ORGANIZATION}/${image}:latest", + "${REGISTRY}${ORGANIZATION}/${image}:newly-build" ] } @@ -54,17 +45,15 @@ target "base-with-services-meta" { target "base" { inherits = ["base-meta"] - context = "base" + context = "../stacks/base" platforms = "${PLATFORMS}" args = { - "BASE" = "${BASE_IMAGE}" "PYTHON_VERSION" = "${PYTHON_VERSION}" - "AIIDA_VERSION" = "${AIIDA_VERSION}" } } target "base-with-services" { inherits = ["base-with-services-meta"] - context = "base-with-services" + context = "../stacks/base-with-services" contexts = { base = "target:base" } From b13b53eec1cfd5b7b1fe8a49ef58ba8360f6fd2b Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Tue, 4 Jul 2023 15:40:48 +0200 Subject: [PATCH 09/19] workable s6-overlay solution --- .docker/base-with-services/Dockerfile | 69 ++++++++ .docker/base-with-services/_init.sh | 20 +++ .../config-quick-setup.yaml | 3 + .../init.d/start-postgres.sh | 37 ++++ .../init.d/start-rabbitmq-amd64.sh | 25 +++ .../init.d/start-rabbitmq-arm64.sh | 32 ++++ .../suppress-rabbitmq-version-warning.sh | 7 + .docker/base/Dockerfile | 162 ++++++++++++++++++ .docker/base/fix-permissions | 35 ++++ .../base/init.d/aiida-profile-preparation.sh | 77 +++++++++ .docker/base/initial-condarc | 6 + .../base/s6-assets/config-quick-setup.yaml | 15 ++ .docker/base/s6-assets/init/aiida-prepare | 83 +++++++++ .../s6-rc.d/aiida-prepare/dependencies.d/base | 0 .../s6-rc.d/aiida-prepare/timeout-up | 1 + .../base/s6-assets/s6-rc.d/aiida-prepare/type | 1 + .../base/s6-assets/s6-rc.d/aiida-prepare/up | 4 + .../s6-rc.d/user/contents.d/aiida-prepare | 0 .docker/docker-bake.hcl | 7 +- ...er-compose.yml => docker-compose.base.yml} | 2 +- .docker/dodo.py | 62 ++++--- Dockerfile | 23 --- 22 files changed, 618 insertions(+), 53 deletions(-) create mode 100644 .docker/base-with-services/Dockerfile create mode 100644 .docker/base-with-services/_init.sh create mode 100644 .docker/base-with-services/config-quick-setup.yaml create mode 100644 .docker/base-with-services/init.d/start-postgres.sh create mode 100644 .docker/base-with-services/init.d/start-rabbitmq-amd64.sh create mode 100644 .docker/base-with-services/init.d/start-rabbitmq-arm64.sh create mode 100644 .docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh create mode 100644 .docker/base/Dockerfile create mode 100644 .docker/base/fix-permissions create mode 100755 .docker/base/init.d/aiida-profile-preparation.sh create mode 100644 .docker/base/initial-condarc create mode 100644 .docker/base/s6-assets/config-quick-setup.yaml create mode 100755 .docker/base/s6-assets/init/aiida-prepare create mode 100644 .docker/base/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/base create mode 100644 .docker/base/s6-assets/s6-rc.d/aiida-prepare/timeout-up create mode 100644 .docker/base/s6-assets/s6-rc.d/aiida-prepare/type create mode 100644 .docker/base/s6-assets/s6-rc.d/aiida-prepare/up create mode 100644 .docker/base/s6-assets/s6-rc.d/user/contents.d/aiida-prepare rename .docker/{docker-compose.yml => docker-compose.base.yml} (93%) delete mode 100644 Dockerfile diff --git a/.docker/base-with-services/Dockerfile b/.docker/base-with-services/Dockerfile new file mode 100644 index 0000000000..cc0b2057e3 --- /dev/null +++ b/.docker/base-with-services/Dockerfile @@ -0,0 +1,69 @@ +# syntax=docker/dockerfile:1 +FROM base + +LABEL maintainer="AiiDAlab Team " + +USER root +WORKDIR /opt/ + +ARG AIIDA_VERSION +ARG PGSQL_VERSION +ARG TARGETARCH + +RUN micromamba create -p /opt/conda/envs/aiida-core-services --yes \ + --channel conda-forge \ + postgresql=${PGSQL_VERSION} \ + && micromamba clean --all -f -y + +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + locales && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + locale-gen en_US.UTF-8 + +# Install RabbitMQ in a dedicated conda environment. +# If the architecture is arm64, we install the default version of rabbitmq provided by the generic binary, +# # https://www.rabbitmq.com/install-generic-unix.html the version needs to be compatible with system's erlang version. +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + micromamba install -p /opt/conda/envs/aiida-core-services --yes \ + --channel conda-forge \ + rabbitmq-server=3.8.14 && \ + micromamba clean --all -f -y; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + # Install erlang. + apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + erlang \ + xz-utils && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + # Install rabbitmq. + export RMQ_VERSION=3.9.13 && \ + wget -c --no-check-certificate https://github.com/rabbitmq/rabbitmq-server/releases/download/v${RMQ_VERSION}/rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ + tar -xf rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ + rm rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ + mv rabbitmq_server-${RMQ_VERSION} /opt/conda/envs/aiida-core-services/ && \ + ln -sf /opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/sbin/* /opt/conda/envs/aiida-core-services/bin/; \ +else \ + echo "Unknown architecture: ${TARGETARCH}."; \ +fi + +# Configure AiiDA profile. +COPY config-quick-setup.yaml . +COPY --chmod=755 init.d/start-postgres.sh /usr/local/bin/init.d/ +COPY --chmod=755 init.d/start-rabbitmq-${TARGETARCH}.sh /usr/local/bin/init.d/start-rabbitmq.sh +COPY --chmod=755 _init.sh /usr/local/bin/_init.sh + +# Supress rabbitmq version warning for arm64 since +# it is built using latest version rabbitmq from apt install. +# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf +COPY --chmod=755 init.d/suppress-rabbitmq-version-warning.sh /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + rm /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh; \ +fi + +USER $MAMBA_USER + +WORKDIR "/home/${MAMBA_USER}" + +ENTRYPOINT [ "/usr/local/bin/_init.sh" ] +# ENTRYPOINT ["_entrypoint.sh"] diff --git a/.docker/base-with-services/_init.sh b/.docker/base-with-services/_init.sh new file mode 100644 index 0000000000..f0fd0ffead --- /dev/null +++ b/.docker/base-with-services/_init.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# This is the entrypoint script for the docker container. + +# Start the ssh-agent +/usr/local/bin/_entrypoint.sh ssh-agent + +# Start postgresql +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start-postgres.sh + +# Start rabbitmq, needs to bu run by root +/usr/local/bin/init.d/start-rabbitmq.sh + +# setup aiida must be run in the environment +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/aiida-profile-preparation.sh + +# supress the warning of incorrect rabbitmq version +/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh + +exec "$@" diff --git a/.docker/base-with-services/config-quick-setup.yaml b/.docker/base-with-services/config-quick-setup.yaml new file mode 100644 index 0000000000..24c516270d --- /dev/null +++ b/.docker/base-with-services/config-quick-setup.yaml @@ -0,0 +1,3 @@ +--- +db_name: aiida_db +db_username: aiida diff --git a/.docker/base-with-services/init.d/start-postgres.sh b/.docker/base-with-services/init.d/start-postgres.sh new file mode 100644 index 0000000000..8cb18dbbaa --- /dev/null +++ b/.docker/base-with-services/init.d/start-postgres.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Activate the conda environment with PostgreSQL installed in it. +# conda activate pgsql + +# -w waits until server is up +SYSTEM_USER=${MAMBA_USER} +PSQL_START_CMD="pg_ctl --timeout=180 -w -D /home/${SYSTEM_USER}/.postgresql -l /home/${SYSTEM_USER}/.postgresql/logfile start" +PSQL_STOP_CMD="pg_ctl -w -D /home/${SYSTEM_USER}/.postgresql stop" +PSQL_STATUS_CMD="pg_ctl -D /home/${SYSTEM_USER}/.postgresql status" + +MAMBA_RUN="micromamba run -n aiida-core-services" + +# make DB directory, if not existent +if [ ! -d /home/${SYSTEM_USER}/.postgresql ]; then + mkdir /home/${SYSTEM_USER}/.postgresql + ${MAMBA_RUN} initdb -D /home/${SYSTEM_USER}/.postgresql + echo "unix_socket_directories = '/tmp'" >> /home/${SYSTEM_USER}/.postgresql/postgresql.conf + ${MAMBA_RUN} ${PSQL_START_CMD} + +# else don't +else + # Fix problem with kubernetes cluster that adds rws permissions to the group + # for more details see: https://github.com/materialscloud-org/aiidalab-z2jh-eosc/issues/5 + chmod g-rwxs /home/${NBSYSTEM_USER_USER}/.postgresql -R + + # stores return value in $? + running=true + ${MAMBA_RUN} ${PSQL_STATUS_CMD} || running=false + + # Postgresql was probably not shutdown properly. Cleaning up the mess... + if ! $running ; then + echo "" > /home/${SYSTEM_USER}/.postgresql/logfile # empty log files + rm -vf /home/${SYSTEM_USER}/.postgresql/postmaster.pid + ${MAMBA_RUN} ${PSQL_START_CMD} + fi +fi diff --git a/.docker/base-with-services/init.d/start-rabbitmq-amd64.sh b/.docker/base-with-services/init.d/start-rabbitmq-amd64.sh new file mode 100644 index 0000000000..5b48aa0b9e --- /dev/null +++ b/.docker/base-with-services/init.d/start-rabbitmq-amd64.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -em + +SYSTEM_USER=${MAMBA_USER} +RABBITMQ_DATA_DIR="/home/${SYSTEM_USER}/.rabbitmq" + +mkdir -p "${RABBITMQ_DATA_DIR}" + +# Fix issue where the erlang cookie permissions are corrupted. +chmod 400 "/home/${SYSTEM_USER}/.erlang.cookie" || echo "erlang cookie not created yet." + +# Set base directory for RabbitMQ to persist its data. This needs to be set to a folder in the system user's home +# directory as that is the only folder that is persisted outside of the container. +RMQ_ETC_DIR="/opt/conda/envs/aiida-core-services/etc/rabbitmq" +echo MNESIA_BASE="${RABBITMQ_DATA_DIR}" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" +echo LOG_BASE="${RABBITMQ_DATA_DIR}/log" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" + +# Explicitly define the node name. This is necessary because the mnesia subdirectory contains the hostname, which by +# default is set to the value of $(hostname -s), which for docker containers, will be a random hexadecimal string. Upon +# restart, this will be different and so the original mnesia folder with the persisted data will not be found. The +# reason RabbitMQ is built this way is through this way it allows to run multiple nodes on a single machine each with +# isolated mnesia directories. Since in the AiiDA setup we only need and run a single node, we can simply use localhost. +echo NODENAME=rabbit@localhost >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" + +micromamba run -n aiida-core-services rabbitmq-server -detached diff --git a/.docker/base-with-services/init.d/start-rabbitmq-arm64.sh b/.docker/base-with-services/init.d/start-rabbitmq-arm64.sh new file mode 100644 index 0000000000..11d71a2b6f --- /dev/null +++ b/.docker/base-with-services/init.d/start-rabbitmq-arm64.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -em + +SYSTEM_USER=${MAMBA_USER} +RABBITMQ_DATA_DIR="/home/${SYSTEM_USER}/.rabbitmq" + +mkdir -p "${RABBITMQ_DATA_DIR}" + +# Fix issue where the erlang cookie permissions are corrupted. +chmod 400 "/home/${SYSTEM_USER}/.erlang.cookie" || echo "erlang cookie not created yet." + +# Set base directory for RabbitMQ to persist its data. This needs to be set to a folder in the system user's home +# directory as that is the only folder that is persisted outside of the container. +export RMQ_VERSION=3.9.13 +RMQ_ETC_DIR="/opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/etc/rabbitmq" +echo MNESIA_BASE="${RABBITMQ_DATA_DIR}" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" +echo LOG_BASE="${RABBITMQ_DATA_DIR}/log" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" + +# RabbitMQ with versions >= 3.8.15 have reduced some default timeouts +# baseimage phusion/baseimage:jammy-1.0.0 running ubuntu 22.04 will install higher version of rabbimq by apt. +# using workaround from https://github.com/aiidateam/aiida-core/wiki/RabbitMQ-version-to-use +# set timeout to 100 hours +echo "consumer_timeout=3600000" >> "${RMQ_ETC_DIR}/rabbitmq.conf" + +# Explicitly define the node name. This is necessary because the mnesia subdirectory contains the hostname, which by +# default is set to the value of $(hostname -s), which for docker containers, will be a random hexadecimal string. Upon +# restart, this will be different and so the original mnesia folder with the persisted data will not be found. The +# reason RabbitMQ is built this way is through this way it allows to run multiple nodes on a single machine each with +# isolated mnesia directories. Since in the AiiDA setup we only need and run a single node, we can simply use localhost. +echo NODENAME=rabbit@localhost >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" + +micromamba run -n aiida-core-services rabbitmq-server -detached diff --git a/.docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh b/.docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh new file mode 100644 index 0000000000..f778bbb5db --- /dev/null +++ b/.docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -em + +# Supress rabbitmq version warning for arm64 since +# it is built using latest version rabbitmq from apt install. +# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf +verdi config set warnings.rabbitmq_version False diff --git a/.docker/base/Dockerfile b/.docker/base/Dockerfile new file mode 100644 index 0000000000..f73a56f195 --- /dev/null +++ b/.docker/base/Dockerfile @@ -0,0 +1,162 @@ +# syntax=docker/dockerfile:1 + +# Inspired by jupyter's docker-stacks-fundation image: +# https://github.com/jupyter/docker-stacks/blob/main/docker-stacks-foundation/Dockerfile + +ARG BASE=ubuntu:22.04 + +FROM $BASE + +LABEL maintainer="AiiDAlab Team " + +ARG SYSTEM_USER="aiida" +ARG SYSTEM_UID="1000" +ARG SYSTEM_GID="100" + + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + + + +# Install all OS dependencies for notebook server that starts but lacks all +# features (e.g., download as all possible file formats) +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update --yes && \ + # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as + # the ubuntu base image is rebuilt too seldom sometimes (less than once a month) + apt-get upgrade --yes && \ + apt-get install --yes --no-install-recommends \ + # - bzip2 is necessary to extract the micromamba executable. + bzip2 \ + # - xz-utils is necessary to extract the s6-overlay. + xz-utils \ + ca-certificates \ + locales \ + sudo \ + # the gcc compiler need to build some python packages e.g. psutil and pymatgen + build-essential \ + wget && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen + +# Install s6-overlay to handle startup and shutdown of services +ARG S6_OVERLAY_VERSION=3.1.5.0 +RUN wget --progress=dot:giga -O /tmp/s6-overlay-noarch.tar.xz \ + "https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz" && \ + tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && \ + rm /tmp/s6-overlay-noarch.tar.xz + +RUN set -x && \ + arch=$(uname -m) && \ + wget --progress=dot:giga -O /tmp/s6-overlay-binary.tar.xz \ + "https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-${arch}.tar.xz" && \ + tar -C / -Jxpf /tmp/s6-overlay-binary.tar.xz && \ + rm /tmp/s6-overlay-binary.tar.xz + +# Configure environment +ENV CONDA_DIR=/opt/conda \ + SHELL=/bin/bash \ + SYSTEM_USER="${SYSTEM_USER}" \ + SYSTEM_UID=${SYSTEM_UID} \ + SYSTEM_GID=${SYSTEM_GID} \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 +ENV PATH="${CONDA_DIR}/bin:${PATH}" \ + HOME="/home/${SYSTEM_USER}" + + +# Copy a script that we will use to correct permissions after running certain commands +COPY fix-permissions /usr/local/bin/fix-permissions +RUN chmod a+rx /usr/local/bin/fix-permissions + +# Enable prompt color in the skeleton .bashrc before creating the default SYSTEM_USER +# hadolint ignore=SC2016 +RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ + # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 + echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc + +# Create SYSTEM_USER with name jovyan user with UID=1000 and in the 'users' group +# and make sure these dirs are writable by the `users` group. +RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ + sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ + sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ + useradd -l -m -s /bin/bash -N -u "${SYSTEM_UID}" "${SYSTEM_USER}" && \ + mkdir -p "${CONDA_DIR}" && \ + chown "${SYSTEM_USER}:${SYSTEM_GID}" "${CONDA_DIR}" && \ + chmod g+w /etc/passwd && \ + fix-permissions "${HOME}" && \ + fix-permissions "${CONDA_DIR}" + +USER ${SYSTEM_UID} + +# Pin python version here +ARG PYTHON_VERSION + +# Download and install Micromamba, and initialize Conda prefix. +# +# Similar projects using Micromamba: +# - Micromamba-Docker: +# - repo2docker: +# Install Python, Mamba and jupyter_core +# Cleanup temporary files and remove Micromamba +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" initial-condarc "${CONDA_DIR}/.condarc" +WORKDIR /tmp +RUN set -x && \ + arch=$(uname -m) && \ + if [ "${arch}" = "x86_64" ]; then \ + # Should be simpler, see + arch="64"; \ + fi && \ + wget --progress=dot:giga -O /tmp/micromamba.tar.bz2 \ + "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ + tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ + rm /tmp/micromamba.tar.bz2 && \ + PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ + if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ + # Install the packages + ./micromamba install \ + --root-prefix="${CONDA_DIR}" \ + --prefix="${CONDA_DIR}" \ + --yes \ + "${PYTHON_SPECIFIER}" \ + 'mamba' && \ + rm micromamba && \ + # Pin major.minor version of python + mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${SYSTEM_USER}" + +# Switch to root to install AiiDA and set AiiDA as service +USER root +COPY --from=src . /tmp/aiida-core +# RUN pip install /tmp/aiida-core --no-cache-dir && \ +# rm -rf /tmp/aiida-core +RUN pip install aiida-core==2.4.0 + +# Enable verdi autocompletion. +RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \ + echo 'eval "$(_VERDI_COMPLETE=bash_source verdi)"' >> "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ + chmod +x "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ + fix-permissions "${CONDA_DIR}" + +# COPY AiiDA profile configuration for profile setup init script +COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" s6-assets/config-quick-setup.yaml "/aiida/assets/config-quick-setup.yaml" +COPY s6-assets/s6-rc.d /etc/s6-overlay/s6-rc.d +COPY s6-assets/init /etc/init + +# Switch back to USER aiida to avoid accidental container runs as root +USER ${SYSTEM_UID} + +ENTRYPOINT ["/init"] + +WORKDIR "${HOME}" \ No newline at end of file diff --git a/.docker/base/fix-permissions b/.docker/base/fix-permissions new file mode 100644 index 0000000000..ad8a1cc839 --- /dev/null +++ b/.docker/base/fix-permissions @@ -0,0 +1,35 @@ +#!/bin/bash +# This is brought from jupyter docker-stacks: +# https://github.com/jupyter/docker-stacks/blob/main/docker-stacks-foundation/fix-permissions +# set permissions on a directory +# after any installation, if a directory needs to be (human) user-writable, +# run this script on it. +# It will make everything in the directory owned by the group ${SYSTEM_GID} +# and writable by that group. + +# uses find to avoid touching files that already have the right permissions, +# which would cause massive image explosion + +# right permissions are: +# group=${SYSEM_GID} +# AND permissions include group rwX (directory-execute) +# AND directories have setuid,setgid bits set + +set -e + +for d in "$@"; do + find "${d}" \ + ! \( \ + -group "${SYSTEM_GID}" \ + -a -perm -g+rwX \ + \) \ + -exec chgrp "${SYSTEM_GID}" -- {} \+ \ + -exec chmod g+rwX -- {} \+ + # setuid, setgid *on directories only* + find "${d}" \ + \( \ + -type d \ + -a ! -perm -6000 \ + \) \ + -exec chmod +6000 -- {} \+ +done \ No newline at end of file diff --git a/.docker/base/init.d/aiida-profile-preparation.sh b/.docker/base/init.d/aiida-profile-preparation.sh new file mode 100755 index 0000000000..05dde219a0 --- /dev/null +++ b/.docker/base/init.d/aiida-profile-preparation.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# This script is executed whenever the docker container is (re)started. + +# Debugging. +set -x + +# Environment. +export SHELL=/bin/bash + +# Check if user requested to set up AiiDA profile (and if it exists already) +if [[ ${SETUP_DEFAULT_AIIDA_PROFILE} == true ]] && ! verdi profile show ${AIIDA_PROFILE_NAME} &> /dev/null; then + NEED_SETUP_PROFILE=true; +else + NEED_SETUP_PROFILE=false; +fi + +# Setup AiiDA profile if needed. +if [[ ${NEED_SETUP_PROFILE} == true ]]; then + + # Create AiiDA profile. + verdi quicksetup \ + --non-interactive \ + --profile "${AIIDA_PROFILE_NAME}" \ + --email "${AIIDA_USER_EMAIL}" \ + --first-name "${AIIDA_USER_FIRST_NAME}" \ + --last-name "${AIIDA_USER_LAST_NAME}" \ + --institution "${AIIDA_USER_INSTITUTION}" \ + --config /opt/config-quick-setup.yaml + + # Setup and configure local computer. + computer_name=localhost + + # Determine the number of physical cores as a default for the number of + # available MPI ranks on the localhost. We do not count "logical" cores, + # since MPI parallelization over hyper-threaded cores is typically + # associated with a significant performance penalty. We use the + # `psutil.cpu_count(logical=False)` function as opposed to simply + # `os.cpu_count()` since the latter would include hyperthreaded (logical + # cores). + NUM_PHYSICAL_CORES=$(python -c 'import psutil; print(int(psutil.cpu_count(logical=False)))' 2>/dev/null) + LOCALHOST_MPI_PROCS_PER_MACHINE=${LOCALHOST_MPI_PROCS_PER_MACHINE:-${NUM_PHYSICAL_CORES}} + + if [ -z $LOCALHOST_MPI_PROCS_PER_MACHINE ]; then + echo "Unable to automatically determine the number of logical CPUs on this " + echo "machine. Please set the LOCALHOST_MPI_PROCS_PER_MACHINE variable to " + echo "explicitly set the number of available MPI ranks." + exit 1 + fi + + verdi computer show ${computer_name} || verdi computer setup \ + --non-interactive \ + --label "${computer_name}" \ + --description "this computer" \ + --hostname "${computer_name}" \ + --transport core.local \ + --scheduler core.direct \ + --work-dir /home/${MAMBA_USER}/aiida_run/ \ + --mpirun-command "mpirun -np {tot_num_mpiprocs}" \ + --mpiprocs-per-machine ${LOCALHOST_MPI_PROCS_PER_MACHINE} && \ + verdi computer configure core.local "${computer_name}" \ + --non-interactive \ + --safe-interval 0.0 +fi + + +# Show the default profile +verdi profile show || echo "The default profile is not set." + +# Make sure that the daemon is not running, otherwise the migration will abort. +verdi daemon stop + +# Migration will run for the default profile. +verdi storage migrate --force + +# Daemon will start only if the database exists and is migrated to the latest version. +verdi daemon start || echo "AiiDA daemon is not running." diff --git a/.docker/base/initial-condarc b/.docker/base/initial-condarc new file mode 100644 index 0000000000..66ecf60813 --- /dev/null +++ b/.docker/base/initial-condarc @@ -0,0 +1,6 @@ +# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html + +auto_update_conda: false +show_channel_urls: true +channels: + - conda-forge \ No newline at end of file diff --git a/.docker/base/s6-assets/config-quick-setup.yaml b/.docker/base/s6-assets/config-quick-setup.yaml new file mode 100644 index 0000000000..f910069e1d --- /dev/null +++ b/.docker/base/s6-assets/config-quick-setup.yaml @@ -0,0 +1,15 @@ +--- +db_engine: postgresql_psycopg2 +db_backend: core.psql_dos +db_host: database +db_port: 5432 +su_db_username: postgres +su_db_password: password +su_db_name: template1 +db_name: aiida_db +db_username: aiida +db_password: password +broker_host: messaging +broker_port: 5672 +broker_username: guest +broker_password: guest diff --git a/.docker/base/s6-assets/init/aiida-prepare b/.docker/base/s6-assets/init/aiida-prepare new file mode 100755 index 0000000000..e370c181e9 --- /dev/null +++ b/.docker/base/s6-assets/init/aiida-prepare @@ -0,0 +1,83 @@ +#!/bin/bash + +# This script is executed whenever the docker container is (re)started. + +# Environment. +export SHELL=/bin/bash + +# Configure AiiDA. +export SETUP_DEFAULT_AIIDA_PROFILE=true +export AIIDA_PROFILE_NAME=default +export AIIDA_USER_EMAIL=aiida@localhost +export AIIDA_USER_FIRST_NAME=Giuseppe +export AIIDA_USER_LAST_NAME=Verdi +export AIIDA_USER_INSTITUTION=Khedivial +export AIIDA_PROFILE_PATH=/aiida/assets/config-quick-setup.yaml + +# Check if user requested to set up AiiDA profile (and if it exists already) +if [[ ${SETUP_DEFAULT_AIIDA_PROFILE} == true ]] && ! verdi profile show ${AIIDA_PROFILE_NAME} &> /dev/null; then + NEED_SETUP_PROFILE=true; +else + NEED_SETUP_PROFILE=false; +fi + +# Setup AiiDA profile if needed. +if [[ ${NEED_SETUP_PROFILE} == true ]]; then + + # Create AiiDA profile. + verdi quicksetup \ + --non-interactive \ + --profile "${AIIDA_PROFILE_NAME}" \ + --email "${AIIDA_USER_EMAIL}" \ + --first-name "${AIIDA_USER_FIRST_NAME}" \ + --last-name "${AIIDA_USER_LAST_NAME}" \ + --institution "${AIIDA_USER_INSTITUTION}" \ + --config "${AIIDA_PROFILE_PATH}" + + # Setup and configure local computer. + computer_name=localhost + + # Determine the number of physical cores as a default for the number of + # available MPI ranks on the localhost. We do not count "logical" cores, + # since MPI parallelization over hyper-threaded cores is typically + # associated with a significant performance penalty. We use the + # `psutil.cpu_count(logical=False)` function as opposed to simply + # `os.cpu_count()` since the latter would include hyperthreaded (logical + # cores). + NUM_PHYSICAL_CORES=$(python -c 'import psutil; print(int(psutil.cpu_count(logical=False)))' 2>/dev/null) + LOCALHOST_MPI_PROCS_PER_MACHINE=${LOCALHOST_MPI_PROCS_PER_MACHINE:-${NUM_PHYSICAL_CORES}} + + if [ -z $LOCALHOST_MPI_PROCS_PER_MACHINE ]; then + echo "Unable to automatically determine the number of logical CPUs on this " + echo "machine. Please set the LOCALHOST_MPI_PROCS_PER_MACHINE variable to " + echo "explicitly set the number of available MPI ranks." + exit 1 + fi + + verdi computer show ${computer_name} || verdi computer setup \ + --non-interactive \ + --label "${computer_name}" \ + --description "this computer" \ + --hostname "${computer_name}" \ + --transport core.local \ + --scheduler core.direct \ + --work-dir /home/${MAMBA_USER}/aiida_run/ \ + --mpirun-command "mpirun -np {tot_num_mpiprocs}" \ + --mpiprocs-per-machine ${LOCALHOST_MPI_PROCS_PER_MACHINE} && \ + verdi computer configure core.local "${computer_name}" \ + --non-interactive \ + --safe-interval 0.0 +fi + + +# Show the default profile +verdi profile show || echo "The default profile is not set." + +# Make sure that the daemon is not running, otherwise the migration will abort. +verdi daemon stop + +# Migration will run for the default profile. +verdi storage migrate --force + +# Daemon will start only if the database exists and is migrated to the latest version. +verdi daemon start || echo "AiiDA daemon is not running." diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/base b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/base new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-prepare/timeout-up b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/timeout-up new file mode 100644 index 0000000000..c227083464 --- /dev/null +++ b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/timeout-up @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-prepare/type b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/type new file mode 100644 index 0000000000..3d92b15f2d --- /dev/null +++ b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/type @@ -0,0 +1 @@ +oneshot \ No newline at end of file diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-prepare/up b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/up new file mode 100644 index 0000000000..703069c963 --- /dev/null +++ b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/up @@ -0,0 +1,4 @@ +#!/command/execlineb -S0 + +foreground { s6-echo "Calling /etc/init/aiida-prepare" } +/etc/init/aiida-prepare \ No newline at end of file diff --git a/.docker/base/s6-assets/s6-rc.d/user/contents.d/aiida-prepare b/.docker/base/s6-assets/s6-rc.d/user/contents.d/aiida-prepare new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/docker-bake.hcl b/.docker/docker-bake.hcl index 88a5e766c4..cb65d4e683 100644 --- a/.docker/docker-bake.hcl +++ b/.docker/docker-bake.hcl @@ -45,7 +45,10 @@ target "base-with-services-meta" { target "base" { inherits = ["base-meta"] - context = "../stacks/base" + context = "base" + contexts = { + src = ".." + } platforms = "${PLATFORMS}" args = { "PYTHON_VERSION" = "${PYTHON_VERSION}" @@ -53,7 +56,7 @@ target "base" { } target "base-with-services" { inherits = ["base-with-services-meta"] - context = "../stacks/base-with-services" + context = "base-with-services" contexts = { base = "target:base" } diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.base.yml similarity index 93% rename from .docker/docker-compose.yml rename to .docker/docker-compose.base.yml index 37ae6f0793..8487313b6d 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.base.yml @@ -24,7 +24,7 @@ services: - aiida-rmq-data:/var/lib/rabbitmq/ aiida: - image: ${REGISTRY:-}${BASE_IMAGE:-aiidateam/base}${VERSION:-} + image: ${REGISTRY:-}${BASE_IMAGE:-aiidateam/base}:${VERSION:-latest} environment: RMQHOST: messaging TZ: Europe/Zurich diff --git a/.docker/dodo.py b/.docker/dodo.py index 19eedda420..333ec4bdf4 100644 --- a/.docker/dodo.py +++ b/.docker/dodo.py @@ -47,7 +47,7 @@ _VERSION_PARAM = { "name": "version", "long": "version", - "type": "str", + "type": str, "default": VERSION, "help": ( "Specify the version of the stack for building / testing. Defaults to a " @@ -125,29 +125,37 @@ def task_tests(): } -# def task_up(): -# """Start AiiDAlab server for testing.""" -# return { -# "actions": [ -# "AIIDALAB_PORT=%(port)i REGISTRY=%(registry)s VERSION=:%(version)s " -# "docker-compose up --detach" -# ], -# "params": [ -# { -# "name": "port", -# "short": "p", -# "long": "port", -# "type": int, -# "default": 8888, -# "help": "Specify the AiiDAlab host port.", -# }, -# _REGISTRY_PARAM, -# _VERSION_PARAM, -# ], -# "verbosity": 2, -# } - - -# def task_down(): -# """Stop AiiDAlab server.""" -# return {"actions": ["docker-compose down"], "verbosity": 2} \ No newline at end of file +def task_up(): + """Start AiiDAlab server for testing.""" + return { + "actions": [ + "AIIDALAB_PORT=%(port)i REGISTRY=%(registry)s VERSION=%(version)s " + "docker-compose -f docker-compose.%(target)s.yml up" + ], + "params": [ + { + "name": "port", + "short": "p", + "long": "port", + "type": int, + "default": 8888, + "help": "Specify the AiiDAlab host port.", + }, + { + "name": "target", + "short": "t", + "long": "target", + "type": str, + "default": "base-with-services", + "help": "Specify the target to run.", + }, + _REGISTRY_PARAM, + _VERSION_PARAM, + ], + "verbosity": 2, + } + + +def task_down(): + """Stop AiiDAlab server.""" + return {"actions": ["docker-compose down"], "verbosity": 2} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 9085a5e3ab..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM aiidateam/aiida-prerequisites:0.7.0 - -USER root - -ENV SETUP_DEFAULT_PROFILE true - -ENV PROFILE_NAME default -ENV USER_EMAIL aiida@localhost -ENV USER_FIRST_NAME Giuseppe -ENV USER_LAST_NAME Verdi -ENV USER_INSTITUTION Khedivial -ENV AIIDADB_BACKEND core.psql_dos - -# Copy and install AiiDA -COPY . aiida-core -RUN pip install ./aiida-core[atomic_tools] - -# Configure aiida for the user -COPY .docker/opt/configure-aiida.sh /opt/configure-aiida.sh -COPY .docker/my_init.d/configure-aiida.sh /etc/my_init.d/40_configure-aiida.sh - -# Use phusion baseimage docker init system. -CMD ["/sbin/my_init"] From d150363ad8c8133ae4884b9ce9fe552b056ca38e Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Wed, 5 Jul 2023 11:55:05 +0200 Subject: [PATCH 10/19] tear down the aiida daemon rmq still not working no home cookie file error work [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .docker/README.md | 2 +- .docker/base-with-services/Dockerfile | 64 +++----- .docker/base-with-services/_init.sh | 20 --- .../init.d/start-postgres.sh | 37 ----- .../init.d/start-rabbitmq-amd64.sh | 25 --- .../{ => s6-assets}/config-quick-setup.yaml | 0 .../s6-assets/init/postgresql-init.sh | 28 ++++ .../s6-assets/init/postgresql-prepare.sh | 10 ++ .../init/rabbitmq-init.sh} | 10 +- .../s6-rc.d/aiida-prepare/dependencies.d/base | 0 .../aiida-prepare/dependencies.d/postgresql | 0 .../dependencies.d/postgresql-prepare | 0 .../postgresql-init/dependencies.d/base | 0 .../s6-rc.d/postgresql-init/timeout-up | 1 + .../s6-assets/s6-rc.d/postgresql-init/type | 1 + .../s6-assets/s6-rc.d/postgresql-init/up | 6 + .../postgresql-prepare/dependencies.d/base | 0 .../dependencies.d/postgresql | 0 .../s6-rc.d/postgresql-prepare/timeout-up | 1 + .../s6-assets/s6-rc.d/postgresql-prepare/type | 1 + .../s6-assets/s6-rc.d/postgresql-prepare/up | 6 + .../s6-rc.d/postgresql/dependencies.d/base | 0 .../postgresql/dependencies.d/postgresql-init | 0 .../s6-assets/s6-rc.d/postgresql/down-signal | 1 + .../s6-assets/s6-rc.d/postgresql/run | 29 ++++ .../s6-assets/s6-rc.d/postgresql/type | 1 + .../s6-rc.d/rabbitmq-init/dependencies.d/base | 0 .../s6-rc.d/rabbitmq-init/timeout-up | 1 + .../s6-assets/s6-rc.d/rabbitmq-init/type | 1 + .../s6-assets/s6-rc.d/rabbitmq-init/up | 4 + .../s6-rc.d/rabbitmq/dependencies.d/base | 0 .../rabbitmq/dependencies.d/rabbitmq-init | 0 .../s6-assets/s6-rc.d/rabbitmq/down-signal | 1 + .../s6-assets/s6-rc.d/rabbitmq/run | 6 + .../s6-assets/s6-rc.d/rabbitmq/type | 1 + .../s6-rc.d/user/contents.d/postgresql | 0 .../s6-rc.d/user/contents.d/postgresql-init | 0 .../user/contents.d/postgresql-prepare | 0 .../s6-rc.d/user/contents.d/rabbitmq | 0 .../s6-rc.d/user/contents.d/rabbitmq-init | 0 .../verdi-rabbitmq-warning-suppress | 0 .../dependencies.d/aiida-prepare | 0 .../dependencies.d/base | 0 .../timeout-up | 1 + .../verdi-rabbitmq-warning-suppress/type | 1 + .../verdi-rabbitmq-warning-suppress/up} | 3 +- .docker/base/Dockerfile | 8 +- .docker/base/fix-permissions | 2 +- .../base/init.d/aiida-profile-preparation.sh | 77 --------- .docker/base/initial-condarc | 2 +- .../init/{aiida-prepare => aiida-prepare.sh} | 6 - .../dependencies.d/aiida-prepare | 0 .../aiida-daemon-start/dependencies.d/base | 0 .../s6-assets/s6-rc.d/aiida-daemon-start/down | 1 + .../s6-rc.d/aiida-daemon-start/timeout-up | 1 + .../s6-assets/s6-rc.d/aiida-daemon-start/type | 1 + .../s6-assets/s6-rc.d/aiida-daemon-start/up | 3 + .../s6-rc.d/aiida-prepare/timeout-up | 2 +- .../base/s6-assets/s6-rc.d/aiida-prepare/type | 2 +- .../base/s6-assets/s6-rc.d/aiida-prepare/up | 2 +- .../user/contents.d/aiida-daemon-start | 0 .docker/build.json | 5 +- .docker/docker-bake.hcl | 1 + .docker/docker-compose.base-with-services.yml | 15 ++ .docker/docker-compose.base.yml | 10 +- .docker/dodo.py | 146 +++++++++--------- .docker/requirements.txt | 2 +- .gitignore | 2 +- 68 files changed, 243 insertions(+), 307 deletions(-) delete mode 100644 .docker/base-with-services/_init.sh delete mode 100644 .docker/base-with-services/init.d/start-postgres.sh delete mode 100644 .docker/base-with-services/init.d/start-rabbitmq-amd64.sh rename .docker/base-with-services/{ => s6-assets}/config-quick-setup.yaml (100%) create mode 100755 .docker/base-with-services/s6-assets/init/postgresql-init.sh create mode 100755 .docker/base-with-services/s6-assets/init/postgresql-prepare.sh rename .docker/base-with-services/{init.d/start-rabbitmq-arm64.sh => s6-assets/init/rabbitmq-init.sh} (83%) mode change 100644 => 100755 create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/base create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/postgresql create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/postgresql-prepare create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/dependencies.d/base create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/timeout-up create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/type create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/up create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/dependencies.d/base create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/dependencies.d/postgresql create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/timeout-up create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/type create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/up create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql/dependencies.d/base create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql/dependencies.d/postgresql-init create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql/down-signal create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql/run create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/postgresql/type create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/dependencies.d/base create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/timeout-up create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/type create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/up create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/dependencies.d/base create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/dependencies.d/rabbitmq-init create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/down-signal create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/run create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/type create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/postgresql create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/postgresql-init create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/postgresql-prepare create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/rabbitmq create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/rabbitmq-init create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/verdi-rabbitmq-warning-suppress create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/dependencies.d/aiida-prepare create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/dependencies.d/base create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/timeout-up create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/type rename .docker/base-with-services/{init.d/suppress-rabbitmq-version-warning.sh => s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/up} (91%) delete mode 100755 .docker/base/init.d/aiida-profile-preparation.sh rename .docker/base/s6-assets/init/{aiida-prepare => aiida-prepare.sh} (93%) create mode 100644 .docker/base/s6-assets/s6-rc.d/aiida-daemon-start/dependencies.d/aiida-prepare create mode 100644 .docker/base/s6-assets/s6-rc.d/aiida-daemon-start/dependencies.d/base create mode 100644 .docker/base/s6-assets/s6-rc.d/aiida-daemon-start/down create mode 100644 .docker/base/s6-assets/s6-rc.d/aiida-daemon-start/timeout-up create mode 100644 .docker/base/s6-assets/s6-rc.d/aiida-daemon-start/type create mode 100644 .docker/base/s6-assets/s6-rc.d/aiida-daemon-start/up create mode 100644 .docker/base/s6-assets/s6-rc.d/user/contents.d/aiida-daemon-start create mode 100644 .docker/docker-compose.base-with-services.yml diff --git a/.docker/README.md b/.docker/README.md index 4df6dd7413..27f4cf7fdf 100644 --- a/.docker/README.md +++ b/.docker/README.md @@ -8,4 +8,4 @@ The build system will attempt to detect the local architecture and automatically All commands `build`, `tests`, and `up` will use the locally detected platform and use a version tag based on the state of the local git repository. However, you can also specify a custom platform or version with the `--platform` and `--version` parameters, example: `doit build --arch=amd64 --version=my-version`. -You can specify target stacks to build with `--target`, example: `doit build --target base --target base`. \ No newline at end of file +You can specify target stacks to build with `--target`, example: `doit build --target base --target base`. diff --git a/.docker/base-with-services/Dockerfile b/.docker/base-with-services/Dockerfile index cc0b2057e3..f238ffefdc 100644 --- a/.docker/base-with-services/Dockerfile +++ b/.docker/base-with-services/Dockerfile @@ -6,64 +6,42 @@ LABEL maintainer="AiiDAlab Team " USER root WORKDIR /opt/ -ARG AIIDA_VERSION ARG PGSQL_VERSION -ARG TARGETARCH +ARG RMQ_VERSION -RUN micromamba create -p /opt/conda/envs/aiida-core-services --yes \ +ENV PGSQL_VERSION=${PGSQL_VERSION} +ENV RMQ_VERSION=${RMQ_VERSION} + +RUN mamba create -p /opt/conda/envs/aiida-core-services --yes \ --channel conda-forge \ - postgresql=${PGSQL_VERSION} \ - && micromamba clean --all -f -y + postgresql=${PGSQL_VERSION} && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" +# Install erlang. RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends \ - locales && \ - apt-get clean && rm -rf /var/lib/apt/lists/* && \ - locale-gen en_US.UTF-8 - -# Install RabbitMQ in a dedicated conda environment. -# If the architecture is arm64, we install the default version of rabbitmq provided by the generic binary, -# # https://www.rabbitmq.com/install-generic-unix.html the version needs to be compatible with system's erlang version. -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - micromamba install -p /opt/conda/envs/aiida-core-services --yes \ - --channel conda-forge \ - rabbitmq-server=3.8.14 && \ - micromamba clean --all -f -y; \ -elif [ "$TARGETARCH" = "arm64" ]; then \ - # Install erlang. - apt-get update --yes && \ apt-get install --yes --no-install-recommends \ erlang \ xz-utils && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ # Install rabbitmq. - export RMQ_VERSION=3.9.13 && \ wget -c --no-check-certificate https://github.com/rabbitmq/rabbitmq-server/releases/download/v${RMQ_VERSION}/rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ tar -xf rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ rm rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ mv rabbitmq_server-${RMQ_VERSION} /opt/conda/envs/aiida-core-services/ && \ - ln -sf /opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/sbin/* /opt/conda/envs/aiida-core-services/bin/; \ -else \ - echo "Unknown architecture: ${TARGETARCH}."; \ -fi - -# Configure AiiDA profile. -COPY config-quick-setup.yaml . -COPY --chmod=755 init.d/start-postgres.sh /usr/local/bin/init.d/ -COPY --chmod=755 init.d/start-rabbitmq-${TARGETARCH}.sh /usr/local/bin/init.d/start-rabbitmq.sh -COPY --chmod=755 _init.sh /usr/local/bin/_init.sh + ln -sf /opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/sbin/* /opt/conda/envs/aiida-core-services/bin/ && \ + fix-permissions "${CONDA_DIR}" -# Supress rabbitmq version warning for arm64 since -# it is built using latest version rabbitmq from apt install. -# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf -COPY --chmod=755 init.d/suppress-rabbitmq-version-warning.sh /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - rm /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh; \ -fi +# s6-overlay to start services +COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" s6-assets/config-quick-setup.yaml "/aiida/assets/config-quick-setup.yaml" +COPY s6-assets/s6-rc.d /etc/s6-overlay/s6-rc.d +COPY s6-assets/init /etc/init -USER $MAMBA_USER +# Otherwise will stuck on oneshot services +# https://github.com/just-containers/s6-overlay/issues/467 +ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0 -WORKDIR "/home/${MAMBA_USER}" +USER ${SYSTEM_UID} -ENTRYPOINT [ "/usr/local/bin/_init.sh" ] -# ENTRYPOINT ["_entrypoint.sh"] +WORKDIR "/home/${SYSTEM_USER}" diff --git a/.docker/base-with-services/_init.sh b/.docker/base-with-services/_init.sh deleted file mode 100644 index f0fd0ffead..0000000000 --- a/.docker/base-with-services/_init.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# This is the entrypoint script for the docker container. - -# Start the ssh-agent -/usr/local/bin/_entrypoint.sh ssh-agent - -# Start postgresql -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/start-postgres.sh - -# Start rabbitmq, needs to bu run by root -/usr/local/bin/init.d/start-rabbitmq.sh - -# setup aiida must be run in the environment -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/aiida-profile-preparation.sh - -# supress the warning of incorrect rabbitmq version -/usr/local/bin/_entrypoint.sh /usr/local/bin/init.d/suppress-rabbitmq-version-warning.sh - -exec "$@" diff --git a/.docker/base-with-services/init.d/start-postgres.sh b/.docker/base-with-services/init.d/start-postgres.sh deleted file mode 100644 index 8cb18dbbaa..0000000000 --- a/.docker/base-with-services/init.d/start-postgres.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Activate the conda environment with PostgreSQL installed in it. -# conda activate pgsql - -# -w waits until server is up -SYSTEM_USER=${MAMBA_USER} -PSQL_START_CMD="pg_ctl --timeout=180 -w -D /home/${SYSTEM_USER}/.postgresql -l /home/${SYSTEM_USER}/.postgresql/logfile start" -PSQL_STOP_CMD="pg_ctl -w -D /home/${SYSTEM_USER}/.postgresql stop" -PSQL_STATUS_CMD="pg_ctl -D /home/${SYSTEM_USER}/.postgresql status" - -MAMBA_RUN="micromamba run -n aiida-core-services" - -# make DB directory, if not existent -if [ ! -d /home/${SYSTEM_USER}/.postgresql ]; then - mkdir /home/${SYSTEM_USER}/.postgresql - ${MAMBA_RUN} initdb -D /home/${SYSTEM_USER}/.postgresql - echo "unix_socket_directories = '/tmp'" >> /home/${SYSTEM_USER}/.postgresql/postgresql.conf - ${MAMBA_RUN} ${PSQL_START_CMD} - -# else don't -else - # Fix problem with kubernetes cluster that adds rws permissions to the group - # for more details see: https://github.com/materialscloud-org/aiidalab-z2jh-eosc/issues/5 - chmod g-rwxs /home/${NBSYSTEM_USER_USER}/.postgresql -R - - # stores return value in $? - running=true - ${MAMBA_RUN} ${PSQL_STATUS_CMD} || running=false - - # Postgresql was probably not shutdown properly. Cleaning up the mess... - if ! $running ; then - echo "" > /home/${SYSTEM_USER}/.postgresql/logfile # empty log files - rm -vf /home/${SYSTEM_USER}/.postgresql/postmaster.pid - ${MAMBA_RUN} ${PSQL_START_CMD} - fi -fi diff --git a/.docker/base-with-services/init.d/start-rabbitmq-amd64.sh b/.docker/base-with-services/init.d/start-rabbitmq-amd64.sh deleted file mode 100644 index 5b48aa0b9e..0000000000 --- a/.docker/base-with-services/init.d/start-rabbitmq-amd64.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -em - -SYSTEM_USER=${MAMBA_USER} -RABBITMQ_DATA_DIR="/home/${SYSTEM_USER}/.rabbitmq" - -mkdir -p "${RABBITMQ_DATA_DIR}" - -# Fix issue where the erlang cookie permissions are corrupted. -chmod 400 "/home/${SYSTEM_USER}/.erlang.cookie" || echo "erlang cookie not created yet." - -# Set base directory for RabbitMQ to persist its data. This needs to be set to a folder in the system user's home -# directory as that is the only folder that is persisted outside of the container. -RMQ_ETC_DIR="/opt/conda/envs/aiida-core-services/etc/rabbitmq" -echo MNESIA_BASE="${RABBITMQ_DATA_DIR}" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" -echo LOG_BASE="${RABBITMQ_DATA_DIR}/log" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" - -# Explicitly define the node name. This is necessary because the mnesia subdirectory contains the hostname, which by -# default is set to the value of $(hostname -s), which for docker containers, will be a random hexadecimal string. Upon -# restart, this will be different and so the original mnesia folder with the persisted data will not be found. The -# reason RabbitMQ is built this way is through this way it allows to run multiple nodes on a single machine each with -# isolated mnesia directories. Since in the AiiDA setup we only need and run a single node, we can simply use localhost. -echo NODENAME=rabbit@localhost >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" - -micromamba run -n aiida-core-services rabbitmq-server -detached diff --git a/.docker/base-with-services/config-quick-setup.yaml b/.docker/base-with-services/s6-assets/config-quick-setup.yaml similarity index 100% rename from .docker/base-with-services/config-quick-setup.yaml rename to .docker/base-with-services/s6-assets/config-quick-setup.yaml diff --git a/.docker/base-with-services/s6-assets/init/postgresql-init.sh b/.docker/base-with-services/s6-assets/init/postgresql-init.sh new file mode 100755 index 0000000000..e3c5edf0b9 --- /dev/null +++ b/.docker/base-with-services/s6-assets/init/postgresql-init.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# -w waits until server is up +MAMBA_RUN="mamba run -n aiida-core-services" +SYSTEM_USER="aiida" + +# make DB directory, if not existent +if [ ! -d /home/${SYSTEM_USER}/.postgresql ]; then + mkdir /home/${SYSTEM_USER}/.postgresql + ${MAMBA_RUN} initdb -D /home/${SYSTEM_USER}/.postgresql + echo "unix_socket_directories = '/tmp'" >> /home/${SYSTEM_USER}/.postgresql/postgresql.conf +fi + +PSQL_STATUS_CMD="pg_ctl -D /home/${SYSTEM_USER}/.postgresql status" + +# Fix problem with kubernetes cluster that adds rws permissions to the group +# for more details see: https://github.com/materialscloud-org/aiidalab-z2jh-eosc/issues/5 +chmod g-rwxs /home/${SYSTEM_USER}/.postgresql -R + +# stores return value in $? +running=true +${MAMBA_RUN} ${PSQL_STATUS_CMD} || running=false + +# Postgresql was probably not shutdown properly. Cleaning up the mess... +if ! $running ; then + echo "" > /home/${SYSTEM_USER}/.postgresql/logfile # empty log files + rm -vf /home/${SYSTEM_USER}/.postgresql/postmaster.pid +fi diff --git a/.docker/base-with-services/s6-assets/init/postgresql-prepare.sh b/.docker/base-with-services/s6-assets/init/postgresql-prepare.sh new file mode 100755 index 0000000000..9822ab4d04 --- /dev/null +++ b/.docker/base-with-services/s6-assets/init/postgresql-prepare.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +MAMBA_RUN="mamba run -n aiida-core-services" + +PG_ISREADY=1 +while [ "$PG_ISREADY" != "0" ]; do + sleep 1 + ${MAMBA_RUN} pg_isready --quiet + PG_ISREADY=$? +done diff --git a/.docker/base-with-services/init.d/start-rabbitmq-arm64.sh b/.docker/base-with-services/s6-assets/init/rabbitmq-init.sh old mode 100644 new mode 100755 similarity index 83% rename from .docker/base-with-services/init.d/start-rabbitmq-arm64.sh rename to .docker/base-with-services/s6-assets/init/rabbitmq-init.sh index 11d71a2b6f..42c996ae0f --- a/.docker/base-with-services/init.d/start-rabbitmq-arm64.sh +++ b/.docker/base-with-services/s6-assets/init/rabbitmq-init.sh @@ -1,23 +1,21 @@ #!/bin/bash -set -em +RMQ_VERSION="3.10.18" +SYSTEM_USER="aiida" -SYSTEM_USER=${MAMBA_USER} RABBITMQ_DATA_DIR="/home/${SYSTEM_USER}/.rabbitmq" mkdir -p "${RABBITMQ_DATA_DIR}" +fix-permissions "${RABBITMQ_DATA_DIR}" # Fix issue where the erlang cookie permissions are corrupted. chmod 400 "/home/${SYSTEM_USER}/.erlang.cookie" || echo "erlang cookie not created yet." # Set base directory for RabbitMQ to persist its data. This needs to be set to a folder in the system user's home # directory as that is the only folder that is persisted outside of the container. -export RMQ_VERSION=3.9.13 RMQ_ETC_DIR="/opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/etc/rabbitmq" echo MNESIA_BASE="${RABBITMQ_DATA_DIR}" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" echo LOG_BASE="${RABBITMQ_DATA_DIR}/log" >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" -# RabbitMQ with versions >= 3.8.15 have reduced some default timeouts -# baseimage phusion/baseimage:jammy-1.0.0 running ubuntu 22.04 will install higher version of rabbimq by apt. # using workaround from https://github.com/aiidateam/aiida-core/wiki/RabbitMQ-version-to-use # set timeout to 100 hours echo "consumer_timeout=3600000" >> "${RMQ_ETC_DIR}/rabbitmq.conf" @@ -28,5 +26,3 @@ echo "consumer_timeout=3600000" >> "${RMQ_ETC_DIR}/rabbitmq.conf" # reason RabbitMQ is built this way is through this way it allows to run multiple nodes on a single machine each with # isolated mnesia directories. Since in the AiiDA setup we only need and run a single node, we can simply use localhost. echo NODENAME=rabbit@localhost >> "${RMQ_ETC_DIR}/rabbitmq-env.conf" - -micromamba run -n aiida-core-services rabbitmq-server -detached diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/base b/.docker/base-with-services/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/base new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/postgresql b/.docker/base-with-services/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/postgresql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/postgresql-prepare b/.docker/base-with-services/s6-assets/s6-rc.d/aiida-prepare/dependencies.d/postgresql-prepare new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/dependencies.d/base b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/dependencies.d/base new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/timeout-up b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/timeout-up new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/timeout-up @@ -0,0 +1 @@ +0 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/type b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/type new file mode 100644 index 0000000000..bdd22a1850 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/type @@ -0,0 +1 @@ +oneshot diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/up b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/up new file mode 100644 index 0000000000..6fc0f06f57 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-init/up @@ -0,0 +1,6 @@ +#!/command/execlineb -S0 + +with-contenv + +foreground { s6-echo "Calling /etc/init/postgresql-init" } +/etc/init/postgresql-init.sh diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/dependencies.d/base b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/dependencies.d/base new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/dependencies.d/postgresql b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/dependencies.d/postgresql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/timeout-up b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/timeout-up new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/timeout-up @@ -0,0 +1 @@ +0 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/type b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/type new file mode 100644 index 0000000000..bdd22a1850 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/type @@ -0,0 +1 @@ +oneshot diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/up b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/up new file mode 100644 index 0000000000..df5f5f83f9 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql-prepare/up @@ -0,0 +1,6 @@ +#!/command/execlineb -S0 + +with-contenv + +foreground { s6-echo "Calling /etc/init/postgresql-prepare" } +/etc/init/postgresql-prepare.sh diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/dependencies.d/base b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/dependencies.d/base new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/dependencies.d/postgresql-init b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/dependencies.d/postgresql-init new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/down-signal b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/down-signal new file mode 100644 index 0000000000..d751378e19 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/down-signal @@ -0,0 +1 @@ +SIGINT diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/run b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/run new file mode 100644 index 0000000000..5f59ce730e --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/run @@ -0,0 +1,29 @@ +#!/command/execlineb -P + +with-contenv + +# # -w waits until server is up +# echo "DEBUG" +# echo "${SYSTEM_USER}" +# +# PSQL_START_CMD="pg_ctl --timeout=180 -w -D /home/${SYSTEM_USER}/.postgresql -l /home/${SYSTEM_USER}/.postgresql/logfile start" +# PSQL_STATUS_CMD="pg_ctl -D /home/${SYSTEM_USER}/.postgresql status" +# +# MAMBA_RUN="mamba run -n aiida-core-services" +# +# # Fix problem with kubernetes cluster that adds rws permissions to the group +# # for more details see: https://github.com/materialscloud-org/aiidalab-z2jh-eosc/issues/5 +# chmod g-rwxs /home/${SYSTEM_USER}/.postgresql -R +# +# # stores return value in $? +# running=true +# ${MAMBA_RUN} ${PSQL_STATUS_CMD} || running=false +# +# # Postgresql was probably not shutdown properly. Cleaning up the mess... +# if ! $running ; then +# echo "" > /home/${SYSTEM_USER}/.postgresql/logfile # empty log files +# rm -vf /home/${SYSTEM_USER}/.postgresql/postmaster.pid +# ${MAMBA_RUN} ${PSQL_START_CMD} +# fi + +mamba run -n aiida-core-services postgres -D /home/aiida/.postgresql diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/type b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/postgresql/type @@ -0,0 +1 @@ +longrun diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/dependencies.d/base b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/dependencies.d/base new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/timeout-up b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/timeout-up new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/timeout-up @@ -0,0 +1 @@ +0 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/type b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/type new file mode 100644 index 0000000000..bdd22a1850 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/type @@ -0,0 +1 @@ +oneshot diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/up b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/up new file mode 100644 index 0000000000..6fb77e1f87 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/up @@ -0,0 +1,4 @@ +#!/command/execlineb -S0 + +foreground { s6-echo "Calling /etc/init/rabbitmq-init.sh" } +/etc/init/rabbitmq-init.sh diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/dependencies.d/base b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/dependencies.d/base new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/dependencies.d/rabbitmq-init b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/dependencies.d/rabbitmq-init new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/down-signal b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/down-signal new file mode 100644 index 0000000000..d751378e19 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/down-signal @@ -0,0 +1 @@ +SIGINT diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/run b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/run new file mode 100644 index 0000000000..e7e1de189f --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/run @@ -0,0 +1,6 @@ +#!/command/execlineb -P + +with-contenv + +foreground { s6-echo "Calling /etc/init/rabbitmq.sh" } +mamba run -n aiida-core-services rabbitmq-server diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/type b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq/type @@ -0,0 +1 @@ +longrun diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/postgresql b/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/postgresql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/postgresql-init b/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/postgresql-init new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/postgresql-prepare b/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/postgresql-prepare new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/rabbitmq b/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/rabbitmq new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/rabbitmq-init b/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/rabbitmq-init new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/verdi-rabbitmq-warning-suppress b/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/verdi-rabbitmq-warning-suppress new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/dependencies.d/aiida-prepare b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/dependencies.d/aiida-prepare new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/dependencies.d/base b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/dependencies.d/base new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/timeout-up b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/timeout-up new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/timeout-up @@ -0,0 +1 @@ +0 diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/type b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/type new file mode 100644 index 0000000000..bdd22a1850 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/type @@ -0,0 +1 @@ +oneshot diff --git a/.docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/up similarity index 91% rename from .docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh rename to .docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/up index f778bbb5db..ce522d341e 100644 --- a/.docker/base-with-services/init.d/suppress-rabbitmq-version-warning.sh +++ b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/up @@ -1,5 +1,4 @@ -#!/bin/bash -set -em +#!/command/execlineb -P # Supress rabbitmq version warning for arm64 since # it is built using latest version rabbitmq from apt install. diff --git a/.docker/base/Dockerfile b/.docker/base/Dockerfile index f73a56f195..da15a7d2c3 100644 --- a/.docker/base/Dockerfile +++ b/.docker/base/Dockerfile @@ -20,7 +20,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root - +ENV SYSTEM_USER="${SYSTEM_USER}" # Install all OS dependencies for notebook server that starts but lacks all # features (e.g., download as all possible file formats) @@ -138,7 +138,7 @@ RUN set -x && \ # Switch to root to install AiiDA and set AiiDA as service USER root -COPY --from=src . /tmp/aiida-core +# COPY --from=src . /tmp/aiida-core # RUN pip install /tmp/aiida-core --no-cache-dir && \ # rm -rf /tmp/aiida-core RUN pip install aiida-core==2.4.0 @@ -155,8 +155,8 @@ COPY s6-assets/s6-rc.d /etc/s6-overlay/s6-rc.d COPY s6-assets/init /etc/init # Switch back to USER aiida to avoid accidental container runs as root -USER ${SYSTEM_UID} +# USER ${SYSTEM_UID} ENTRYPOINT ["/init"] -WORKDIR "${HOME}" \ No newline at end of file +WORKDIR "${HOME}" diff --git a/.docker/base/fix-permissions b/.docker/base/fix-permissions index ad8a1cc839..840173c605 100644 --- a/.docker/base/fix-permissions +++ b/.docker/base/fix-permissions @@ -32,4 +32,4 @@ for d in "$@"; do -a ! -perm -6000 \ \) \ -exec chmod +6000 -- {} \+ -done \ No newline at end of file +done diff --git a/.docker/base/init.d/aiida-profile-preparation.sh b/.docker/base/init.d/aiida-profile-preparation.sh deleted file mode 100755 index 05dde219a0..0000000000 --- a/.docker/base/init.d/aiida-profile-preparation.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -# This script is executed whenever the docker container is (re)started. - -# Debugging. -set -x - -# Environment. -export SHELL=/bin/bash - -# Check if user requested to set up AiiDA profile (and if it exists already) -if [[ ${SETUP_DEFAULT_AIIDA_PROFILE} == true ]] && ! verdi profile show ${AIIDA_PROFILE_NAME} &> /dev/null; then - NEED_SETUP_PROFILE=true; -else - NEED_SETUP_PROFILE=false; -fi - -# Setup AiiDA profile if needed. -if [[ ${NEED_SETUP_PROFILE} == true ]]; then - - # Create AiiDA profile. - verdi quicksetup \ - --non-interactive \ - --profile "${AIIDA_PROFILE_NAME}" \ - --email "${AIIDA_USER_EMAIL}" \ - --first-name "${AIIDA_USER_FIRST_NAME}" \ - --last-name "${AIIDA_USER_LAST_NAME}" \ - --institution "${AIIDA_USER_INSTITUTION}" \ - --config /opt/config-quick-setup.yaml - - # Setup and configure local computer. - computer_name=localhost - - # Determine the number of physical cores as a default for the number of - # available MPI ranks on the localhost. We do not count "logical" cores, - # since MPI parallelization over hyper-threaded cores is typically - # associated with a significant performance penalty. We use the - # `psutil.cpu_count(logical=False)` function as opposed to simply - # `os.cpu_count()` since the latter would include hyperthreaded (logical - # cores). - NUM_PHYSICAL_CORES=$(python -c 'import psutil; print(int(psutil.cpu_count(logical=False)))' 2>/dev/null) - LOCALHOST_MPI_PROCS_PER_MACHINE=${LOCALHOST_MPI_PROCS_PER_MACHINE:-${NUM_PHYSICAL_CORES}} - - if [ -z $LOCALHOST_MPI_PROCS_PER_MACHINE ]; then - echo "Unable to automatically determine the number of logical CPUs on this " - echo "machine. Please set the LOCALHOST_MPI_PROCS_PER_MACHINE variable to " - echo "explicitly set the number of available MPI ranks." - exit 1 - fi - - verdi computer show ${computer_name} || verdi computer setup \ - --non-interactive \ - --label "${computer_name}" \ - --description "this computer" \ - --hostname "${computer_name}" \ - --transport core.local \ - --scheduler core.direct \ - --work-dir /home/${MAMBA_USER}/aiida_run/ \ - --mpirun-command "mpirun -np {tot_num_mpiprocs}" \ - --mpiprocs-per-machine ${LOCALHOST_MPI_PROCS_PER_MACHINE} && \ - verdi computer configure core.local "${computer_name}" \ - --non-interactive \ - --safe-interval 0.0 -fi - - -# Show the default profile -verdi profile show || echo "The default profile is not set." - -# Make sure that the daemon is not running, otherwise the migration will abort. -verdi daemon stop - -# Migration will run for the default profile. -verdi storage migrate --force - -# Daemon will start only if the database exists and is migrated to the latest version. -verdi daemon start || echo "AiiDA daemon is not running." diff --git a/.docker/base/initial-condarc b/.docker/base/initial-condarc index 66ecf60813..383aad3cb0 100644 --- a/.docker/base/initial-condarc +++ b/.docker/base/initial-condarc @@ -3,4 +3,4 @@ auto_update_conda: false show_channel_urls: true channels: - - conda-forge \ No newline at end of file + - conda-forge diff --git a/.docker/base/s6-assets/init/aiida-prepare b/.docker/base/s6-assets/init/aiida-prepare.sh similarity index 93% rename from .docker/base/s6-assets/init/aiida-prepare rename to .docker/base/s6-assets/init/aiida-prepare.sh index e370c181e9..766706e03a 100755 --- a/.docker/base/s6-assets/init/aiida-prepare +++ b/.docker/base/s6-assets/init/aiida-prepare.sh @@ -73,11 +73,5 @@ fi # Show the default profile verdi profile show || echo "The default profile is not set." -# Make sure that the daemon is not running, otherwise the migration will abort. -verdi daemon stop - # Migration will run for the default profile. verdi storage migrate --force - -# Daemon will start only if the database exists and is migrated to the latest version. -verdi daemon start || echo "AiiDA daemon is not running." diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/dependencies.d/aiida-prepare b/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/dependencies.d/aiida-prepare new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/dependencies.d/base b/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/dependencies.d/base new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/down b/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/down new file mode 100644 index 0000000000..b8a14495ad --- /dev/null +++ b/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/down @@ -0,0 +1 @@ +verdi daemon stop diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/timeout-up b/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/timeout-up new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/timeout-up @@ -0,0 +1 @@ +0 diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/type b/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/type new file mode 100644 index 0000000000..bdd22a1850 --- /dev/null +++ b/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/type @@ -0,0 +1 @@ +oneshot diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/up b/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/up new file mode 100644 index 0000000000..12f199a2b3 --- /dev/null +++ b/.docker/base/s6-assets/s6-rc.d/aiida-daemon-start/up @@ -0,0 +1,3 @@ +#!/command/execlineb -S0 + +verdi daemon start diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-prepare/timeout-up b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/timeout-up index c227083464..573541ac97 100644 --- a/.docker/base/s6-assets/s6-rc.d/aiida-prepare/timeout-up +++ b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/timeout-up @@ -1 +1 @@ -0 \ No newline at end of file +0 diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-prepare/type b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/type index 3d92b15f2d..bdd22a1850 100644 --- a/.docker/base/s6-assets/s6-rc.d/aiida-prepare/type +++ b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/type @@ -1 +1 @@ -oneshot \ No newline at end of file +oneshot diff --git a/.docker/base/s6-assets/s6-rc.d/aiida-prepare/up b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/up index 703069c963..60e82d7e43 100644 --- a/.docker/base/s6-assets/s6-rc.d/aiida-prepare/up +++ b/.docker/base/s6-assets/s6-rc.d/aiida-prepare/up @@ -1,4 +1,4 @@ #!/command/execlineb -S0 foreground { s6-echo "Calling /etc/init/aiida-prepare" } -/etc/init/aiida-prepare \ No newline at end of file +/etc/init/aiida-prepare.sh diff --git a/.docker/base/s6-assets/s6-rc.d/user/contents.d/aiida-daemon-start b/.docker/base/s6-assets/s6-rc.d/user/contents.d/aiida-daemon-start new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.docker/build.json b/.docker/build.json index 3bfe39ce05..a274737600 100644 --- a/.docker/build.json +++ b/.docker/build.json @@ -5,7 +5,10 @@ }, "PGSQL_VERSION": { "default": "15" + }, + "RMQ_VERSION": { + "default": "3.10.18" } } } - \ No newline at end of file + diff --git a/.docker/docker-bake.hcl b/.docker/docker-bake.hcl index cb65d4e683..a7d3f066a2 100644 --- a/.docker/docker-bake.hcl +++ b/.docker/docker-bake.hcl @@ -63,5 +63,6 @@ target "base-with-services" { platforms = "${PLATFORMS}" args = { "PGSQL_VERSION" = "${PGSQL_VERSION}" + "RMQ_VERSION" = "${RMQ_VERSION}" } } diff --git a/.docker/docker-compose.base-with-services.yml b/.docker/docker-compose.base-with-services.yml new file mode 100644 index 0000000000..eb6c8fa225 --- /dev/null +++ b/.docker/docker-compose.base-with-services.yml @@ -0,0 +1,15 @@ +--- +version: '3.4' + +services: + + aiida: + image: ${REGISTRY:-}${BASE_IMAGE:-aiidateam/aiida-core}:${VERSION:-latest} + environment: + TZ: Europe/Zurich + SETUP_DEFAULT_AIIDA_PROFILE: 'true' + # volumes: + # - aiida-home-folder:/home/aiida + +volumes: + aiida-home-folder: diff --git a/.docker/docker-compose.base.yml b/.docker/docker-compose.base.yml index 8487313b6d..ce5c59100f 100644 --- a/.docker/docker-compose.base.yml +++ b/.docker/docker-compose.base.yml @@ -11,6 +11,7 @@ services: volumes: - aiida-postgres-db:/var/lib/postgresql/data healthcheck: + test: [ "CMD-SHELL", "pg_isready"] interval: 5s timeout: 5s retries: 10 @@ -22,6 +23,11 @@ services: RABBITMQ_DEFAULT_PASS: guest volumes: - aiida-rmq-data:/var/lib/rabbitmq/ + healthcheck: + test: rabbitmq-diagnostics check_port_connectivity + interval: 30s + timeout: 30s + retries: 10 aiida: image: ${REGISTRY:-}${BASE_IMAGE:-aiidateam/base}:${VERSION:-latest} @@ -33,9 +39,9 @@ services: - aiida-home-folder:/home/aiida depends_on: database: - condition: service_started + condition: service_healthy messaging: - condition: service_started + condition: service_healthy command: tail -f /dev/null volumes: diff --git a/.docker/dodo.py b/.docker/dodo.py index 333ec4bdf4..dcd1dd718d 100644 --- a/.docker/dodo.py +++ b/.docker/dodo.py @@ -1,83 +1,79 @@ +# -*- coding: utf-8 -*- import json -import platform from pathlib import Path - -from doit.tools import title_with_actions +import platform import docker +from doit.tools import title_with_actions from dunamai import Version _DOCKER_CLIENT = docker.from_env() -_DOCKER_ARCHITECTURE = _DOCKER_CLIENT.info()["Architecture"] +_DOCKER_ARCHITECTURE = _DOCKER_CLIENT.info()['Architecture'] -DOIT_CONFIG = {"default_tasks": ["build"]} +DOIT_CONFIG = {'default_tasks': ['build']} -VERSION = Version.from_git().serialize().replace("+", "_") +VERSION = Version.from_git().serialize().replace('+', '_') _ARCH_MAPPING = { - "x86_64": "amd64", - "aarch64": "arm64", + 'x86_64': 'amd64', + 'aarch64': 'arm64', } ARCH = _ARCH_MAPPING.get(_DOCKER_ARCHITECTURE) if ARCH is None: - raise RuntimeError( - f"Unsupported architecture {ARCH} on platform {platform.system()}." - ) + raise RuntimeError(f'Unsupported architecture {ARCH} on platform {platform.system()}.') _REGISTRY_PARAM = { - "name": "registry", - "short": "r", - "long": "registry", - "type": str, - "default": "", - "help": "Specify the docker image registry.", + 'name': 'registry', + 'short': 'r', + 'long': 'registry', + 'type': str, + 'default': '', + 'help': 'Specify the docker image registry.', } _ORGANIZATION_PARAM = { - "name": "organization", - "short": "o", - "long": "organization", - "type": str, - "default": "aiidalab", - "help": "Specify the docker image organization.", + 'name': 'organization', + 'short': 'o', + 'long': 'organization', + 'type': str, + 'default': 'aiidalab', + 'help': 'Specify the docker image organization.', } _VERSION_PARAM = { - "name": "version", - "long": "version", - "type": str, - "default": VERSION, - "help": ( - "Specify the version of the stack for building / testing. Defaults to a " - "version determined from the state of the local git repository." + 'name': 'version', + 'long': 'version', + 'type': str, + 'default': VERSION, + 'help': ( + 'Specify the version of the stack for building / testing. Defaults to a ' + 'version determined from the state of the local git repository.' ), } _ARCH_PARAM = { - "name": "architecture", - "long": "arch", - "type": str, - "default": ARCH, - "help": "Specify the platform to build for. Examples: arm64, amd64.", + 'name': 'architecture', + 'long': 'arch', + 'type': str, + 'default': ARCH, + 'help': 'Specify the platform to build for. Examples: arm64, amd64.', } def task_build(): """Build all docker images.""" - def generate_version_override( - version, registry, targets, architecture, organization - ): + def generate_version_override(version, registry, targets, architecture, organization): if len(targets) > 2: # Workaround of issue of doit, which rather than override the default value, it appends # https://github.com/pydoit/doit/issues/436 targets = targets[2:] - platforms = [f"linux/{architecture}"] + platforms = [f'linux/{architecture}'] - Path("docker-bake.override.json").write_text( + Path('docker-bake.override.json').write_text( json.dumps( dict( VERSION=version, @@ -90,28 +86,28 @@ def generate_version_override( ) return { - "actions": [ + 'actions': [ generate_version_override, - "docker buildx bake -f docker-bake.hcl -f build.json " - "-f docker-bake.override.json " - "--load", + 'docker buildx bake -f docker-bake.hcl -f build.json ' + '-f docker-bake.override.json ' + '--load', ], - "title": title_with_actions, - "params": [ + 'title': title_with_actions, + 'params': [ _ORGANIZATION_PARAM, _REGISTRY_PARAM, _VERSION_PARAM, _ARCH_PARAM, { - "name": "targets", - "long": "targets", - "short": "t", - "type": list, - "default": ["base", "base-with-services"], - "help": "Specify the target to build.", + 'name': 'targets', + 'long': 'targets', + 'short': 't', + 'type': list, + 'default': ['base', 'base-with-services'], + 'help': 'Specify the target to build.', }, ], - "verbosity": 2, + 'verbosity': 2, } @@ -119,43 +115,43 @@ def task_tests(): """Run tests with pytest.""" return { - "actions": ["REGISTRY=%(registry)s VERSION=:%(version)s pytest -v"], - "params": [_REGISTRY_PARAM, _VERSION_PARAM], - "verbosity": 2, + 'actions': ['REGISTRY=%(registry)s VERSION=:%(version)s pytest -v'], + 'params': [_REGISTRY_PARAM, _VERSION_PARAM], + 'verbosity': 2, } def task_up(): """Start AiiDAlab server for testing.""" return { - "actions": [ - "AIIDALAB_PORT=%(port)i REGISTRY=%(registry)s VERSION=%(version)s " - "docker-compose -f docker-compose.%(target)s.yml up" + 'actions': [ + 'AIIDALAB_PORT=%(port)i REGISTRY=%(registry)s VERSION=%(version)s ' + 'docker-compose -f docker-compose.%(target)s.yml up' ], - "params": [ + 'params': [ { - "name": "port", - "short": "p", - "long": "port", - "type": int, - "default": 8888, - "help": "Specify the AiiDAlab host port.", + 'name': 'port', + 'short': 'p', + 'long': 'port', + 'type': int, + 'default': 8888, + 'help': 'Specify the AiiDAlab host port.', }, { - "name": "target", - "short": "t", - "long": "target", - "type": str, - "default": "base-with-services", - "help": "Specify the target to run.", + 'name': 'target', + 'short': 't', + 'long': 'target', + 'type': str, + 'default': 'base-with-services', + 'help': 'Specify the target to run.', }, _REGISTRY_PARAM, _VERSION_PARAM, ], - "verbosity": 2, + 'verbosity': 2, } def task_down(): - """Stop AiiDAlab server.""" - return {"actions": ["docker-compose down"], "verbosity": 2} \ No newline at end of file + """Stop AiiDAlab server.""" + return {'actions': ['docker-compose down'], 'verbosity': 2} diff --git a/.docker/requirements.txt b/.docker/requirements.txt index 6763b51984..db18601f89 100644 --- a/.docker/requirements.txt +++ b/.docker/requirements.txt @@ -7,4 +7,4 @@ tabulate pytest-docker docker-compose doit -dunamai \ No newline at end of file +dunamai diff --git a/.gitignore b/.gitignore index d41253317a..ef7191ae15 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,4 @@ pplot_out/ # docker .doit.* -docker-bake.override.json \ No newline at end of file +docker-bake.override.json From e14740df2f2792ad0555da9828a3b4264b352e94 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Fri, 7 Jul 2023 17:17:12 +0200 Subject: [PATCH 11/19] WIP: CI --- .github/actions/create-dev-env/action.yml | 27 ++++ .github/actions/load-image/action.yml | 31 +++++ .../workflows/build_and_test_docker_on_pr.yml | 65 ---------- .../workflows/docker-build-test-upload.yml | 54 ++++++++ .github/workflows/docker-merge-tags.yml | 65 ++++++++++ .github/workflows/docker-push.yml | 92 ++++++++++++++ .github/workflows/docker.yml | 115 ++++++++++++++++++ .github/workflows/push_image_to_dockerhub.yml | 54 -------- 8 files changed, 384 insertions(+), 119 deletions(-) create mode 100644 .github/actions/create-dev-env/action.yml create mode 100644 .github/actions/load-image/action.yml delete mode 100644 .github/workflows/build_and_test_docker_on_pr.yml create mode 100644 .github/workflows/docker-build-test-upload.yml create mode 100644 .github/workflows/docker-merge-tags.yml create mode 100644 .github/workflows/docker-push.yml create mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/push_image_to_dockerhub.yml diff --git a/.github/actions/create-dev-env/action.yml b/.github/actions/create-dev-env/action.yml new file mode 100644 index 0000000000..1142844757 --- /dev/null +++ b/.github/actions/create-dev-env/action.yml @@ -0,0 +1,27 @@ +--- +name: Build environment +description: Create build environment + +inputs: + architecture: + description: architecture to be run on + required: true + type: string + +runs: + using: composite + steps: + # actions/setup-python doesn't support Linux arm64 runners + # See: https://github.com/actions/setup-python/issues/108 + # python3 is manually preinstalled in the arm64 VM self-hosted runner + - name: Set Up Python ๐Ÿ + uses: actions/setup-python@v4 + with: + python-version: 3.x + if: ${{ inputs.architecture == 'amd64' }} + + - name: Install Dev Dependencies ๐Ÿ“ฆ + run: | + pip install --upgrade pip + pip install --upgrade -r .docker/requirements.txt + shell: bash diff --git a/.github/actions/load-image/action.yml b/.github/actions/load-image/action.yml new file mode 100644 index 0000000000..9384089e5f --- /dev/null +++ b/.github/actions/load-image/action.yml @@ -0,0 +1,31 @@ +--- +name: Load Docker image +description: Download image tar and load it to docker + +inputs: + image: + description: Image name + required: true + type: string + architecture: + description: Image architecture + required: true + type: string + +runs: + using: composite + steps: + - name: Download built image ๐Ÿ“ฅ + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.image }}-${{ inputs.architecture }} + path: /tmp/ + - name: Load downloaded image to docker ๐Ÿ“ฅ + run: | + docker load --input /tmp/${{ inputs.image }}-${{ inputs.architecture }}.tar + docker image ls --all + shell: bash + - name: Delete the file ๐Ÿ—‘๏ธ + run: rm -f /tmp/${{ inputs.image }}-${{ inputs.architecture }}.tar + shell: bash + if: always() \ No newline at end of file diff --git a/.github/workflows/build_and_test_docker_on_pr.yml b/.github/workflows/build_and_test_docker_on_pr.yml deleted file mode 100644 index 9078daefc9..0000000000 --- a/.github/workflows/build_and_test_docker_on_pr.yml +++ /dev/null @@ -1,65 +0,0 @@ -# Test the Docker image on every pull request. -# -# The steps are: -# 1. Build docker image using cached data. -# 2. Start the docker container. -# 3. Check that AiiDA is responsive. - -name: build-and-test-image-from-pull-request - -on: - pull_request: - path_ignore: - - 'docs/**' - -# https://docs.github.com/en/actions/using-jobs/using-concurrency -concurrency: - # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - - build-and-test: - - # Only run this job on the main repository and not on forks - if: github.repository == 'aiidateam/aiida-core' - - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - - uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Build image locally - uses: docker/build-push-action@v2 - with: - load: true - push: false - tags: aiida-core:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Start and test the container - run: | - export DOCKERID=`docker run -d aiida-core:latest` - docker exec --tty $DOCKERID wait-for-services - docker logs $DOCKERID - docker exec --tty --user aiida $DOCKERID /bin/bash -l -c 'verdi profile show default' - docker exec --tty --user aiida $DOCKERID /bin/bash -l -c 'verdi computer show localhost' - docker exec --tty --user aiida $DOCKERID /bin/bash -l -c 'verdi daemon status' diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml new file mode 100644 index 0000000000..feaac10cc6 --- /dev/null +++ b/.github/workflows/docker-build-test-upload.yml @@ -0,0 +1,54 @@ +--- +name: Build image and then upload the image, tags and manifests to GitHub artifacts + +env: + OWNER: ${{ github.repository_owner }} + +on: + workflow_call: + inputs: + architecture: + description: Image architecture, e.g. amd64, arm64 + required: true + type: string + runsOn: + description: GitHub Actions Runner image + required: true + type: string + +jobs: + build-test-upload: + runs-on: ${{ inputs.runsOn }} + strategy: + matrix: + image: ["base", "aiida-core"] + + steps: + - name: Checkout Repo โšก๏ธ + uses: actions/checkout@v3 + - name: Create dev environment ๐Ÿ“ฆ + uses: ./.github/actions/create-dev-env + with: + architecture: ${{ inputs.architecture }} + + - name: Build image ๐Ÿ›  + run: doit build --target ${{ matrix.image }} --arch ${{ inputs.architecture }} --organization ${{ env.OWNER }} + env: + # Full logs for CI build + BUILDKIT_PROGRESS: plain + shell: bash + + - name: Run tests โœ… + run: VERSION=newly-build pytest -s tests/test-common.py tests/test-${{ matrix.image }}.py --variant ${{ matrix.image }} + shell: bash + + - name: Save image as a tar for later use ๐Ÿ’พ + run: docker save ${{ env.OWNER }}/${{ matrix.image }} -o /tmp/${{ matrix.image }}-${{ inputs.architecture }}.tar + shell: bash + + - name: Upload image as artifact ๐Ÿ’พ + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.image }}-${{ inputs.architecture }} + path: /tmp/${{ matrix.image }}-${{ inputs.architecture }}.tar + retention-days: 3 diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/docker-merge-tags.yml new file mode 100644 index 0000000000..781c755496 --- /dev/null +++ b/.github/workflows/docker-merge-tags.yml @@ -0,0 +1,65 @@ +--- +name: Download images tags from GitHub artifacts and create multi-platform manifests + +on: + workflow_call: + inputs: + image: + description: Image name + required: true + type: string + registry: + description: Docker registry, e.g. ghcr.io, docker.io + required: true + type: string + secrets: + REGISTRY_USERNAME: + required: true + REGISTRY_TOKEN: + required: true + + +jobs: + merge-tags: + runs-on: ubuntu-latest + + steps: + - name: Checkout Repo โšก๏ธ + uses: actions/checkout@v3 + - name: Create dev environment ๐Ÿ“ฆ + uses: ./.github/actions/create-dev-env + with: + architecture: amd64 + + - name: Download amd64 tags file ๐Ÿ“ฅ + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.registry }}-${{ inputs.image }}-amd64-tags + path: /tmp/ + - name: Download arm64 tags file ๐Ÿ“ฅ + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.registry }}-${{ inputs.image }}-arm64-tags + path: /tmp/ + + - name: Login to Container Registry ๐Ÿ”‘ + uses: docker/login-action@v2 + with: + registry: ${{ inputs.registry }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Merge tags for the images of different arch ๐Ÿ”€ + run: | + for arch_tag in $(cat /tmp/${{ inputs.image }}-amd64-tags.txt); do + tag=$(echo $arch_tag | sed "s/:amd64-/:/") + docker manifest create $tag --amend $arch_tag + docker manifest push $tag + done + + for arch_tag in $(cat /tmp/${{ inputs.image }}-arm64-tags.txt); do + tag=$(echo $arch_tag | sed "s/:arm64-/:/") + docker manifest create $tag --amend $arch_tag + docker manifest push $tag + done + shell: bash \ No newline at end of file diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml new file mode 100644 index 0000000000..358abd3aa9 --- /dev/null +++ b/.github/workflows/docker-push.yml @@ -0,0 +1,92 @@ +--- +name: Download Docker image and its tags from GitHub artifacts, apply them and push the image to container registry + +env: + OWNER: ${{ github.repository_owner }} + +on: + workflow_call: + inputs: + architecture: + description: Image architecture + required: true + type: string + registry: + description: Docker registry + required: true + type: string + secrets: + REGISTRY_USERNAME: + required: true + REGISTRY_TOKEN: + required: true + +jobs: + tag-push: + runs-on: ubuntu-latest + strategy: + matrix: + image: ["base", "aiida-core"] + + steps: + - name: Checkout Repo โšก๏ธ + uses: actions/checkout@v3 + - name: Create dev environment ๐Ÿ“ฆ + uses: ./.github/actions/create-dev-env + with: + architecture: ${{ inputs.architecture }} + - name: Load image to Docker ๐Ÿ“ฅ + uses: ./.github/actions/load-image + with: + image: ${{ matrix.image }} + architecture: ${{ inputs.architecture }} + + - name: Read build variables + id: build_vars + run: | + vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries') + echo "vars=$vars" >> "${GITHUB_OUTPUT}" + + - name: Docker meta ๐Ÿ“ + id: meta + uses: docker/metadata-action@v4 + env: ${{ fromJson(steps.build_vars.outputs.vars) }} + with: + images: | + name=${{ inputs.registry }}/${{ env.OWNER }}/${{ matrix.image }} + tags: | + type=edge,enable={{is_default_branch}} + type=sha,enable=${{ github.ref_type != 'tag' }} + type=ref,event=pr + type=match,pattern=v(\d{4}\.\d{4}),group=1 + type=raw,value={{tag}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=raw,value=aiida-${{ env.AIIDA_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + + - name: Login to Container Registry ๐Ÿ”‘ + uses: docker/login-action@v2 + with: + registry: ${{ inputs.registry }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Set tags for image and push ๐Ÿท๏ธ๐Ÿ“ค๐Ÿ’พ + run: | + declare -a arr=(${{ steps.meta.outputs.tags }}) + for tag in "${arr[@]}"; do + arch_tag=$(echo ${tag} | sed "s/:/:${{ inputs.architecture }}-/") + docker tag aiidalab/${{ matrix.image }}:newly-build ${arch_tag} + docker push ${arch_tag} + + # write tag to file + echo ${arch_tag} >> /tmp/${{ matrix.image }}-${{ inputs.architecture }}-tags.txt + done + shell: bash + + - name: Upload tags file ๐Ÿ“ค + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.registry }}-${{ matrix.image }}-${{ inputs.architecture }}-tags + path: /tmp/${{ matrix.image }}-${{ inputs.architecture }}-tags.txt + retention-days: 3 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000000..4be6fb0950 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,115 @@ +--- +name: Build, test and push Docker Images + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + amd64-build: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + architecture: amd64 + runsOn: ubuntu-latest + + arm64-build: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + architecture: arm64 + runsOn: buildjet-2vcpu-ubuntu-2204-arm + + #amd64-push-ghcr: + # uses: ./.github/workflows/docker-push.yml + # strategy: + # matrix: + # image: ["base", "aiida-core"] + # with: + # architecture: amd64 + # image: ${{ matrix.image }} + # registry: ghcr.io + # secrets: + # REGISTRY_USERNAME: ${{ github.actor }} + # REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # needs: [amd64-build] + + #arm64-push-ghcr: + # uses: ./.github/workflows/docker-push.yml + # strategy: + # matrix: + # image: ["base", "aiida-core"] + # with: + # architecture: arm64 + # image: ${{ matrix.image }} + # registry: ghcr.io + # secrets: + # REGISTRY_USERNAME: ${{ github.actor }} + # REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # needs: [arm64-build] + + #merge-tags-ghcr: + # uses: ./.github/workflows/docker-merge-tags.yml + # strategy: + # matrix: + # image: ["base", "aiida-core"] + # with: + # image: ${{ matrix.image }} + # registry: ghcr.io + # secrets: + # REGISTRY_USERNAME: ${{ github.actor }} + # REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # needs: [amd64-push-ghcr, arm64-push-ghcr] + + #amd64-push-dockerhub: + # if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + # uses: ./.github/workflows/docker-push.yml + # strategy: + # matrix: + # image: ["base", "aiida-core"] + # with: + # architecture: amd64 + # image: ${{ matrix.image }} + # registry: docker.io + # secrets: + # REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + # REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + # needs: [amd64-build] + + #arm64-push-dockerhub: + # if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + # uses: ./.github/workflows/docker-push.yml + # strategy: + # matrix: + # image: ["base", "aiida-core"] + # with: + # architecture: arm64 + # image: ${{ matrix.image }} + # registry: docker.io + # secrets: + # REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + # REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + # needs: [arm64-build] + + #merge-tags-dockerhub: + # if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + # uses: ./.github/workflows/docker-merge-tags.yml + # strategy: + # matrix: + # image: ["base", "aiida-core"] + # with: + # image: ${{ matrix.image }} + # registry: docker.io + # secrets: + # REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + # REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + # needs: [amd64-push-dockerhub, arm64-push-dockerhub] + + \ No newline at end of file diff --git a/.github/workflows/push_image_to_dockerhub.yml b/.github/workflows/push_image_to_dockerhub.yml deleted file mode 100644 index 3178e78e04..0000000000 --- a/.github/workflows/push_image_to_dockerhub.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Build the new Docker image on every commit to the main branch and on every new tag. -# No caching is involved for the image build. The new image is then pushed to the Docker Hub. - -name: build-and-push-to-dockerhub - -on: - push: - branches: - - main - tags: - - "v[0-9]+.[0-9]+.[0-9]+*" - -jobs: - - build-and-push: - - # Only run this job on the main repository and not on forks - if: github.repository == 'aiidateam/aiida-core' - - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - - uses: actions/checkout@v2 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v3 - with: - images: ${{ github.repository }} - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - platforms: linux/amd64, linux/arm64 - tags: ${{ steps.meta.outputs.tags }} From 9cf716de4c4c165ab5d8e0af8004b2e4d8ef3de1 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Sat, 8 Jul 2023 00:25:44 +0200 Subject: [PATCH 12/19] tset pass locally --- .docker/base-with-services/Dockerfile | 4 -- .docker/base/Dockerfile | 14 ++-- .docker/docker-compose.base-with-services.yml | 4 +- .docker/docker-compose.base.yml | 25 ++++--- .docker/pytest.ini | 5 ++ .docker/requirements.txt | 1 - .docker/tests/conftest.py | 59 +++++++++++++++++ .docker/tests/test_aiida.py | 28 ++++++++ .../workflows/docker-build-test-upload.yml | 33 ++++++---- .github/workflows/docker-merge-tags.yml | 17 +++-- .github/workflows/docker-push.yml | 2 +- .github/workflows/docker.yml | 66 ++++++++----------- 12 files changed, 170 insertions(+), 88 deletions(-) create mode 100644 .docker/pytest.ini create mode 100644 .docker/tests/conftest.py create mode 100644 .docker/tests/test_aiida.py diff --git a/.docker/base-with-services/Dockerfile b/.docker/base-with-services/Dockerfile index f238ffefdc..0b7e2af6a1 100644 --- a/.docker/base-with-services/Dockerfile +++ b/.docker/base-with-services/Dockerfile @@ -38,10 +38,6 @@ COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" s6-assets/config-quick-setup.yaml "/a COPY s6-assets/s6-rc.d /etc/s6-overlay/s6-rc.d COPY s6-assets/init /etc/init -# Otherwise will stuck on oneshot services -# https://github.com/just-containers/s6-overlay/issues/467 -ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0 - USER ${SYSTEM_UID} WORKDIR "/home/${SYSTEM_USER}" diff --git a/.docker/base/Dockerfile b/.docker/base/Dockerfile index da15a7d2c3..904bd18de3 100644 --- a/.docker/base/Dockerfile +++ b/.docker/base/Dockerfile @@ -137,11 +137,11 @@ RUN set -x && \ fix-permissions "/home/${SYSTEM_USER}" # Switch to root to install AiiDA and set AiiDA as service +# Install AiiDA from source code USER root -# COPY --from=src . /tmp/aiida-core -# RUN pip install /tmp/aiida-core --no-cache-dir && \ -# rm -rf /tmp/aiida-core -RUN pip install aiida-core==2.4.0 +COPY --from=src . /tmp/aiida-core +RUN pip install /tmp/aiida-core --no-cache-dir && \ + rm -rf /tmp/aiida-core # Enable verdi autocompletion. RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \ @@ -154,8 +154,12 @@ COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" s6-assets/config-quick-setup.yaml "/a COPY s6-assets/s6-rc.d /etc/s6-overlay/s6-rc.d COPY s6-assets/init /etc/init +# Otherwise will stuck on oneshot services +# https://github.com/just-containers/s6-overlay/issues/467 +ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0 + # Switch back to USER aiida to avoid accidental container runs as root -# USER ${SYSTEM_UID} +USER ${SYSTEM_UID} ENTRYPOINT ["/init"] diff --git a/.docker/docker-compose.base-with-services.yml b/.docker/docker-compose.base-with-services.yml index eb6c8fa225..cf78f252be 100644 --- a/.docker/docker-compose.base-with-services.yml +++ b/.docker/docker-compose.base-with-services.yml @@ -8,8 +8,8 @@ services: environment: TZ: Europe/Zurich SETUP_DEFAULT_AIIDA_PROFILE: 'true' - # volumes: - # - aiida-home-folder:/home/aiida + #volumes: + # - aiida-home-folder:/home/aiida volumes: aiida-home-folder: diff --git a/.docker/docker-compose.base.yml b/.docker/docker-compose.base.yml index ce5c59100f..a6a8d48971 100644 --- a/.docker/docker-compose.base.yml +++ b/.docker/docker-compose.base.yml @@ -8,8 +8,8 @@ services: environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: password - volumes: - - aiida-postgres-db:/var/lib/postgresql/data + # volumes: + # - aiida-postgres-db:/var/lib/postgresql/data healthcheck: test: [ "CMD-SHELL", "pg_isready"] interval: 5s @@ -21,8 +21,8 @@ services: environment: RABBITMQ_DEFAULT_USER: guest RABBITMQ_DEFAULT_PASS: guest - volumes: - - aiida-rmq-data:/var/lib/rabbitmq/ + # volumes: + # - aiida-rmq-data:/var/lib/rabbitmq/ healthcheck: test: rabbitmq-diagnostics check_port_connectivity interval: 30s @@ -35,16 +35,15 @@ services: RMQHOST: messaging TZ: Europe/Zurich SETUP_DEFAULT_AIIDA_PROFILE: 'true' - volumes: - - aiida-home-folder:/home/aiida + # volumes: + # - aiida-home-folder:/home/aiida depends_on: database: condition: service_healthy - messaging: - condition: service_healthy - command: tail -f /dev/null + #messaging: + # condition: service_healthy -volumes: - aiida-postgres-db: - aiida-rmq-data: - aiida-home-folder: +#volumes: +# aiida-postgres-db: +# aiida-rmq-data: +# aiida-home-folder: diff --git a/.docker/pytest.ini b/.docker/pytest.ini new file mode 100644 index 0000000000..d1e7877377 --- /dev/null +++ b/.docker/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +minversion = 7.0 +addopts = -ra -q +testpaths = + tests diff --git a/.docker/requirements.txt b/.docker/requirements.txt index db18601f89..a97ab8ec14 100644 --- a/.docker/requirements.txt +++ b/.docker/requirements.txt @@ -1,7 +1,6 @@ docker pre-commit pytest -pytest-xdist requests tabulate pytest-docker diff --git a/.docker/tests/conftest.py b/.docker/tests/conftest.py new file mode 100644 index 0000000000..77cb6961cc --- /dev/null +++ b/.docker/tests/conftest.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +import json +from pathlib import Path + +import pytest + + +@pytest.fixture(scope='session', params=['base', 'base-with-services']) +def variant(request): + return request.param + + +@pytest.fixture(scope='session') +def docker_compose_file(pytestconfig, variant): + return f'docker-compose.{variant}.yml' + + +@pytest.fixture(scope='session') +def docker_compose(docker_services): + return docker_services._docker_compose + + +@pytest.fixture +def timeout(): + """Container and service startup timeout""" + return 30 + + +@pytest.fixture +def container_user(): + return 'aiida' + + +@pytest.fixture +def aiida_exec(docker_compose): + + def execute(command, user=None, **kwargs): + if user: + command = f'exec -T --user={user} aiida {command}' + else: + command = f'exec -T aiida {command}' + return docker_compose.execute(command, **kwargs) + + return execute + + +@pytest.fixture(scope='session') +def _build_config(): + return json.loads(Path('build.json').read_text())['variable'] + + +@pytest.fixture(scope='session') +def python_version(_build_config): + return _build_config['PYTHON_VERSION']['default'] + + +@pytest.fixture(scope='session') +def pgsql_version(_build_config): + return _build_config['PGSQL_VERSION']['default'] diff --git a/.docker/tests/test_aiida.py b/.docker/tests/test_aiida.py new file mode 100644 index 0000000000..327c3fa8ad --- /dev/null +++ b/.docker/tests/test_aiida.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +import json +import time + +from packaging.version import parse +import pytest + + +def test_correct_python_version_installed(aiida_exec, python_version): + info = json.loads(aiida_exec('mamba list --json --full-name python').decode())[0] + assert info['name'] == 'python' + assert parse(info['version']) == parse(python_version) + + +def test_correct_pgsql_version_installed(aiida_exec, pgsql_version, variant): + if variant == 'base': + pytest.skip('PostgreSQL is not installed in the base image') + + info = json.loads(aiida_exec('mamba list -n aiida-core-services --json --full-name postgresql').decode())[0] + assert info['name'] == 'postgresql' + assert parse(info['version']).major == parse(pgsql_version).major + + +def test_verdi_status(aiida_exec, container_user, timeout): + time.sleep(timeout) + output = aiida_exec('verdi status', user=container_user).decode().strip() + assert 'Connected to RabbitMQ' in output + assert 'Daemon is running' in output diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index feaac10cc6..26060d5840 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -19,9 +19,10 @@ on: jobs: build-test-upload: runs-on: ${{ inputs.runsOn }} - strategy: - matrix: - image: ["base", "aiida-core"] + defaults: + run: + shell: bash + working-directory: .docker steps: - name: Checkout Repo โšก๏ธ @@ -31,24 +32,30 @@ jobs: with: architecture: ${{ inputs.architecture }} - - name: Build image ๐Ÿ›  - run: doit build --target ${{ matrix.image }} --arch ${{ inputs.architecture }} --organization ${{ env.OWNER }} + - name: Build image base and base-with-services (output image name aiida-coer ยง) ๐Ÿ›  + run: doit build --target base --target base-with-services --arch ${{ inputs.architecture }} --organization ${{ env.OWNER }} env: # Full logs for CI build BUILDKIT_PROGRESS: plain - shell: bash - name: Run tests โœ… - run: VERSION=newly-build pytest -s tests/test-common.py tests/test-${{ matrix.image }}.py --variant ${{ matrix.image }} - shell: bash + run: VERSION=newly-build python -m pytest -s tests - name: Save image as a tar for later use ๐Ÿ’พ - run: docker save ${{ env.OWNER }}/${{ matrix.image }} -o /tmp/${{ matrix.image }}-${{ inputs.architecture }}.tar - shell: bash + run: | + docker save ${{ env.OWNER }}/base -o /tmp/base-${{ inputs.architecture }}.tar + docker save ${{ env.OWNER }}/aiida-core -o /tmp/aiida-core-${{ inputs.architecture }}.tar + + - name: Upload base image as artifact ๐Ÿ’พ + uses: actions/upload-artifact@v3 + with: + name: base-${{ inputs.architecture }} + path: /tmp/base-${{ inputs.architecture }}.tar + retention-days: 3 - - name: Upload image as artifact ๐Ÿ’พ + - name: Upload aiida-core image as artifact ๐Ÿ’พ uses: actions/upload-artifact@v3 with: - name: ${{ matrix.image }}-${{ inputs.architecture }} - path: /tmp/${{ matrix.image }}-${{ inputs.architecture }}.tar + name: aiida-core-${{ inputs.architecture }} + path: /tmp/aiida-core-${{ inputs.architecture }}.tar retention-days: 3 diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/docker-merge-tags.yml index 781c755496..a75b6c3224 100644 --- a/.github/workflows/docker-merge-tags.yml +++ b/.github/workflows/docker-merge-tags.yml @@ -4,10 +4,6 @@ name: Download images tags from GitHub artifacts and create multi-platform manif on: workflow_call: inputs: - image: - description: Image name - required: true - type: string registry: description: Docker registry, e.g. ghcr.io, docker.io required: true @@ -22,6 +18,9 @@ on: jobs: merge-tags: runs-on: ubuntu-latest + strategy: + matrix: + image: ["base", "aiida-core"] steps: - name: Checkout Repo โšก๏ธ @@ -34,12 +33,12 @@ jobs: - name: Download amd64 tags file ๐Ÿ“ฅ uses: actions/download-artifact@v3 with: - name: ${{ inputs.registry }}-${{ inputs.image }}-amd64-tags + name: ${{ inputs.registry }}-${{ matrix.image }}-amd64-tags path: /tmp/ - name: Download arm64 tags file ๐Ÿ“ฅ uses: actions/download-artifact@v3 with: - name: ${{ inputs.registry }}-${{ inputs.image }}-arm64-tags + name: ${{ inputs.registry }}-${{ matrix.image }}-arm64-tags path: /tmp/ - name: Login to Container Registry ๐Ÿ”‘ @@ -51,15 +50,15 @@ jobs: - name: Merge tags for the images of different arch ๐Ÿ”€ run: | - for arch_tag in $(cat /tmp/${{ inputs.image }}-amd64-tags.txt); do + for arch_tag in $(cat /tmp/${{ matrix.image }}-amd64-tags.txt); do tag=$(echo $arch_tag | sed "s/:amd64-/:/") docker manifest create $tag --amend $arch_tag docker manifest push $tag done - for arch_tag in $(cat /tmp/${{ inputs.image }}-arm64-tags.txt); do + for arch_tag in $(cat /tmp/${{ matrix.image }}-arm64-tags.txt); do tag=$(echo $arch_tag | sed "s/:arm64-/:/") docker manifest create $tag --amend $arch_tag docker manifest push $tag done - shell: bash \ No newline at end of file + shell: bash diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml index 358abd3aa9..a038bcb091 100644 --- a/.github/workflows/docker-push.yml +++ b/.github/workflows/docker-push.yml @@ -76,7 +76,7 @@ jobs: declare -a arr=(${{ steps.meta.outputs.tags }}) for tag in "${arr[@]}"; do arch_tag=$(echo ${tag} | sed "s/:/:${{ inputs.architecture }}-/") - docker tag aiidalab/${{ matrix.image }}:newly-build ${arch_tag} + docker tag ${{ env.OWNER }}/${{ matrix.image }}:newly-build ${arch_tag} docker push ${arch_tag} # write tag to file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4be6fb0950..c8dab04528 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,46 +27,34 @@ jobs: architecture: arm64 runsOn: buildjet-2vcpu-ubuntu-2204-arm - #amd64-push-ghcr: - # uses: ./.github/workflows/docker-push.yml - # strategy: - # matrix: - # image: ["base", "aiida-core"] - # with: - # architecture: amd64 - # image: ${{ matrix.image }} - # registry: ghcr.io - # secrets: - # REGISTRY_USERNAME: ${{ github.actor }} - # REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # needs: [amd64-build] + amd64-push-ghcr: + uses: ./.github/workflows/docker-push.yml + with: + architecture: amd64 + registry: ghcr.io + secrets: + REGISTRY_USERNAME: ${{ github.actor }} + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} + needs: [amd64-build] - #arm64-push-ghcr: - # uses: ./.github/workflows/docker-push.yml - # strategy: - # matrix: - # image: ["base", "aiida-core"] - # with: - # architecture: arm64 - # image: ${{ matrix.image }} - # registry: ghcr.io - # secrets: - # REGISTRY_USERNAME: ${{ github.actor }} - # REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # needs: [arm64-build] + arm64-push-ghcr: + uses: ./.github/workflows/docker-push.yml + with: + architecture: arm64 + registry: ghcr.io + secrets: + REGISTRY_USERNAME: ${{ github.actor }} + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} + needs: [arm64-build] - #merge-tags-ghcr: - # uses: ./.github/workflows/docker-merge-tags.yml - # strategy: - # matrix: - # image: ["base", "aiida-core"] - # with: - # image: ${{ matrix.image }} - # registry: ghcr.io - # secrets: - # REGISTRY_USERNAME: ${{ github.actor }} - # REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # needs: [amd64-push-ghcr, arm64-push-ghcr] + merge-tags-ghcr: + uses: ./.github/workflows/docker-merge-tags.yml + with: + registry: ghcr.io + secrets: + REGISTRY_USERNAME: ${{ github.actor }} + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} + needs: [amd64-push-ghcr, arm64-push-ghcr] #amd64-push-dockerhub: # if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) @@ -111,5 +99,3 @@ jobs: # REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} # REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} # needs: [amd64-push-dockerhub, arm64-push-dockerhub] - - \ No newline at end of file From ae2f72b4d8854e94e516eb1ec2b1d1ef3821d9b4 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Sat, 8 Jul 2023 22:50:29 +0200 Subject: [PATCH 13/19] fix env variable in s6 scripts --- .docker/base-with-services/Dockerfile | 4 +++- .docker/base-with-services/s6-assets/init/postgresql-init.sh | 4 ---- .../base-with-services/s6-assets/init/postgresql-prepare.sh | 2 -- .docker/base-with-services/s6-assets/init/rabbitmq-init.sh | 3 --- .docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/up | 1 + 5 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.docker/base-with-services/Dockerfile b/.docker/base-with-services/Dockerfile index 0b7e2af6a1..7fb607e37f 100644 --- a/.docker/base-with-services/Dockerfile +++ b/.docker/base-with-services/Dockerfile @@ -17,7 +17,9 @@ RUN mamba create -p /opt/conda/envs/aiida-core-services --yes \ postgresql=${PGSQL_VERSION} && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" + fix-permissions "/home/${SYSTEM_USER}" + +ENV MAMBA_RUN="mamba run -n aiida-core-services" # Install erlang. RUN apt-get update --yes && \ diff --git a/.docker/base-with-services/s6-assets/init/postgresql-init.sh b/.docker/base-with-services/s6-assets/init/postgresql-init.sh index e3c5edf0b9..f376afb6a4 100755 --- a/.docker/base-with-services/s6-assets/init/postgresql-init.sh +++ b/.docker/base-with-services/s6-assets/init/postgresql-init.sh @@ -1,9 +1,5 @@ #!/bin/bash -# -w waits until server is up -MAMBA_RUN="mamba run -n aiida-core-services" -SYSTEM_USER="aiida" - # make DB directory, if not existent if [ ! -d /home/${SYSTEM_USER}/.postgresql ]; then mkdir /home/${SYSTEM_USER}/.postgresql diff --git a/.docker/base-with-services/s6-assets/init/postgresql-prepare.sh b/.docker/base-with-services/s6-assets/init/postgresql-prepare.sh index 9822ab4d04..5659f3fd1f 100755 --- a/.docker/base-with-services/s6-assets/init/postgresql-prepare.sh +++ b/.docker/base-with-services/s6-assets/init/postgresql-prepare.sh @@ -1,7 +1,5 @@ #!/bin/bash -MAMBA_RUN="mamba run -n aiida-core-services" - PG_ISREADY=1 while [ "$PG_ISREADY" != "0" ]; do sleep 1 diff --git a/.docker/base-with-services/s6-assets/init/rabbitmq-init.sh b/.docker/base-with-services/s6-assets/init/rabbitmq-init.sh index 42c996ae0f..1856c318f9 100755 --- a/.docker/base-with-services/s6-assets/init/rabbitmq-init.sh +++ b/.docker/base-with-services/s6-assets/init/rabbitmq-init.sh @@ -1,7 +1,4 @@ #!/bin/bash -RMQ_VERSION="3.10.18" -SYSTEM_USER="aiida" - RABBITMQ_DATA_DIR="/home/${SYSTEM_USER}/.rabbitmq" mkdir -p "${RABBITMQ_DATA_DIR}" diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/up b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/up index 6fb77e1f87..e574020053 100644 --- a/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/up +++ b/.docker/base-with-services/s6-assets/s6-rc.d/rabbitmq-init/up @@ -1,4 +1,5 @@ #!/command/execlineb -S0 +with-contenv foreground { s6-echo "Calling /etc/init/rabbitmq-init.sh" } /etc/init/rabbitmq-init.sh From ab421d3ee42a28548c234fcbf338e9347bdf6996 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Sat, 8 Jul 2023 23:01:41 +0200 Subject: [PATCH 14/19] suppress dev version warning --- ...abbitmq-warning-suppress => verdi-warning-suppress} | 0 .../s6-rc.d/verdi-rabbitmq-warning-suppress/up | 6 ------ .../dependencies.d/aiida-prepare | 0 .../dependencies.d/base | 0 .../timeout-up | 0 .../type | 0 .../s6-assets/s6-rc.d/verdi-warning-suppress/up | 10 ++++++++++ .github/workflows/docker-push.yml | 8 +++++--- 8 files changed, 15 insertions(+), 9 deletions(-) rename .docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/{verdi-rabbitmq-warning-suppress => verdi-warning-suppress} (100%) delete mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/up rename .docker/base-with-services/s6-assets/s6-rc.d/{verdi-rabbitmq-warning-suppress => verdi-warning-suppress}/dependencies.d/aiida-prepare (100%) rename .docker/base-with-services/s6-assets/s6-rc.d/{verdi-rabbitmq-warning-suppress => verdi-warning-suppress}/dependencies.d/base (100%) rename .docker/base-with-services/s6-assets/s6-rc.d/{verdi-rabbitmq-warning-suppress => verdi-warning-suppress}/timeout-up (100%) rename .docker/base-with-services/s6-assets/s6-rc.d/{verdi-rabbitmq-warning-suppress => verdi-warning-suppress}/type (100%) create mode 100644 .docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/up diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/verdi-rabbitmq-warning-suppress b/.docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/verdi-warning-suppress similarity index 100% rename from .docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/verdi-rabbitmq-warning-suppress rename to .docker/base-with-services/s6-assets/s6-rc.d/user/contents.d/verdi-warning-suppress diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/up b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/up deleted file mode 100644 index ce522d341e..0000000000 --- a/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/up +++ /dev/null @@ -1,6 +0,0 @@ -#!/command/execlineb -P - -# Supress rabbitmq version warning for arm64 since -# it is built using latest version rabbitmq from apt install. -# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf -verdi config set warnings.rabbitmq_version False diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/dependencies.d/aiida-prepare b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/dependencies.d/aiida-prepare similarity index 100% rename from .docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/dependencies.d/aiida-prepare rename to .docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/dependencies.d/aiida-prepare diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/dependencies.d/base b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/dependencies.d/base similarity index 100% rename from .docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/dependencies.d/base rename to .docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/dependencies.d/base diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/timeout-up b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/timeout-up similarity index 100% rename from .docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/timeout-up rename to .docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/timeout-up diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/type b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/type similarity index 100% rename from .docker/base-with-services/s6-assets/s6-rc.d/verdi-rabbitmq-warning-suppress/type rename to .docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/type diff --git a/.docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/up b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/up new file mode 100644 index 0000000000..a2ceeb2ef6 --- /dev/null +++ b/.docker/base-with-services/s6-assets/s6-rc.d/verdi-warning-suppress/up @@ -0,0 +1,10 @@ +#!/command/execlineb -P + +# Supress rabbitmq version warning +# it is built using RMQ version > 3.8.15 which has the issue as described in +# https://github.com/aiidateam/aiida-core/wiki/RabbitMQ-version-to-use +# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf +verdi config set warnings.rabbitmq_version False + +# Supress verdi version warning +verdi config set warnings.development_version False diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml index a038bcb091..0589548be1 100644 --- a/.github/workflows/docker-push.yml +++ b/.github/workflows/docker-push.yml @@ -27,6 +27,10 @@ jobs: strategy: matrix: image: ["base", "aiida-core"] + defaults: + run: + shell: bash + working-directory: .docker steps: - name: Checkout Repo โšก๏ธ @@ -58,9 +62,8 @@ jobs: type=edge,enable={{is_default_branch}} type=sha,enable=${{ github.ref_type != 'tag' }} type=ref,event=pr - type=match,pattern=v(\d{4}\.\d{4}),group=1 + type=match,pattern=v(\d+\.\d+.\d+),group=1 type=raw,value={{tag}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=raw,value=aiida-${{ env.AIIDA_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} @@ -82,7 +85,6 @@ jobs: # write tag to file echo ${arch_tag} >> /tmp/${{ matrix.image }}-${{ inputs.architecture }}-tags.txt done - shell: bash - name: Upload tags file ๐Ÿ“ค uses: actions/upload-artifact@v3 From 37312bd2457ace7623a423afd2c0b49cdb2b48ae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 8 Jul 2023 22:01:00 +0000 Subject: [PATCH 15/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .docker/build.json | 1 - .github/actions/load-image/action.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.docker/build.json b/.docker/build.json index a274737600..554a4d95c7 100644 --- a/.docker/build.json +++ b/.docker/build.json @@ -11,4 +11,3 @@ } } } - diff --git a/.github/actions/load-image/action.yml b/.github/actions/load-image/action.yml index 9384089e5f..5909cdd518 100644 --- a/.github/actions/load-image/action.yml +++ b/.github/actions/load-image/action.yml @@ -28,4 +28,4 @@ runs: - name: Delete the file ๐Ÿ—‘๏ธ run: rm -f /tmp/${{ inputs.image }}-${{ inputs.architecture }}.tar shell: bash - if: always() \ No newline at end of file + if: always() From 8853d167e6734aab8a629fe8744e78b72d2c5fbb Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Mon, 10 Jul 2023 11:31:14 +0200 Subject: [PATCH 16/19] push ghcr only when it is pull request form org --- .github/workflows/docker.yml | 85 +++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c8dab04528..bba565fcdf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,6 +28,7 @@ jobs: runsOn: buildjet-2vcpu-ubuntu-2204-arm amd64-push-ghcr: + if: github.repository == 'aiidateam/aiida-core' uses: ./.github/workflows/docker-push.yml with: architecture: amd64 @@ -38,6 +39,7 @@ jobs: needs: [amd64-build] arm64-push-ghcr: + if: github.repository == 'aiidateam/aiida-core' uses: ./.github/workflows/docker-push.yml with: architecture: arm64 @@ -48,6 +50,7 @@ jobs: needs: [arm64-build] merge-tags-ghcr: + if: github.repository == 'aiidateam/aiida-core' uses: ./.github/workflows/docker-merge-tags.yml with: registry: ghcr.io @@ -56,46 +59,46 @@ jobs: REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} needs: [amd64-push-ghcr, arm64-push-ghcr] - #amd64-push-dockerhub: - # if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) - # uses: ./.github/workflows/docker-push.yml - # strategy: - # matrix: - # image: ["base", "aiida-core"] - # with: - # architecture: amd64 - # image: ${{ matrix.image }} - # registry: docker.io - # secrets: - # REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - # REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - # needs: [amd64-build] + amd64-push-dockerhub: + if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + uses: ./.github/workflows/docker-push.yml + strategy: + matrix: + image: ["base", "aiida-core"] + with: + architecture: amd64 + image: ${{ matrix.image }} + registry: docker.io + secrets: + REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + needs: [amd64-build] - #arm64-push-dockerhub: - # if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) - # uses: ./.github/workflows/docker-push.yml - # strategy: - # matrix: - # image: ["base", "aiida-core"] - # with: - # architecture: arm64 - # image: ${{ matrix.image }} - # registry: docker.io - # secrets: - # REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - # REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - # needs: [arm64-build] + arm64-push-dockerhub: + if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + uses: ./.github/workflows/docker-push.yml + strategy: + matrix: + image: ["base", "aiida-core"] + with: + architecture: arm64 + image: ${{ matrix.image }} + registry: docker.io + secrets: + REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + needs: [arm64-build] - #merge-tags-dockerhub: - # if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) - # uses: ./.github/workflows/docker-merge-tags.yml - # strategy: - # matrix: - # image: ["base", "aiida-core"] - # with: - # image: ${{ matrix.image }} - # registry: docker.io - # secrets: - # REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - # REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - # needs: [amd64-push-dockerhub, arm64-push-dockerhub] + merge-tags-dockerhub: + if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + uses: ./.github/workflows/docker-merge-tags.yml + strategy: + matrix: + image: ["base", "aiida-core"] + with: + image: ${{ matrix.image }} + registry: docker.io + secrets: + REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + needs: [amd64-push-dockerhub, arm64-push-dockerhub] From 2a46113c756be186fef05734d0f456716fca982a Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Mon, 10 Jul 2023 11:38:03 +0200 Subject: [PATCH 17/19] upload artifact which is time consuming only for PR from org --- .github/workflows/docker-build-test-upload.yml | 2 ++ .github/workflows/docker.yml | 14 ++------------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index 26060d5840..87b96ec4fa 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -52,6 +52,7 @@ jobs: name: base-${{ inputs.architecture }} path: /tmp/base-${{ inputs.architecture }}.tar retention-days: 3 + if: github.repository == 'aiidateam/aiida-core' - name: Upload aiida-core image as artifact ๐Ÿ’พ uses: actions/upload-artifact@v3 @@ -59,3 +60,4 @@ jobs: name: aiida-core-${{ inputs.architecture }} path: /tmp/aiida-core-${{ inputs.architecture }}.tar retention-days: 3 + if: github.repository == 'aiidateam/aiida-core' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bba565fcdf..c611a0bf65 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,6 +22,8 @@ jobs: runsOn: ubuntu-latest arm64-build: + # This cost the runner minutes, so only run on PRs not from forks + if: github.repository == 'aiidateam/aiida-core' uses: ./.github/workflows/docker-build-test-upload.yml with: architecture: arm64 @@ -62,12 +64,8 @@ jobs: amd64-push-dockerhub: if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) uses: ./.github/workflows/docker-push.yml - strategy: - matrix: - image: ["base", "aiida-core"] with: architecture: amd64 - image: ${{ matrix.image }} registry: docker.io secrets: REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} @@ -77,12 +75,8 @@ jobs: arm64-push-dockerhub: if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) uses: ./.github/workflows/docker-push.yml - strategy: - matrix: - image: ["base", "aiida-core"] with: architecture: arm64 - image: ${{ matrix.image }} registry: docker.io secrets: REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} @@ -92,11 +86,7 @@ jobs: merge-tags-dockerhub: if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) uses: ./.github/workflows/docker-merge-tags.yml - strategy: - matrix: - image: ["base", "aiida-core"] with: - image: ${{ matrix.image }} registry: docker.io secrets: REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} From c99fe2989c8766bd2e45ca455d27ee900f057feb Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Mon, 10 Jul 2023 11:52:00 +0200 Subject: [PATCH 18/19] action only for container build --- .github/workflows/docker.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c611a0bf65..f2c7cd68cb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,9 +3,13 @@ name: Build, test and push Docker Images on: pull_request: + paths: + - .docker/** + - .github/workflows/docker-*.yml push: - branches: - - main + paths: + - .docker/** + - .github/workflows/docker-*.yml workflow_dispatch: # https://docs.github.com/en/actions/using-jobs/using-concurrency @@ -22,8 +26,6 @@ jobs: runsOn: ubuntu-latest arm64-build: - # This cost the runner minutes, so only run on PRs not from forks - if: github.repository == 'aiidateam/aiida-core' uses: ./.github/workflows/docker-build-test-upload.yml with: architecture: arm64 From 6fd7e2926c84613f6c480afc911ffb7853b91ce4 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Mon, 10 Jul 2023 12:03:22 +0200 Subject: [PATCH 19/19] not run some container action from fork --- .github/workflows/docker-build-test-upload.yml | 4 ++-- .github/workflows/docker.yml | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index 87b96ec4fa..b9b50ec48f 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -52,7 +52,7 @@ jobs: name: base-${{ inputs.architecture }} path: /tmp/base-${{ inputs.architecture }}.tar retention-days: 3 - if: github.repository == 'aiidateam/aiida-core' + if: ${{ !github.event.pull_request.head.repo.fork }} - name: Upload aiida-core image as artifact ๐Ÿ’พ uses: actions/upload-artifact@v3 @@ -60,4 +60,4 @@ jobs: name: aiida-core-${{ inputs.architecture }} path: /tmp/aiida-core-${{ inputs.architecture }}.tar retention-days: 3 - if: github.repository == 'aiidateam/aiida-core' + if: ${{ !github.event.pull_request.head.repo.fork }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f2c7cd68cb..a1a8b4b3a9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,6 +30,7 @@ jobs: with: architecture: arm64 runsOn: buildjet-2vcpu-ubuntu-2204-arm + if: ${{ !github.event.pull_request.head.repo.fork }} amd64-push-ghcr: if: github.repository == 'aiidateam/aiida-core' @@ -43,7 +44,6 @@ jobs: needs: [amd64-build] arm64-push-ghcr: - if: github.repository == 'aiidateam/aiida-core' uses: ./.github/workflows/docker-push.yml with: architecture: arm64 @@ -52,9 +52,9 @@ jobs: REGISTRY_USERNAME: ${{ github.actor }} REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} needs: [arm64-build] + if: ${{ !github.event.pull_request.head.repo.fork }} merge-tags-ghcr: - if: github.repository == 'aiidateam/aiida-core' uses: ./.github/workflows/docker-merge-tags.yml with: registry: ghcr.io @@ -62,6 +62,7 @@ jobs: REGISTRY_USERNAME: ${{ github.actor }} REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} needs: [amd64-push-ghcr, arm64-push-ghcr] + if: ${{ !github.event.pull_request.head.repo.fork }} amd64-push-dockerhub: if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))