From e28db90957c0589ded593cbfcd42e8835a6b7d5d Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 <60785969+matusdrobuliak66@users.noreply.github.com> Date: Fri, 31 May 2024 09:36:21 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20introducing=20elastic=20file=20syst?= =?UTF-8?q?em=20guardian=20(OPS=20=E2=9A=A0=EF=B8=8F)=20(#5887)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/CODEOWNERS | 1 + .github/workflows/ci-testing-deploy.yml | 60 ++++ Makefile | 1 + .../src/pytest_simcore/environment_configs.py | 2 +- .../src/pytest_simcore/simcore_services.py | 1 + .../simcore_service_director_v2/utils/dask.py | 2 +- services/docker-compose-build.yml | 16 ++ services/docker-compose-deploy.yml | 2 + services/docker-compose.devel.yml | 8 + services/docker-compose.local.yml | 8 + services/docker-compose.yml | 9 + services/efs-guardian/Dockerfile | 193 +++++++++++++ services/efs-guardian/Makefile | 5 + services/efs-guardian/README.md | 4 + services/efs-guardian/VERSION | 1 + services/efs-guardian/docker/boot.sh | 58 ++++ services/efs-guardian/docker/entrypoint.sh | 94 ++++++ services/efs-guardian/docker/healthcheck.py | 41 +++ services/efs-guardian/requirements/Makefile | 10 + services/efs-guardian/requirements/_base.in | 18 ++ services/efs-guardian/requirements/_base.txt | 175 ++++++++++++ services/efs-guardian/requirements/_test.in | 32 +++ services/efs-guardian/requirements/_test.txt | 267 ++++++++++++++++++ services/efs-guardian/requirements/_tools.in | 7 + services/efs-guardian/requirements/_tools.txt | 74 +++++ services/efs-guardian/requirements/ci.txt | 21 ++ .../efs-guardian/requirements/constraints.txt | 0 services/efs-guardian/requirements/dev.txt | 22 ++ services/efs-guardian/requirements/prod.txt | 18 ++ services/efs-guardian/setup.cfg | 13 + services/efs-guardian/setup.py | 69 +++++ .../simcore_service_efs_guardian/__init__.py | 3 + .../src/simcore_service_efs_guardian/_meta.py | 65 +++++ .../api/__init__.py | 0 .../api/rest/__init__.py | 0 .../api/rest/health.py | 18 ++ .../api/rest/routes.py | 17 ++ .../api/rpc/__init__.py | 0 .../api/rpc/rpc_routes.py | 22 ++ .../src/simcore_service_efs_guardian/cli.py | 24 ++ .../core/__init__.py | 0 .../core/application.py | 59 ++++ .../core/settings.py | 85 ++++++ .../exceptions/__init__.py | 5 + .../exceptions/_base.py | 8 + .../exceptions/custom_errors.py | 9 + .../exceptions/handlers/__init__.py | 7 + .../src/simcore_service_efs_guardian/main.py | 17 ++ .../services/__init__.py | 0 .../efs-guardian/tests/integration/.gitkeep | 0 services/efs-guardian/tests/unit/conftest.py | 117 ++++++++ .../tests/unit/test_api_health.py | 13 + services/efs-guardian/tests/unit/test_cli.py | 21 ++ .../tests/unit/test_core_settings.py | 12 + services/efs-guardian/tests/unit/test_main.py | 12 + tests/swarm-deploy/test_service_restart.py | 1 + 56 files changed, 1745 insertions(+), 2 deletions(-) create mode 100644 services/efs-guardian/Dockerfile create mode 100644 services/efs-guardian/Makefile create mode 100644 services/efs-guardian/README.md create mode 100644 services/efs-guardian/VERSION create mode 100755 services/efs-guardian/docker/boot.sh create mode 100755 services/efs-guardian/docker/entrypoint.sh create mode 100755 services/efs-guardian/docker/healthcheck.py create mode 100644 services/efs-guardian/requirements/Makefile create mode 100644 services/efs-guardian/requirements/_base.in create mode 100644 services/efs-guardian/requirements/_base.txt create mode 100644 services/efs-guardian/requirements/_test.in create mode 100644 services/efs-guardian/requirements/_test.txt create mode 100644 services/efs-guardian/requirements/_tools.in create mode 100644 services/efs-guardian/requirements/_tools.txt create mode 100644 services/efs-guardian/requirements/ci.txt create mode 100644 services/efs-guardian/requirements/constraints.txt create mode 100644 services/efs-guardian/requirements/dev.txt create mode 100644 services/efs-guardian/requirements/prod.txt create mode 100644 services/efs-guardian/setup.cfg create mode 100755 services/efs-guardian/setup.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/__init__.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/_meta.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/api/__init__.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/api/rest/__init__.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/api/rest/health.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/api/rest/routes.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/__init__.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/rpc_routes.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/cli.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/core/__init__.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/core/application.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/exceptions/__init__.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/exceptions/_base.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/exceptions/custom_errors.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/exceptions/handlers/__init__.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/main.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/__init__.py create mode 100644 services/efs-guardian/tests/integration/.gitkeep create mode 100644 services/efs-guardian/tests/unit/conftest.py create mode 100644 services/efs-guardian/tests/unit/test_api_health.py create mode 100644 services/efs-guardian/tests/unit/test_cli.py create mode 100644 services/efs-guardian/tests/unit/test_core_settings.py create mode 100644 services/efs-guardian/tests/unit/test_main.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6f40095b947..e982fee49ac 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -29,6 +29,7 @@ Makefile @pcrespov @sanderegg /services/director*/ @sanderegg @pcrespov @GitHK /services/docker-compose*.yml @sanderegg @mrnicegyu11 @YuryHrytsuk /services/dynamic-sidecar/ @GitHK +/services/efs-guardian/ @matusdrobuliak66 /services/invitations/ @pcrespov /services/migration/ @pcrespov /services/osparc-gateway-server/ @sanderegg diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index d87c462c613..87c2ee0c7b2 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -70,6 +70,7 @@ jobs: director: ${{ steps.filter.outputs.director }} director-v2: ${{ steps.filter.outputs.director-v2 }} dynamic-sidecar: ${{ steps.filter.outputs.dynamic-sidecar }} + efs-guardian: ${{ steps.filter.outputs.efs-guardian }} invitations: ${{ steps.filter.outputs.invitations }} migration: ${{ steps.filter.outputs.migration }} osparc-gateway-server: ${{ steps.filter.outputs.osparc-gateway-server }} @@ -199,6 +200,12 @@ jobs: - 'services/docker-compose*' - 'scripts/mypy/*' - 'mypy.ini' + efs-guardian: + - 'packages/**' + - 'services/efs-guardian/**' + - 'services/docker-compose*' + - 'scripts/mypy/*' + - 'mypy.ini' invitations: - 'packages/**' - 'services/invitations/**' @@ -1207,6 +1214,58 @@ jobs: with: flags: unittests #optional + unit-test-efs-guardian: + needs: changes + if: ${{ needs.changes.outputs.efs-guardian == 'true' || github.event_name == 'push' }} + timeout-minutes: 18 # if this timeout gets too small, then split the tests + name: "[unit] efs-guardian" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: ["3.10"] + os: [ubuntu-22.04] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + - name: setup python environment + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: install uv + uses: yezz123/setup-uv@v4 + - uses: actions/cache@v4 + id: cache-uv + with: + path: ~/.cache/uv + key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - name: install + run: | + make devenv + source .venv/bin/activate && \ + pushd services/efs-guardian && \ + make install-ci + - name: typecheck + run: | + source .venv/bin/activate && \ + pushd services/efs-guardian && \ + make mypy + - name: test + if: always() + run: | + source .venv/bin/activate && \ + pushd services/efs-guardian && \ + make test-ci-unit + - uses: codecov/codecov-action@v4.4.1 + with: + flags: unittests #optional + unit-test-frontend: needs: changes if: ${{ needs.changes.outputs.static-webserver == 'true' || github.event_name == 'push' }} @@ -1638,6 +1697,7 @@ jobs: unit-test-director-v2, unit-test-director, unit-test-dynamic-sidecar, + unit-test-efs-guardian, unit-test-frontend, unit-test-models-library, unit-test-notifications-library, diff --git a/Makefile b/Makefile index 6ec735dac95..6938cfc0afd 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,7 @@ SERVICES_NAMES_TO_BUILD := \ director \ director-v2 \ dynamic-sidecar \ + efs-guardian \ invitations \ migration \ osparc-gateway-server \ diff --git a/packages/pytest-simcore/src/pytest_simcore/environment_configs.py b/packages/pytest-simcore/src/pytest_simcore/environment_configs.py index fa023cef28f..45dbc64ccb0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/environment_configs.py +++ b/packages/pytest-simcore/src/pytest_simcore/environment_configs.py @@ -11,7 +11,7 @@ from .helpers.utils_envs import delenvs_from_dict, load_dotenv, setenvs_from_dict -@pytest.fixture(scope="session") +@pytest.fixture(scope="session") # MD: get this, I will mock it with my app environmnet def env_devel_dict(env_devel_file: Path) -> EnvVarsDict: assert env_devel_file.exists() assert env_devel_file.name == ".env-devel" diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index 1243d6914d6..51f278fbde4 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -46,6 +46,7 @@ "datcore-adapter": "/v0/live", "director-v2": "/", "dynamic-schdlr": "/", + "efs-guardian": "/", "invitations": "/", "payments": "/", "resource-usage-tracker": "/", diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index 5cb5d2b0f53..e78ad6f03d6 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -601,7 +601,7 @@ def check_if_cluster_is_able_to_run_pipeline( json_dumps(task_resources, indent=2), ) - if can_a_worker_run_task: + if can_a_worker_run_task: # OsparcErrorMixin return # check if we have missing resources diff --git a/services/docker-compose-build.yml b/services/docker-compose-build.yml index 779254c3951..e51ca8dbd7b 100644 --- a/services/docker-compose-build.yml +++ b/services/docker-compose-build.yml @@ -264,6 +264,22 @@ services: org.opencontainers.image.source: "${VCS_URL}" org.opencontainers.image.revision: "${VCS_REF}" + efs-guardian: + image: local/efs-guardian:${BUILD_TARGET:?build_target_required} + build: + context: ../ + dockerfile: services/efs-guardian/Dockerfile + cache_from: + - local/efs-guardian:${BUILD_TARGET:?build_target_required} + - ${DOCKER_REGISTRY:-itisfoundation}/efs-guardian:master-github-latest + - ${DOCKER_REGISTRY:-itisfoundation}/efs-guardian:staging-github-latest + - ${DOCKER_REGISTRY:-itisfoundation}/efs-guardian:release-github-latest + target: ${BUILD_TARGET:?build_target_required} + labels: + org.opencontainers.image.created: "${BUILD_DATE}" + org.opencontainers.image.source: "${VCS_URL}" + org.opencontainers.image.revision: "${VCS_REF}" + invitations: image: local/invitations:${BUILD_TARGET:?build_target_required} build: diff --git a/services/docker-compose-deploy.yml b/services/docker-compose-deploy.yml index a3ec02d7bc9..f8e306b0ed2 100644 --- a/services/docker-compose-deploy.yml +++ b/services/docker-compose-deploy.yml @@ -20,6 +20,8 @@ services: image: ${DOCKER_REGISTRY:-itisfoundation}/director-v2:${DOCKER_IMAGE_TAG:-latest} dynamic-sidecar: image: ${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest} + efs-guardian: + image: ${DOCKER_REGISTRY:-itisfoundation}/efs-guardian:${DOCKER_IMAGE_TAG:-latest} invitations: image: ${DOCKER_REGISTRY:-itisfoundation}/invitations:${DOCKER_IMAGE_TAG:-latest} migration: diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index d25a007c005..840497e81e3 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -102,6 +102,14 @@ services: - ./director-v2:/devel/services/director-v2 - ../packages:/devel/packages + efs-guardian: + environment: + <<: *common-environment + EFS_GUARDIAN_LOGLEVEL: DEBUG + volumes: + - ./efs-guardian:/devel/services/efs-guardian + - ../packages:/devel/packages + static-webserver: volumes: - ./static-webserver/client/source-output:/static-content diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index d9b50768459..b5859000ddb 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -70,6 +70,14 @@ services: - "8000" - "3010:3000" + efs-guardian: + environment: + <<: *common_environment + EFS_GUARDIAN_REMOTE_DEBUGGING_PORT : 3000 + ports: + - "8013:8000" + - "3020:3000" + invitations: environment: <<: *common_environment diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 3ef810a2a3d..30c0f7f724f 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -343,6 +343,15 @@ services: - computational_services_subnet secrets: *dask_tls_secrets + efs-guardian: + image: ${DOCKER_REGISTRY:-itisfoundation}/efs-guardian:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + networks: + - default + environment: + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + invitations: image: ${DOCKER_REGISTRY:-itisfoundation}/invitations:${DOCKER_IMAGE_TAG:-latest} init: true diff --git a/services/efs-guardian/Dockerfile b/services/efs-guardian/Dockerfile new file mode 100644 index 00000000000..d1468f443f2 --- /dev/null +++ b/services/efs-guardian/Dockerfile @@ -0,0 +1,193 @@ +# syntax=docker/dockerfile:1 +ARG PYTHON_VERSION="3.10.10" +FROM python:${PYTHON_VERSION}-slim-buster as base + +# +# USAGE: +# cd sercices/efs-guardian +# docker build -f Dockerfile -t efs-guardian:prod --target production ../../ +# docker run efs-guardian:prod +# +# REQUIRED: context expected at ``osparc-simcore/`` folder because we need access to osparc-simcore/packages + +LABEL maintainer=sanderegg + +# NOTE: to list the latest version run `make` inside `scripts/apt-packages-versions` +ENV DOCKER_APT_VERSION="5:24.0.5-1~debian.10~buster" + +# for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] +RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache +RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ + --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ + set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + gosu \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # only the cli is needed and we remove the unnecessary stuff again + docker-ce-cli=${DOCKER_APT_VERSION} \ + && apt-get remove -y\ + gnupg \ + curl \ + lsb-release \ + && apt-get clean -y\ + # verify that the binary works + && gosu nobody true + +# simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) +ENV SC_USER_ID=8004 \ + SC_USER_NAME=scu \ + SC_BUILD_TARGET=base \ + SC_BOOT_MODE=default + +RUN adduser \ + --uid ${SC_USER_ID} \ + --disabled-password \ + --gecos "" \ + --shell /bin/sh \ + --home /home/${SC_USER_NAME} \ + ${SC_USER_NAME} + + +# Sets utf-8 encoding for Python et al +ENV LANG=C.UTF-8 + +# Turns off writing .pyc files; superfluous on an ephemeral container. +ENV PYTHONDONTWRITEBYTECODE=1 \ + VIRTUAL_ENV=/home/scu/.venv + +# Ensures that the python and pip executables used in the image will be +# those from our virtualenv. +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" + +EXPOSE 8000 +EXPOSE 3000 + +# -------------------------- Build stage ------------------- +# Installs build/package management tools and third party dependencies +# +# + /build WORKDIR +# +FROM base as build + +ENV SC_BUILD_TARGET=build + +RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ + --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ + set -eux \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential + +# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install uv~=0.1 + +# NOTE: python virtualenv is used here such that installed +# packages may be moved to production image easily by copying the venv +RUN uv venv "${VIRTUAL_ENV}" + +RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ + uv pip install --upgrade \ + pip~=24.0 \ + wheel \ + setuptools + +WORKDIR /build + +# install base 3rd party dependencies +# NOTE: copies to /build to avoid overwriting later which would invalidate this layer +RUN \ + --mount=type=bind,source=services/efs-guardian/requirements/_base.txt,target=_base.txt \ + --mount=type=cache,mode=0755,target=/root/.cache/uv \ + uv pip install \ + --requirement _base.txt + + +# --------------------------Prod-depends-only stage ------------------- +# This stage is for production only dependencies that get partially wiped out afterwards (final docker image concerns) +# +# + /build +# + services/efs-guardian [scu:scu] WORKDIR +# +FROM build as prod-only-deps + +ENV SC_BUILD_TARGET prod-only-deps + +WORKDIR /build/services/efs-guardian + +RUN \ + --mount=type=bind,source=packages,target=/build/packages,rw \ + --mount=type=bind,source=services/efs-guardian,target=/build/services/efs-guardian,rw \ + --mount=type=cache,mode=0755,target=/root/.cache/uv \ + uv pip install \ + --requirement requirements/prod.txt \ + && uv pip list + + +# --------------------------Production stage ------------------- +# Final cleanup up to reduce image size and startup setup +# Runs as scu (non-root user) +# +# + /home/scu $HOME = WORKDIR +# + services/efs-guardian [scu:scu] +# +FROM base as production + +ENV SC_BUILD_TARGET=production \ + SC_BOOT_MODE=production + +ENV PYTHONOPTIMIZE=TRUE + +WORKDIR /home/scu +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu + +# Starting from clean base image, copies pre-installed virtualenv from prod-only-deps +COPY --chown=scu:scu --from=prod-only-deps ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +# Copies booting scripts +COPY --chown=scu:scu services/efs-guardian/docker services/efs-guardian/docker +RUN chmod +x services/efs-guardian/docker/*.sh + + +HEALTHCHECK --interval=10s \ + --timeout=5s \ + --start-period=30s \ + --start-interval=1s \ + --retries=5 \ + CMD ["python3", "services/efs-guardian/docker/healthcheck.py", "http://localhost:8000/"] + +ENTRYPOINT [ "/bin/sh", "services/efs-guardian/docker/entrypoint.sh" ] +CMD ["/bin/sh", "services/efs-guardian/docker/boot.sh"] + + +# --------------------------Development stage ------------------- +# Source code accessible in host but runs in container +# Runs as myu with same gid/uid as host +# Placed at the end to speed-up the build if images targeting production +# +# + /devel WORKDIR +# + services (mounted volume) +# +FROM build as development + +ENV SC_BUILD_TARGET=development \ + SC_DEVEL_MOUNT=/devel/services/efs-guardian + +WORKDIR /devel + +RUN chown -R scu:scu "${VIRTUAL_ENV}" + +ENTRYPOINT ["/bin/sh", "services/efs-guardian/docker/entrypoint.sh"] +CMD ["/bin/sh", "services/efs-guardian/docker/boot.sh"] diff --git a/services/efs-guardian/Makefile b/services/efs-guardian/Makefile new file mode 100644 index 00000000000..af13c225526 --- /dev/null +++ b/services/efs-guardian/Makefile @@ -0,0 +1,5 @@ +# +# DEVELOPMENT recipes for efs-guardian service +# +include ../../scripts/common.Makefile +include ../../scripts/common-service.Makefile diff --git a/services/efs-guardian/README.md b/services/efs-guardian/README.md new file mode 100644 index 00000000000..503bdb93b1b --- /dev/null +++ b/services/efs-guardian/README.md @@ -0,0 +1,4 @@ +# efs-guardian + + +Service to monitor and manage elastic file system diff --git a/services/efs-guardian/VERSION b/services/efs-guardian/VERSION new file mode 100644 index 00000000000..3eefcb9dd5b --- /dev/null +++ b/services/efs-guardian/VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/services/efs-guardian/docker/boot.sh b/services/efs-guardian/docker/boot.sh new file mode 100755 index 00000000000..d4d046d5a14 --- /dev/null +++ b/services/efs-guardian/docker/boot.sh @@ -0,0 +1,58 @@ +#!/bin/sh +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + +INFO="INFO: [$(basename "$0")] " + +echo "$INFO" "Booting in ${SC_BOOT_MODE} mode ..." +echo "$INFO" "User :$(id "$(whoami)")" +echo "$INFO" "Workdir : $(pwd)" + +# +# DEVELOPMENT MODE +# +# - prints environ info +# - installs requirements in mounted volume +# +if [ "${SC_BUILD_TARGET}" = "development" ]; then + echo "$INFO" "Environment :" + printenv | sed 's/=/: /' | sed 's/^/ /' | sort + echo "$INFO" "Python :" + python --version | sed 's/^/ /' + command -v python | sed 's/^/ /' + + cd services/efs-guardian || exit 1 + pip install uv + uv pip --quiet --no-cache-dir install -r requirements/dev.txt + cd - || exit 1 + echo "$INFO" "PIP :" + uv pip list | sed 's/^/ /' +fi + +# +# RUNNING application +# + +APP_LOG_LEVEL=${API_SERVER_LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL:-INFO}}} +SERVER_LOG_LEVEL=$(echo "${APP_LOG_LEVEL}" | tr '[:upper:]' '[:lower:]') +echo "$INFO" "Log-level app/server: $APP_LOG_LEVEL/$SERVER_LOG_LEVEL" + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + reload_dir_packages=$(find /devel/packages -maxdepth 3 -type d -path "*/src/*" ! -path "*.*" -exec echo '--reload-dir {} \' \;) + + exec sh -c " + cd services/efs-guardian/src/simcore_service_efs_guardian && \ + python -m debugpy --listen 0.0.0.0:${EFS_GUARDIAN_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \ + --host 0.0.0.0 \ + --reload \ + $reload_dir_packages + --reload-dir . \ + --log-level \"${SERVER_LOG_LEVEL}\" + " +else + exec uvicorn simcore_service_efs_guardian.main:the_app \ + --host 0.0.0.0 \ + --log-level "${SERVER_LOG_LEVEL}" +fi diff --git a/services/efs-guardian/docker/entrypoint.sh b/services/efs-guardian/docker/entrypoint.sh new file mode 100755 index 00000000000..ac4bcf76085 --- /dev/null +++ b/services/efs-guardian/docker/entrypoint.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# +# - Executes *inside* of the container upon start as --user [default root] +# - Notice that the container *starts* as --user [default root] but +# *runs* as non-root user [scu] +# +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + +INFO="INFO: [$(basename "$0")] " +WARNING="WARNING: [$(basename "$0")] " +ERROR="ERROR: [$(basename "$0")] " + +# Read self-signed SSH certificates (if applicable) +# +# In case efs-guardian must access a docker registry in a secure way using +# non-standard certificates (e.g. such as self-signed certificates), this call is needed. +# It needs to be executed as root. Also required to any access for example to secure rabbitmq. +update-ca-certificates + +echo "$INFO" "Entrypoint for stage ${SC_BUILD_TARGET} ..." +echo "$INFO" "User :$(id "$(whoami)")" +echo "$INFO" "Workdir : $(pwd)" +echo "$INFO" "User : $(id scu)" +echo "$INFO" "python : $(command -v python)" +echo "$INFO" "pip : $(command -v pip)" + +# +# DEVELOPMENT MODE +# - expects docker run ... -v $(pwd):$SC_DEVEL_MOUNT +# - mounts source folders +# - deduces host's uid/gip and assigns to user within docker +# +if [ "${SC_BUILD_TARGET}" = "development" ]; then + echo "$INFO" "development mode detected..." + stat "${SC_DEVEL_MOUNT}" >/dev/null 2>&1 || + (echo "$ERROR" "You must mount '$SC_DEVEL_MOUNT' to deduce user and group ids" && exit 1) + + echo "$INFO" "setting correct user id/group id..." + HOST_USERID=$(stat --format=%u "${SC_DEVEL_MOUNT}") + HOST_GROUPID=$(stat --format=%g "${SC_DEVEL_MOUNT}") + CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) + if [ "$HOST_USERID" -eq 0 ]; then + echo "$WARNING" "Folder mounted owned by root user... adding $SC_USER_NAME to root..." + adduser "$SC_USER_NAME" root + else + echo "$INFO" "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." + # take host's credentials in $SC_USER_NAME + if [ -z "$CONT_GROUPNAME" ]; then + echo "$WARNING" "Creating new group grp$SC_USER_NAME" + CONT_GROUPNAME=grp$SC_USER_NAME + addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" + else + echo "$INFO" "group already exists" + fi + echo "$INFO" "Adding $SC_USER_NAME to group $CONT_GROUPNAME..." + adduser "$SC_USER_NAME" "$CONT_GROUPNAME" + + echo "$WARNING" "Changing ownership [this could take some time]" + echo "$INFO" "Changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" + usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" + + echo "$INFO" "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; + # change user property of files already around + echo "$INFO" "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; + fi +fi + + +# Appends docker group if socket is mounted +DOCKER_MOUNT=/var/run/docker.sock +if stat $DOCKER_MOUNT >/dev/null 2>&1; then + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker + + if ! addgroup --gid "$GROUPID" $GROUPNAME >/dev/null 2>&1; then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." + # if group already exists in container, then reuse name + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" + fi + adduser "$SC_USER_NAME" "$GROUPNAME" +fi + +echo "$INFO Starting $* ..." +echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" +echo " local dir : $(ls -al)" + +exec gosu "$SC_USER_NAME" "$@" diff --git a/services/efs-guardian/docker/healthcheck.py b/services/efs-guardian/docker/healthcheck.py new file mode 100755 index 00000000000..10e58d00e21 --- /dev/null +++ b/services/efs-guardian/docker/healthcheck.py @@ -0,0 +1,41 @@ +#!/bin/python +""" Healthcheck script to run inside docker + +Example of usage in a Dockerfile +``` + COPY --chown=scu:scu docker/healthcheck.py docker/healthcheck.py + HEALTHCHECK --interval=30s \ + --timeout=30s \ + --start-period=1s \ + --retries=3 \ + CMD python3 docker/healthcheck.py http://localhost:8000/ +``` + +Q&A: + 1. why not to use curl instead of a python script? + - SEE https://blog.sixeyed.com/docker-healthchecks-why-not-to-use-curl-or-iwr/ +""" + +import os +import sys +from contextlib import suppress +from urllib.request import urlopen + +# Disabled if boots with debugger (e.g. debug, pdb-debug, debug-ptvsd, etc) +SC_BOOT_MODE = os.environ.get("SC_BOOT_MODE", "") + +# Adds a base-path if defined in environ +SIMCORE_NODE_BASEPATH = os.environ.get("SIMCORE_NODE_BASEPATH", "") + + +def is_service_healthy() -> bool: + if "debug" in SC_BOOT_MODE.lower(): + return True + + with suppress(Exception): + with urlopen(f"{sys.argv[1]}{SIMCORE_NODE_BASEPATH}") as f: + return f.getcode() == 200 + return False + + +sys.exit(os.EX_OK if is_service_healthy() else os.EX_UNAVAILABLE) diff --git a/services/efs-guardian/requirements/Makefile b/services/efs-guardian/requirements/Makefile new file mode 100644 index 00000000000..e1319af9d7f --- /dev/null +++ b/services/efs-guardian/requirements/Makefile @@ -0,0 +1,10 @@ +# +# Targets to pip-compile requirements +# +include ../../../requirements/base.Makefile + +# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt + +_base.in: constraints.txt +_test.in: constraints.txt +_tools.in: constraints.txt diff --git a/services/efs-guardian/requirements/_base.in b/services/efs-guardian/requirements/_base.in new file mode 100644 index 00000000000..84e8460fa05 --- /dev/null +++ b/services/efs-guardian/requirements/_base.in @@ -0,0 +1,18 @@ +# +# Specifies third-party dependencies for 'services/efs-guardian/src' +# +# NOTE: ALL version constraints MUST be commented +--constraint ../../../requirements/constraints.txt +--constraint ./constraints.txt + +# intra-repo required dependencies +--requirement ../../../packages/models-library/requirements/_base.in +--requirement ../../../packages/settings-library/requirements/_base.in +--requirement ../../../packages/aws-library/requirements/_base.in +# service-library[fastapi] +--requirement ../../../packages/service-library/requirements/_base.in +--requirement ../../../packages/service-library/requirements/_fastapi.in + + +fastapi +packaging diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt new file mode 100644 index 00000000000..8012fdf97a5 --- /dev/null +++ b/services/efs-guardian/requirements/_base.txt @@ -0,0 +1,175 @@ +aio-pika==9.4.1 +aioboto3==13.0.0 +aiobotocore==2.13.0 + # via aioboto3 +aiocache==0.12.2 +aiodebug==2.3.0 +aiodocker==0.21.0 +aiofiles==23.2.1 + # via aioboto3 +aiohttp==3.9.5 + # via + # aiobotocore + # aiodocker +aioitertools==0.11.0 + # via aiobotocore +aiormq==6.8.0 + # via aio-pika +aiosignal==1.3.1 + # via aiohttp +anyio==4.4.0 + # via + # fast-depends + # faststream + # httpx + # starlette +arrow==1.3.0 +async-timeout==4.0.3 + # via + # aiohttp + # redis +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # referencing +boto3==1.34.106 + # via aiobotocore +botocore==1.34.106 + # via + # aiobotocore + # boto3 + # s3transfer +botocore-stubs==1.34.94 + # via types-aiobotocore +certifi==2024.2.2 + # via + # httpcore + # httpx +click==8.1.7 + # via + # typer + # uvicorn +dnspython==2.6.1 + # via email-validator +email-validator==2.1.1 + # via pydantic +exceptiongroup==1.2.1 + # via anyio +fast-depends==2.4.3 + # via faststream +fastapi==0.99.1 + # via prometheus-fastapi-instrumentator +faststream==0.5.9 +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.5 + # via httpx +httpx==0.27.0 +idna==3.7 + # via + # anyio + # email-validator + # httpx + # yarl +jmespath==1.0.1 + # via + # boto3 + # botocore +jsonschema==4.22.0 +jsonschema-specifications==2023.7.1 + # via jsonschema +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +multidict==6.0.5 + # via + # aiohttp + # yarl +orjson==3.10.3 +packaging==24.0 +pamqp==3.3.0 + # via aiormq +prometheus-client==0.20.0 + # via prometheus-fastapi-instrumentator +prometheus-fastapi-instrumentator==6.1.0 +pydantic==1.10.15 + # via + # fast-depends + # fastapi +pygments==2.18.0 + # via rich +pyinstrument==4.6.2 +python-dateutil==2.9.0.post0 + # via + # arrow + # botocore +pyyaml==6.0.1 +redis==5.0.4 +referencing==0.29.3 + # via + # jsonschema + # jsonschema-specifications +rich==13.7.1 + # via typer +rpds-py==0.18.1 + # via + # jsonschema + # referencing +s3transfer==0.10.1 + # via boto3 +sh==2.0.6 +shellingham==1.5.4 + # via typer +six==1.16.0 + # via python-dateutil +sniffio==1.3.1 + # via + # anyio + # httpx +starlette==0.27.0 + # via fastapi +tenacity==8.3.0 +toolz==0.12.1 +tqdm==4.66.4 +typer==0.12.3 + # via faststream +types-aiobotocore==2.13.0 +types-aiobotocore-ec2==2.13.0 + # via types-aiobotocore +types-aiobotocore-s3==2.13.0 + # via types-aiobotocore +types-awscrt==0.20.9 + # via botocore-stubs +types-python-dateutil==2.9.0.20240316 + # via arrow +typing-extensions==4.11.0 + # via + # aiodebug + # aiodocker + # anyio + # fastapi + # faststream + # pydantic + # typer + # types-aiobotocore + # types-aiobotocore-ec2 + # types-aiobotocore-s3 + # uvicorn +urllib3==2.2.1 + # via botocore +uvicorn==0.30.0 +wrapt==1.16.0 + # via aiobotocore +yarl==1.9.4 + # via + # aio-pika + # aiohttp + # aiormq diff --git a/services/efs-guardian/requirements/_test.in b/services/efs-guardian/requirements/_test.in new file mode 100644 index 00000000000..3d7f73b1cd8 --- /dev/null +++ b/services/efs-guardian/requirements/_test.in @@ -0,0 +1,32 @@ +# +# Specifies dependencies required to run 'services/api-server/test' +# both for unit and integration tests!! +# +--constraint ../../../requirements/constraints.txt +--constraint ./constraints.txt + +# Adds base AS CONSTRAINT specs, not requirement. +# - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt +# +--constraint _base.txt + + +aiodocker +asgi-lifespan +coverage +debugpy +deepdiff +docker +faker +fakeredis[lua] +httpx +moto[server] +parse +psutil +pytest +pytest-asyncio +pytest-cov +pytest-mock +pytest-runner +python-dotenv +respx diff --git a/services/efs-guardian/requirements/_test.txt b/services/efs-guardian/requirements/_test.txt new file mode 100644 index 00000000000..0a40d9e8f25 --- /dev/null +++ b/services/efs-guardian/requirements/_test.txt @@ -0,0 +1,267 @@ +aiodocker==0.21.0 +aiohttp==3.9.5 + # via aiodocker +aiosignal==1.3.1 + # via aiohttp +antlr4-python3-runtime==4.13.1 + # via moto +anyio==4.4.0 + # via httpx +asgi-lifespan==2.1.0 +async-timeout==4.0.3 + # via + # aiohttp + # redis +attrs==23.2.0 + # via + # aiohttp + # jschema-to-python + # jsonschema + # referencing + # sarif-om +aws-sam-translator==1.89.0 + # via cfn-lint +aws-xray-sdk==2.13.1 + # via moto +blinker==1.8.2 + # via flask +boto3==1.34.106 + # via + # aws-sam-translator + # moto +botocore==1.34.106 + # via + # aws-xray-sdk + # boto3 + # moto + # s3transfer +certifi==2024.2.2 + # via + # httpcore + # httpx + # requests +cffi==1.16.0 + # via cryptography +cfn-lint==0.87.3 + # via moto +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via flask +coverage==7.5.3 + # via pytest-cov +cryptography==42.0.7 + # via + # joserfc + # moto +debugpy==1.8.1 +deepdiff==7.0.1 +docker==7.1.0 + # via moto +exceptiongroup==1.2.1 + # via + # anyio + # pytest +faker==25.3.0 +fakeredis==2.23.2 +flask==3.0.3 + # via + # flask-cors + # moto +flask-cors==4.0.1 + # via moto +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal +graphql-core==3.2.3 + # via moto +h11==0.14.0 + # via httpcore +httpcore==1.0.5 + # via httpx +httpx==0.27.0 + # via respx +idna==3.7 + # via + # anyio + # httpx + # requests + # yarl +iniconfig==2.0.0 + # via pytest +itsdangerous==2.2.0 + # via flask +jinja2==3.1.4 + # via + # flask + # moto +jmespath==1.0.1 + # via + # boto3 + # botocore +joserfc==0.10.0 + # via moto +jschema-to-python==1.2.3 + # via cfn-lint +jsondiff==2.0.0 + # via moto +jsonpatch==1.33 + # via cfn-lint +jsonpath-ng==1.6.1 + # via moto +jsonpickle==3.0.4 + # via jschema-to-python +jsonpointer==2.4 + # via jsonpatch +jsonschema==4.22.0 + # via + # aws-sam-translator + # cfn-lint + # openapi-schema-validator + # openapi-spec-validator +jsonschema-path==0.3.2 + # via openapi-spec-validator +jsonschema-specifications==2023.7.1 + # via + # jsonschema + # openapi-schema-validator +junit-xml==1.9 + # via cfn-lint +lazy-object-proxy==1.10.0 + # via openapi-spec-validator +lupa==2.1 + # via fakeredis +markupsafe==2.1.5 + # via + # jinja2 + # werkzeug +moto==5.0.8 +mpmath==1.3.0 + # via sympy +multidict==6.0.5 + # via + # aiohttp + # yarl +networkx==3.3 + # via cfn-lint +openapi-schema-validator==0.6.2 + # via openapi-spec-validator +openapi-spec-validator==0.7.1 + # via moto +ordered-set==4.1.0 + # via deepdiff +packaging==24.0 + # via pytest +parse==1.20.1 +pathable==0.4.3 + # via jsonschema-path +pbr==6.0.0 + # via + # jschema-to-python + # sarif-om +pluggy==1.5.0 + # via pytest +ply==3.11 + # via jsonpath-ng +psutil==5.9.8 +py-partiql-parser==0.5.5 + # via moto +pycparser==2.22 + # via cffi +pydantic==1.10.15 + # via aws-sam-translator +pyparsing==3.1.2 + # via moto +pytest==8.2.1 + # via + # pytest-asyncio + # pytest-cov + # pytest-mock +pytest-asyncio==0.21.2 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-runner==6.0.1 +python-dateutil==2.9.0.post0 + # via + # botocore + # faker + # moto +python-dotenv==1.0.1 +pyyaml==6.0.1 + # via + # cfn-lint + # jsonschema-path + # moto + # responses +redis==5.0.4 + # via fakeredis +referencing==0.29.3 + # via + # jsonschema + # jsonschema-path + # jsonschema-specifications +regex==2024.5.15 + # via cfn-lint +requests==2.32.2 + # via + # docker + # jsonschema-path + # moto + # responses +responses==0.25.0 + # via moto +respx==0.21.1 +rfc3339-validator==0.1.4 + # via openapi-schema-validator +rpds-py==0.18.1 + # via + # jsonschema + # referencing +s3transfer==0.10.1 + # via boto3 +sarif-om==1.0.4 + # via cfn-lint +setuptools==70.0.0 + # via moto +six==1.16.0 + # via + # junit-xml + # python-dateutil + # rfc3339-validator +sniffio==1.3.1 + # via + # anyio + # asgi-lifespan + # httpx +sortedcontainers==2.4.0 + # via fakeredis +sympy==1.12 + # via cfn-lint +tomli==2.0.1 + # via + # coverage + # pytest +typing-extensions==4.11.0 + # via + # aiodocker + # anyio + # aws-sam-translator + # fakeredis + # pydantic +urllib3==2.2.1 + # via + # botocore + # docker + # requests + # responses +werkzeug==3.0.3 + # via + # flask + # moto +wrapt==1.16.0 + # via aws-xray-sdk +xmltodict==0.13.0 + # via moto +yarl==1.9.4 + # via aiohttp diff --git a/services/efs-guardian/requirements/_tools.in b/services/efs-guardian/requirements/_tools.in new file mode 100644 index 00000000000..52a9a39d162 --- /dev/null +++ b/services/efs-guardian/requirements/_tools.in @@ -0,0 +1,7 @@ +--constraint ../../../requirements/constraints.txt +--constraint _base.txt +--constraint _test.txt + +--requirement ../../../requirements/devenv.txt + +watchdog[watchmedo] diff --git a/services/efs-guardian/requirements/_tools.txt b/services/efs-guardian/requirements/_tools.txt new file mode 100644 index 00000000000..a141a791764 --- /dev/null +++ b/services/efs-guardian/requirements/_tools.txt @@ -0,0 +1,74 @@ +astroid==3.2.2 + # via pylint +black==24.4.2 +build==1.2.1 + # via pip-tools +bump2version==1.0.1 +cfgv==3.4.0 + # via pre-commit +click==8.1.7 + # via + # black + # pip-tools +dill==0.3.8 + # via pylint +distlib==0.3.8 + # via virtualenv +filelock==3.14.0 + # via virtualenv +identify==2.5.36 + # via pre-commit +isort==5.13.2 + # via pylint +mccabe==0.7.0 + # via pylint +mypy-extensions==1.0.0 + # via black +nodeenv==1.8.0 + # via pre-commit +packaging==24.0 + # via + # black + # build +pathspec==0.12.1 + # via black +pip==24.0 + # via pip-tools +pip-tools==7.4.1 +platformdirs==4.2.2 + # via + # black + # pylint + # virtualenv +pre-commit==3.7.1 +pylint==3.2.2 +pyproject-hooks==1.1.0 + # via + # build + # pip-tools +pyyaml==6.0.1 + # via + # pre-commit + # watchdog +ruff==0.4.5 +setuptools==70.0.0 + # via + # nodeenv + # pip-tools +tomli==2.0.1 + # via + # black + # build + # pip-tools + # pylint +tomlkit==0.12.5 + # via pylint +typing-extensions==4.11.0 + # via + # astroid + # black +virtualenv==20.26.2 + # via pre-commit +watchdog==4.0.1 +wheel==0.43.0 + # via pip-tools diff --git a/services/efs-guardian/requirements/ci.txt b/services/efs-guardian/requirements/ci.txt new file mode 100644 index 00000000000..85e9fca927f --- /dev/null +++ b/services/efs-guardian/requirements/ci.txt @@ -0,0 +1,21 @@ +# Shortcut to install all packages for the contigous integration (CI) of 'services/efs-guardian' +# +# - As ci.txt but w/ tests +# +# Usage: +# pip install -r requirements/ci.txt +# + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt + +# installs this repo's packages +simcore-aws-library @ ../../packages/aws-library +simcore-models-library @ ../../packages/models-library +pytest-simcore @ ../../packages/pytest-simcore +simcore-service-library[fastapi] @ ../../packages/service-library +simcore-settings-library @ ../../packages/settings-library + +# installs current package +simcore-service-efs-guardian @ . diff --git a/services/efs-guardian/requirements/constraints.txt b/services/efs-guardian/requirements/constraints.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/efs-guardian/requirements/dev.txt b/services/efs-guardian/requirements/dev.txt new file mode 100644 index 00000000000..76ea75d980d --- /dev/null +++ b/services/efs-guardian/requirements/dev.txt @@ -0,0 +1,22 @@ +# Shortcut to install all packages needed to develop 'services/efs-guardian' +# +# - As ci.txt but with current and repo packages in develop (edit) mode +# +# Usage: +# pip install -r requirements/dev.txt +# + +# installs base + tests + tools requirements +--requirement _base.txt +--requirement _test.txt +--requirement _tools.txt + +# installs this repo's packages +--editable ../../packages/aws-library +--editable ../../packages/models-library +--editable ../../packages/pytest-simcore +--editable ../../packages/service-library[fastapi] +--editable ../../packages/settings-library + +# installs current package +--editable . diff --git a/services/efs-guardian/requirements/prod.txt b/services/efs-guardian/requirements/prod.txt new file mode 100644 index 00000000000..0a75d60f13f --- /dev/null +++ b/services/efs-guardian/requirements/prod.txt @@ -0,0 +1,18 @@ +# Shortcut to install 'services/efs-guardian' for production +# +# - As ci.txt but w/o tests +# +# Usage: +# pip install -r requirements/prod.txt +# + +# installs base requirements +--requirement _base.txt + +# installs this repo's packages +simcore-aws-library @ ../../packages/aws-library +simcore-models-library @ ../../packages/models-library +simcore-service-library[fastapi] @ ../../packages/service-library +simcore-settings-library @ ../../packages/settings-library +# installs current package +simcore-service-efs-guardian @ . diff --git a/services/efs-guardian/setup.cfg b/services/efs-guardian/setup.cfg new file mode 100644 index 00000000000..34c42997769 --- /dev/null +++ b/services/efs-guardian/setup.cfg @@ -0,0 +1,13 @@ +[bumpversion] +current_version = 1.0.0 +commit = True +message = services/efs-guardian version: {current_version} → {new_version} +tag = False +commit_args = --no-verify + +[bumpversion:file:VERSION] + +[tool:pytest] +asyncio_mode = auto +markers = + testit: "marks test to run during development" diff --git a/services/efs-guardian/setup.py b/services/efs-guardian/setup.py new file mode 100755 index 00000000000..ed3f29fc23b --- /dev/null +++ b/services/efs-guardian/setup.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import re +import sys +from pathlib import Path + +from setuptools import find_packages, setup + + +def read_reqs(reqs_path: Path) -> set[str]: + return { + r + for r in re.findall( + r"(^[^#\n-][\w\[,\]]+[-~>=<.\w]*)", + reqs_path.read_text(), + re.MULTILINE, + ) + if isinstance(r, str) + } + + +CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + +NAME = "simcore-service-efs-guardian" +VERSION = (CURRENT_DIR / "VERSION").read_text().strip() +AUTHORS = ("Matus Drobuliak (drobuliakmatus66)",) +DESCRIPTION = "Service to monitor and manage elastic file system" +README = (CURRENT_DIR / "README.md").read_text() + +PROD_REQUIREMENTS = tuple( + read_reqs(CURRENT_DIR / "requirements" / "_base.txt") + | { + "simcore-aws-library", + "simcore-models-library", + "simcore-service-library[fastapi]", + "simcore-settings-library", + } +) + +TEST_REQUIREMENTS = tuple(read_reqs(CURRENT_DIR / "requirements" / "_test.txt")) + +SETUP = { + "name": NAME, + "version": VERSION, + "author": AUTHORS, + "description": DESCRIPTION, + "long_description": README, + "license": "MIT license", + "python_requires": "~=3.10", + "packages": find_packages(where="src"), + "package_dir": { + "": "src", + }, + "package_data": {"": ["data/*.yml"]}, + "include_package_data": True, + "install_requires": PROD_REQUIREMENTS, + "test_suite": "tests", + "tests_require": TEST_REQUIREMENTS, + "extras_require": {"test": TEST_REQUIREMENTS}, + "entry_points": { + "console_scripts": [ + "simcore-service-efs-guardian = simcore_service_efs_guardian.cli:main", + "simcore-service = simcore_service_efs_guardian.cli:main", + ], + }, +} + +if __name__ == "__main__": + setup(**SETUP) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/__init__.py b/services/efs-guardian/src/simcore_service_efs_guardian/__init__.py new file mode 100644 index 00000000000..f513c971cca --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/__init__.py @@ -0,0 +1,3 @@ +from ._meta import __version__ + +assert __version__ # nosec diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/_meta.py b/services/efs-guardian/src/simcore_service_efs_guardian/_meta.py new file mode 100644 index 00000000000..27ec8aad7a6 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/_meta.py @@ -0,0 +1,65 @@ +""" Application's metadata + +""" + +from importlib.metadata import distribution, version +from importlib.resources import files +from pathlib import Path +from typing import Final + +from models_library.basic_types import VersionTag +from packaging.version import Version +from pydantic import parse_obj_as + +_current_distribution = distribution("simcore-service-efs-guardian") +__version__: str = version("simcore-service-efs-guardian") + + +APP_NAME: Final[str] = _current_distribution.metadata["Name"] +API_VERSION: Final[str] = __version__ +VERSION: Final[Version] = Version(__version__) +API_VTAG: Final[VersionTag] = parse_obj_as(VersionTag, f"v{VERSION.major}") +RPC_VTAG: Final[VersionTag] = parse_obj_as(VersionTag, f"v{VERSION.major}") + + +def get_summary() -> str: + return _current_distribution.metadata.get_all("Summary", [""])[-1] + + +SUMMARY: Final[str] = get_summary() +PACKAGE_DATA_FOLDER: Final[Path] = Path(f'{files(APP_NAME.replace("-", "_")) / "data"}') + +# https://patorjk.com/software/taag/#p=display&f=ANSI%20Shadow&t=Elastic%20file%0Asystem%20guardian +APP_STARTED_BANNER_MSG = r""" +███████╗██╗ █████╗ ███████╗████████╗██╗ ██████╗ ███████╗██╗██╗ ███████╗ +██╔════╝██║ ██╔══██╗██╔════╝╚══██╔══╝██║██╔════╝ ██╔════╝██║██║ ██╔════╝ +█████╗ ██║ ███████║███████╗ ██║ ██║██║ █████╗ ██║██║ █████╗ +██╔══╝ ██║ ██╔══██║╚════██║ ██║ ██║██║ ██╔══╝ ██║██║ ██╔══╝ +███████╗███████╗██║ ██║███████║ ██║ ██║╚██████╗ ██║ ██║███████╗███████╗ +╚══════╝╚══════╝╚═╝ ╚═╝╚══════╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝╚══════╝╚══════╝ + +███████╗██╗ ██╗███████╗████████╗███████╗███╗ ███╗ ██████╗ ██╗ ██╗ █████╗ ██████╗ ██████╗ ██╗ █████╗ ███╗ ██╗ +██╔════╝╚██╗ ██╔╝██╔════╝╚══██╔══╝██╔════╝████╗ ████║ ██╔════╝ ██║ ██║██╔══██╗██╔══██╗██╔══██╗██║██╔══██╗████╗ ██║ +███████╗ ╚████╔╝ ███████╗ ██║ █████╗ ██╔████╔██║ ██║ ███╗██║ ██║███████║██████╔╝██║ ██║██║███████║██╔██╗ ██║ +╚════██║ ╚██╔╝ ╚════██║ ██║ ██╔══╝ ██║╚██╔╝██║ ██║ ██║██║ ██║██╔══██║██╔══██╗██║ ██║██║██╔══██║██║╚██╗██║ +███████║ ██║ ███████║ ██║ ███████╗██║ ╚═╝ ██║ ╚██████╔╝╚██████╔╝██║ ██║██║ ██║██████╔╝██║██║ ██║██║ ╚████║ +╚══════╝ ╚═╝ ╚══════╝ ╚═╝ ╚══════╝╚═╝ ╚═╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚═════╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═══╝ + 🛡️ Welcome to EFS-Guardian App 🛡️ + Your Elastic File System Manager & Monitor + {} +""".format( + f"v{__version__}" +) + +APP_STARTED_DISABLED_BANNER_MSG = r""" +██████╗ ██╗███████╗ █████╗ ██████╗ ██╗ ███████╗██████╗ +██╔══██╗██║██╔════╝██╔══██╗██╔══██╗██║ ██╔════╝██╔══██╗ +██║ ██║██║███████╗███████║██████╔╝██║ █████╗ ██║ ██║ +██║ ██║██║╚════██║██╔══██║██╔══██╗██║ ██╔══╝ ██║ ██║ +██████╔╝██║███████║██║ ██║██████╔╝███████╗███████╗██████╔╝ +╚═════╝ ╚═╝╚══════╝╚═╝ ╚═╝╚═════╝ ╚══════╝╚══════╝╚═════╝ +""" + +APP_FINISHED_BANNER_MSG = "{:=^100}".format( + f"🎉 App {APP_NAME}=={__version__} shutdown completed 🎉" +) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/api/__init__.py b/services/efs-guardian/src/simcore_service_efs_guardian/api/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/api/rest/__init__.py b/services/efs-guardian/src/simcore_service_efs_guardian/api/rest/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/api/rest/health.py b/services/efs-guardian/src/simcore_service_efs_guardian/api/rest/health.py new file mode 100644 index 00000000000..2c6f160a9e8 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/api/rest/health.py @@ -0,0 +1,18 @@ +""" +All entrypoints used for operations + +for instance: service health-check (w/ different variants), diagnostics, debugging, status, etc +""" + +import datetime + +from fastapi import APIRouter +from fastapi.responses import PlainTextResponse + +router = APIRouter() + + +@router.get("/", include_in_schema=True, response_class=PlainTextResponse) +async def health_check(): + # NOTE: sync url in docker/healthcheck.py with this entrypoint! + return f"{__name__}.health_check@{datetime.datetime.now(datetime.timezone.utc).isoformat()}" diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/api/rest/routes.py b/services/efs-guardian/src/simcore_service_efs_guardian/api/rest/routes.py new file mode 100644 index 00000000000..af7eef7aa26 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/api/rest/routes.py @@ -0,0 +1,17 @@ +from fastapi import APIRouter, FastAPI + +from ..._meta import API_VTAG +from . import health + + +def setup_api_routes(app: FastAPI): + """ + Composes resources/sub-resources routers + """ + router = APIRouter() + + # include operations in / + app.include_router(health.router, tags=["operations"]) + + # include the rest under /vX + app.include_router(router, prefix=f"/{API_VTAG}") diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/__init__.py b/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/rpc_routes.py b/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/rpc_routes.py new file mode 100644 index 00000000000..c79ed1f7ed3 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/api/rpc/rpc_routes.py @@ -0,0 +1,22 @@ +from collections.abc import Awaitable, Callable + +from fastapi import FastAPI + + +def on_app_startup(app: FastAPI) -> Callable[[], Awaitable[None]]: + async def _start() -> None: + assert app # nosec + + return _start + + +def on_app_shutdown(app: FastAPI) -> Callable[[], Awaitable[None]]: + async def _stop() -> None: + assert app # nosec + + return _stop + + +def setup_rpc_routes(app: FastAPI) -> None: + app.add_event_handler("startup", on_app_startup(app)) + app.add_event_handler("shutdown", on_app_shutdown(app)) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/cli.py b/services/efs-guardian/src/simcore_service_efs_guardian/cli.py new file mode 100644 index 00000000000..77d18015ec0 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/cli.py @@ -0,0 +1,24 @@ +import logging + +import typer +from settings_library.utils_cli import create_settings_command + +from ._meta import APP_NAME +from .core.settings import ApplicationSettings + +log = logging.getLogger(__name__) + +# NOTE: 'main' variable is referred in the setup's entrypoint! +main = typer.Typer(name=APP_NAME) + +main.command()(create_settings_command(settings_cls=ApplicationSettings, logger=log)) + + +@main.command() +def run(): + """Runs application""" + typer.secho("Sorry, this entrypoint is intentionally disabled. Use instead") + typer.secho( + "$ uvicorn simcore_service_efs_guardian.main:the_app", + fg=typer.colors.BLUE, + ) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/__init__.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py new file mode 100644 index 00000000000..da0d9deb0d2 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py @@ -0,0 +1,59 @@ +import logging + +from fastapi import FastAPI + +from .._meta import ( + API_VERSION, + API_VTAG, + APP_FINISHED_BANNER_MSG, + APP_NAME, + APP_STARTED_BANNER_MSG, + APP_STARTED_DISABLED_BANNER_MSG, +) +from ..api.rest.routes import setup_api_routes +from ..api.rpc.rpc_routes import setup_rpc_routes +from .settings import ApplicationSettings + +logger = logging.getLogger(__name__) + + +def create_app(settings: ApplicationSettings) -> FastAPI: + logger.info("app settings: %s", settings.json(indent=1)) + + app = FastAPI( + debug=settings.EFS_GUARDIAN_DEBUG, + title=APP_NAME, + description="Service to monitor and manage elastic file system", + version=API_VERSION, + openapi_url=f"/api/{API_VTAG}/openapi.json", + docs_url="/dev/doc", + redoc_url=None, # default disabled + ) + # STATE + app.state.settings = settings + assert app.state.settings.API_VERSION == API_VERSION # nosec + + # PLUGINS SETUP + setup_api_routes(app) + setup_rpc_routes(app) + + # ERROR HANDLERS + + # EVENTS + async def _on_startup() -> None: + print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 + if any( + s is None + for s in [ + settings.EFS_GUARDIAN_AWS_EFS_SETTINGS, + ] + ): + print(APP_STARTED_DISABLED_BANNER_MSG, flush=True) # noqa: T201 + + async def _on_shutdown() -> None: + print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) + + return app diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py new file mode 100644 index 00000000000..aedbca71f0c --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -0,0 +1,85 @@ +from functools import cached_property +from typing import Final, cast + +from fastapi import FastAPI +from models_library.basic_types import ( + BootModeEnum, + BuildTargetEnum, + LogLevel, + VersionTag, +) +from pydantic import Field, PositiveInt, validator +from settings_library.base import BaseCustomSettings +from settings_library.utils_logging import MixinLoggingSettings + +from .._meta import API_VERSION, API_VTAG, APP_NAME + +EFS_GUARDIAN_ENV_PREFIX: Final[str] = "EFS_GUARDIAN_" + + +class AwsEfsSettings(BaseCustomSettings): + EFS_DNS_NAME: str = Field( + description="AWS Elastic File System DNS name", + example="fs-xxx.efs.us-east-1.amazonaws.com", + ) + EFS_BASE_DIRECTORY: str = Field(default="project-specific-data") + + +class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): + # CODE STATICS --------------------------------------------------------- + API_VERSION: str = API_VERSION + APP_NAME: str = APP_NAME + API_VTAG: VersionTag = API_VTAG + + # IMAGE BUILDTIME ------------------------------------------------------ + # @Makefile + SC_BUILD_DATE: str | None = None + SC_BUILD_TARGET: BuildTargetEnum | None = None + SC_VCS_REF: str | None = None + SC_VCS_URL: str | None = None + + # @Dockerfile + SC_BOOT_MODE: BootModeEnum | None = None + SC_BOOT_TARGET: BuildTargetEnum | None = None + SC_HEALTHCHECK_TIMEOUT: PositiveInt | None = Field( + None, + description="If a single run of the check takes longer than timeout seconds " + "then the check is considered to have failed." + "It takes retries consecutive failures of the health check for the container to be considered unhealthy.", + ) + SC_USER_ID: int | None = None + SC_USER_NAME: str | None = None + + # RUNTIME ----------------------------------------------------------- + EFS_GUARDIAN_DEBUG: bool = Field( + default=False, description="Debug mode", env=["EFS_GUARDIAN_DEBUG", "DEBUG"] + ) + EFS_GUARDIAN_LOGLEVEL: LogLevel = Field( + LogLevel.INFO, env=["EFS_GUARDIAN_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"] + ) + EFS_GUARDIAN_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( + default=False, + env=[ + "EFS_GUARDIAN_LOG_FORMAT_LOCAL_DEV_ENABLED", + "LOG_FORMAT_LOCAL_DEV_ENABLED", + ], + description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", + ) + + EFS_GUARDIAN_AWS_EFS_SETTINGS: AwsEfsSettings | None = Field( + auto_default_from_env=True + ) + + @cached_property + def LOG_LEVEL(self) -> LogLevel: # noqa: N802 + return self.EFS_GUARDIAN_LOGLEVEL + + @validator("EFS_GUARDIAN_LOGLEVEL") + @classmethod + def valid_log_level(cls, value: str) -> str: + # NOTE: mypy is not happy without the cast + return cast(str, cls.validate_log_level(value)) + + +def get_application_settings(app: FastAPI) -> ApplicationSettings: + return cast(ApplicationSettings, app.state.settings) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/__init__.py b/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/__init__.py new file mode 100644 index 00000000000..b6036dda040 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/__init__.py @@ -0,0 +1,5 @@ +from . import handlers + +setup_exception_handlers = handlers.setup + +__all__: tuple[str, ...] = ("setup_exception_handlers",) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/_base.py b/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/_base.py new file mode 100644 index 00000000000..61a92118c92 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/_base.py @@ -0,0 +1,8 @@ +from typing import Any + +from models_library.errors_classes import OsparcErrorMixin + + +class EfsGuardianBaseError(OsparcErrorMixin, Exception): + def __init__(self, **ctx: Any) -> None: + super().__init__(**ctx) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/custom_errors.py b/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/custom_errors.py new file mode 100644 index 00000000000..ca702657f53 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/custom_errors.py @@ -0,0 +1,9 @@ +from ._base import EfsGuardianBaseError + + +class CustomBaseError(EfsGuardianBaseError): + pass + + +class ApplicationSetupError(CustomBaseError): + pass diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/handlers/__init__.py b/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/handlers/__init__.py new file mode 100644 index 00000000000..f9a5aefe592 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/handlers/__init__.py @@ -0,0 +1,7 @@ +# pylint: disable=unused-argument + +from fastapi import FastAPI + + +def setup(app: FastAPI, *, is_debug: bool = False): + ... diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/main.py b/services/efs-guardian/src/simcore_service_efs_guardian/main.py new file mode 100644 index 00000000000..6ab24933b02 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/main.py @@ -0,0 +1,17 @@ +"""Main application to be deployed by uvicorn (or equivalent) server + +""" +import logging + +from fastapi import FastAPI +from servicelib.logging_utils import config_all_loggers +from simcore_service_efs_guardian.core.application import create_app +from simcore_service_efs_guardian.core.settings import ApplicationSettings + +the_settings = ApplicationSettings.create_from_envs() +logging.basicConfig(level=the_settings.log_level) +logging.root.setLevel(the_settings.log_level) +config_all_loggers(the_settings.EFS_GUARDIAN_LOG_FORMAT_LOCAL_DEV_ENABLED) + +# SINGLETON FastAPI app +the_app: FastAPI = create_app(the_settings) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/__init__.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/efs-guardian/tests/integration/.gitkeep b/services/efs-guardian/tests/integration/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/efs-guardian/tests/unit/conftest.py b/services/efs-guardian/tests/unit/conftest.py new file mode 100644 index 00000000000..9c53ab29a3f --- /dev/null +++ b/services/efs-guardian/tests/unit/conftest.py @@ -0,0 +1,117 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import re +from collections.abc import AsyncIterator +from pathlib import Path + +import httpx +import pytest +import simcore_service_efs_guardian +import yaml +from asgi_lifespan import LifespanManager +from fastapi import FastAPI +from httpx import ASGITransport +from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict +from simcore_service_efs_guardian.core.application import create_app +from simcore_service_efs_guardian.core.settings import ApplicationSettings + +pytest_plugins = [ + "pytest_simcore.cli_runner", + "pytest_simcore.environment_configs", + "pytest_simcore.repository_paths", +] + + +@pytest.fixture(scope="session") +def project_slug_dir(osparc_simcore_root_dir: Path) -> Path: + # fixtures in pytest_simcore.environs + service_folder = osparc_simcore_root_dir / "services" / "efs_guardian" + assert service_folder.exists() + assert any(service_folder.glob("src/simcore_service_efs_guardian")) + return service_folder + + +@pytest.fixture(scope="session") +def installed_package_dir() -> Path: + dirpath = Path(simcore_service_efs_guardian.__file__).resolve().parent + assert dirpath.exists() + return dirpath + + +@pytest.fixture +def docker_compose_service_efs_guardian_env_vars( + services_docker_compose_file: Path, + env_devel_dict: EnvVarsDict, +) -> EnvVarsDict: + """env vars injected at the docker-compose""" + + payments = yaml.safe_load(services_docker_compose_file.read_text())["services"][ + "efs-guardian" + ] + + def _substitute(key, value): + if m := re.match(r"\${([^{}:-]\w+)", value): + expected_env_var = m.group(1) + try: + # NOTE: if this raises, then the RHS env-vars in the docker-compose are + # not defined in the env-devel + if value := env_devel_dict[expected_env_var]: + return key, value + except KeyError: + pytest.fail( + f"{expected_env_var} is not defined in .env-devel but used in docker-compose services[{payments}].environment[{key}]" + ) + return None + + envs: EnvVarsDict = {} + for key, value in payments.get("environment", {}).items(): + if found := _substitute(key, value): + _, new_value = found + envs[key] = new_value + + return envs + + +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + docker_compose_service_efs_guardian_env_vars: EnvVarsDict, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + **docker_compose_service_efs_guardian_env_vars, + }, + ) + + +@pytest.fixture +def app_settings(app_environment: EnvVarsDict) -> ApplicationSettings: + settings = ApplicationSettings.create_from_envs() + return settings + + +@pytest.fixture +async def app(app_settings: ApplicationSettings) -> AsyncIterator[FastAPI]: + the_test_app = create_app(app_settings) + async with LifespanManager( + the_test_app, + ): + yield the_test_app + + +@pytest.fixture +async def client(app: FastAPI) -> AsyncIterator[httpx.AsyncClient]: + # - Needed for app to trigger start/stop event handlers + # - Prefer this client instead of fastapi.testclient.TestClient + async with httpx.AsyncClient( + app=app, + base_url="http://efs-guardian.testserver.io", + headers={"Content-Type": "application/json"}, + ) as client: + assert isinstance( + client._transport, ASGITransport # pylint: disable=protected-access + ) + yield client diff --git a/services/efs-guardian/tests/unit/test_api_health.py b/services/efs-guardian/tests/unit/test_api_health.py new file mode 100644 index 00000000000..791fb2bee26 --- /dev/null +++ b/services/efs-guardian/tests/unit/test_api_health.py @@ -0,0 +1,13 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import httpx +from starlette import status + + +async def test_healthcheck(client: httpx.AsyncClient): + response = await client.get("/") + response.raise_for_status() + assert response.status_code == status.HTTP_200_OK + assert "simcore_service_efs_guardian" in response.text diff --git a/services/efs-guardian/tests/unit/test_cli.py b/services/efs-guardian/tests/unit/test_cli.py new file mode 100644 index 00000000000..6819ed50a41 --- /dev/null +++ b/services/efs-guardian/tests/unit/test_cli.py @@ -0,0 +1,21 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + + +from simcore_service_efs_guardian.cli import main +from typer.testing import CliRunner + +runner = CliRunner() + + +def test_settings(app_environment): + result = runner.invoke(main, ["settings"]) + assert result.exit_code == 0 + assert "APP_NAME=simcore-service-efs-guardian" in result.stdout + + +def test_run(): + result = runner.invoke(main, ["run"]) + assert result.exit_code == 0 + assert "disabled" in result.stdout diff --git a/services/efs-guardian/tests/unit/test_core_settings.py b/services/efs-guardian/tests/unit/test_core_settings.py new file mode 100644 index 00000000000..a3496b381b5 --- /dev/null +++ b/services/efs-guardian/tests/unit/test_core_settings.py @@ -0,0 +1,12 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +from pytest_simcore.helpers.utils_envs import EnvVarsDict +from simcore_service_efs_guardian.core.settings import ApplicationSettings + + +def test_settings(app_environment: EnvVarsDict): + settings = ApplicationSettings.create_from_envs() + assert settings diff --git a/services/efs-guardian/tests/unit/test_main.py b/services/efs-guardian/tests/unit/test_main.py new file mode 100644 index 00000000000..475673488be --- /dev/null +++ b/services/efs-guardian/tests/unit/test_main.py @@ -0,0 +1,12 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + + +from pytest_simcore.helpers.utils_envs import EnvVarsDict + + +def test_main_app(app_environment: EnvVarsDict): + from simcore_service_efs_guardian.main import the_app, the_settings + + assert the_app.state.settings == the_settings diff --git a/tests/swarm-deploy/test_service_restart.py b/tests/swarm-deploy/test_service_restart.py index 93c081b3d67..d07a20b8a10 100644 --- a/tests/swarm-deploy/test_service_restart.py +++ b/tests/swarm-deploy/test_service_restart.py @@ -20,6 +20,7 @@ ("dask-sidecar", 0), ("datcore-adapter", 0), ("director-v2", 0), + ("efs-guardian", 0), ("migration", 143), ("static-webserver", 15), ("storage", 0),