From 3cd301a24e286ff9601922e83eb522256c9e3b26 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 08:51:37 +0200 Subject: [PATCH 001/102] moved service-sidecar contents --- services/service-sidecar/CHANGELOG.md | 13 + services/service-sidecar/Dockerfile | 147 +++++++++ services/service-sidecar/Makefile | 44 +++ services/service-sidecar/VERSION | 1 + services/service-sidecar/docker/boot.sh | 36 +++ services/service-sidecar/docker/entrypoint.sh | 95 ++++++ .../service-sidecar/docker/healthcheck.py | 40 +++ .../service-sidecar/requirements/Makefile | 6 + .../service-sidecar/requirements/_base.in | 19 ++ .../service-sidecar/requirements/_base.txt | 167 +++++++++++ .../service-sidecar/requirements/_test.in | 8 + .../service-sidecar/requirements/_test.txt | 68 +++++ .../service-sidecar/requirements/_tools.in | 12 + .../service-sidecar/requirements/_tools.txt | 118 ++++++++ services/service-sidecar/requirements/ci.txt | 18 ++ services/service-sidecar/requirements/dev.txt | 20 ++ .../service-sidecar/requirements/prod.txt | 17 ++ services/service-sidecar/setup.py | 42 +++ .../__init__.py | 0 .../api/__init__.py | 1 + .../api/_routing.py | 23 ++ .../api/compose.py | 167 +++++++++++ .../api/container.py | 97 ++++++ .../api/containers.py | 76 +++++ .../api/health.py | 13 + .../api/push.py | 18 ++ .../api/retrive.py | 24 ++ .../api/state.py | 24 ++ .../application.py | 51 ++++ .../simcore_service_service_sidecar/main.py | 31 ++ .../simcore_service_service_sidecar/models.py | 7 + .../remote_debug.py | 33 ++ .../settings.py | 87 ++++++ .../shared_handlers.py | 64 ++++ .../storage.py | 51 ++++ .../simcore_service_service_sidecar/utils.py | 282 ++++++++++++++++++ services/service-sidecar/tests/conftest.py | 102 +++++++ .../tests/unit/test_api_compose.py | 149 +++++++++ .../tests/unit/test_api_container.py | 149 +++++++++ .../tests/unit/test_api_containers.py | 134 +++++++++ .../tests/unit/test_api_health.py | 10 + .../tests/unit/test_api_push_retrive_state.py | 34 +++ services/service-sidecar/tox.ini | 57 ++++ 43 files changed, 2555 insertions(+) create mode 100644 services/service-sidecar/CHANGELOG.md create mode 100644 services/service-sidecar/Dockerfile create mode 100644 services/service-sidecar/Makefile create mode 100644 services/service-sidecar/VERSION create mode 100755 services/service-sidecar/docker/boot.sh create mode 100755 services/service-sidecar/docker/entrypoint.sh create mode 100644 services/service-sidecar/docker/healthcheck.py create mode 100644 services/service-sidecar/requirements/Makefile create mode 100644 services/service-sidecar/requirements/_base.in create mode 100644 services/service-sidecar/requirements/_base.txt create mode 100644 services/service-sidecar/requirements/_test.in create mode 100644 services/service-sidecar/requirements/_test.txt create mode 100644 services/service-sidecar/requirements/_tools.in create mode 100644 services/service-sidecar/requirements/_tools.txt create mode 100644 services/service-sidecar/requirements/ci.txt create mode 100644 services/service-sidecar/requirements/dev.txt create mode 100644 services/service-sidecar/requirements/prod.txt create mode 100644 services/service-sidecar/setup.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/__init__.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/api/__init__.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/api/_routing.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/api/compose.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/api/container.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/api/containers.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/api/health.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/api/push.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/api/retrive.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/api/state.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/application.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/main.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/models.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/remote_debug.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/settings.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/shared_handlers.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/storage.py create mode 100644 services/service-sidecar/src/simcore_service_service_sidecar/utils.py create mode 100644 services/service-sidecar/tests/conftest.py create mode 100644 services/service-sidecar/tests/unit/test_api_compose.py create mode 100644 services/service-sidecar/tests/unit/test_api_container.py create mode 100644 services/service-sidecar/tests/unit/test_api_containers.py create mode 100644 services/service-sidecar/tests/unit/test_api_health.py create mode 100644 services/service-sidecar/tests/unit/test_api_push_retrive_state.py create mode 100644 services/service-sidecar/tox.ini diff --git a/services/service-sidecar/CHANGELOG.md b/services/service-sidecar/CHANGELOG.md new file mode 100644 index 00000000000..692f1de6841 --- /dev/null +++ b/services/service-sidecar/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.0.1] - 2021-04-14 +### Added +- First working version of the service +- FastAPI based service +- all spawned services and networks are removed when receiving `SIGTERM` diff --git a/services/service-sidecar/Dockerfile b/services/service-sidecar/Dockerfile new file mode 100644 index 00000000000..5dc5b1d9c9f --- /dev/null +++ b/services/service-sidecar/Dockerfile @@ -0,0 +1,147 @@ +ARG PYTHON_VERSION="3.6.10" +FROM python:${PYTHON_VERSION}-slim-buster as base +# +# USAGE: +# cd sercices/service-sidecar +# docker build -f Dockerfile -t service-sidecar:prod --target production ../../ +# docker run service-sidecar:prod +# +# REQUIRED: context expected at ``osparc-simcore/`` folder because we need access to osparc-simcore/packages + +LABEL maintainer="Andrei Neagu " + +RUN set -eux; \ + apt-get update; \ + apt-get install -y gosu; \ + rm -rf /var/lib/apt/lists/*; \ +# verify that the binary works + gosu nobody true + +# simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) +ENV SC_USER_ID=8004 \ + SC_USER_NAME=scu \ + SC_BUILD_TARGET=base \ + SC_BOOT_MODE=default + +RUN adduser \ + --uid ${SC_USER_ID} \ + --disabled-password \ + --gecos "" \ + --shell /bin/sh \ + --home /home/${SC_USER_NAME} \ + ${SC_USER_NAME} + +# Sets utf-8 encoding for Python et al +ENV LANG=C.UTF-8 +# Turns off writing .pyc files; superfluous on an ephemeral container. +ENV PYTHONDONTWRITEBYTECODE=1 \ + VIRTUAL_ENV=/home/scu/.venv +# Ensures that the python and pip executables used +# in the image will be those from our virtualenv. +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" + + +# -------------------------- Build stage ------------------- +# Installs build/package management tools and third party dependencies +# +# + /build WORKDIR +# +FROM base as build + +ENV SC_BUILD_TARGET=build + +RUN apt-get update &&\ + apt-get install -y --no-install-recommends \ + build-essential + +# NOTE: python virtualenv is used here such that installed +# packages may be moved to production image easily by copying the venv +RUN python -m venv ${VIRTUAL_ENV} + +RUN pip install --upgrade --no-cache-dir \ + pip~=21.0.1 \ + wheel \ + setuptools + +WORKDIR /build + +# install base 3rd party dependencies +# NOTE: copies to /build to avoid overwriting later which would invalidate this layer +COPY --chown=scu:scu services/service-sidecar/requirements/_base.txt . +RUN pip --no-cache-dir install -r _base.txt + +# --------------------------Cache stage ------------------- +# CI in master buils & pushes this target to speed-up image build +# +# + /build +# + services/service-sidecar [scu:scu] WORKDIR +# +FROM build as cache + +ENV SC_BUILD_TARGET cache + +COPY --chown=scu:scu packages /build/packages +COPY --chown=scu:scu services/service-sidecar /build/services/service-sidecar + +WORKDIR /build/services/service-sidecar + +RUN pip --no-cache-dir install -r requirements/prod.txt &&\ + pip --no-cache-dir list -v + + +# --------------------------Production stage ------------------- +# Final cleanup up to reduce image size and startup setup +# Runs as scu (non-root user) +# +# + /home/scu $HOME = WORKDIR +# + services/service-sidecar [scu:scu] +# +FROM base as production + +ENV SC_BUILD_TARGET=production \ + SC_BOOT_MODE=production + +ENV PYTHONOPTIMIZE=TRUE + +WORKDIR /home/scu + +# Starting from clean base image, copies pre-installed virtualenv from cache +COPY --chown=scu:scu --from=cache ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +# Copies booting scripts +COPY --chown=scu:scu services/service-sidecar/docker services/service-sidecar/docker +RUN chmod +x services/service-sidecar/docker/*.sh + +HEALTHCHECK --interval=30s \ + --timeout=20s \ + --start-period=30s \ + --retries=3 \ + CMD ["python3", "services/service-sidecar/docker/healthcheck.py", "http://localhost:8000/"] + +EXPOSE 8000 + +ENTRYPOINT [ "/bin/sh", "services/service-sidecar/docker/entrypoint.sh" ] +CMD ["/bin/sh", "services/service-sidecar/docker/boot.sh"] + + +# --------------------------Development stage ------------------- +# Source code accessible in host but runs in container +# Runs as myu with same gid/uid as host +# Placed at the end to speed-up the build if images targeting production +# +# + /devel WORKDIR +# + services (mounted volume) +# +FROM build as development + +ENV SC_BUILD_TARGET=development + +WORKDIR /devel + +RUN chown -R scu:scu ${VIRTUAL_ENV} + +EXPOSE 8000 +EXPOSE 3000 + +ENTRYPOINT ["/bin/sh", "services/service-sidecar/docker/entrypoint.sh"] +CMD ["/bin/sh", "services/service-sidecar/docker/boot.sh"] diff --git a/services/service-sidecar/Makefile b/services/service-sidecar/Makefile new file mode 100644 index 00000000000..3c9002043ba --- /dev/null +++ b/services/service-sidecar/Makefile @@ -0,0 +1,44 @@ +include ../../scripts/common.Makefile +include ../../scripts/common-service.Makefile + +APP_NAME := $(notdir $(CURDIR)) + +.DEFAULT_GOAL := help + + +.PHONY: _ensure-in-venv +_ensure-in-venv: + @python3 -c "import os; os.environ['VIRTUAL_ENV']" || (echo "\n>>>> You are not in a virtualenv. Activate one <<<<\n"; exit 1) + +.PHONY: install-dev-dependencies +install-dev-dependencies: _ensure-in-venv ## install depenencies for development + @pip install -r requirements/_base.txt + +.PHONY: install-dev-dependencies +install-dev-dependencies: _ensure-in-venv ## install depenencies for development + @pip install -r requirements/_base.txt + +.PHONY: dev-run +dev-run: _ensure-in-venv ## starts the container on its own + @docker run -it --rm \ + -p 8000:8000 \ + -v $(CURDIR):/devel/services/service-sidecar \ + local/service-sidecar:development + +.PHONY: ci-tox-codestyle +ci-tox-codestyle: ## runs codestyle checks + @tox -r -e codestyle-ci + +.PHONY: ci-tox-tests +ci-tox-tests: ## runs tests with coverage in a new enviornment + @tox -r -e py36,report + +.PHONY: ci-install-requirements +ci-install-requirements: ## runs tests with coverage in a new enviornment + @pip install -r requirements/dev.txt + @pip install tox + +.PHONY: run-github-action-locally +run-github-action-locally: ## runs the defined github action from the workflow locally + # Note: ⚡ act is required https://github.com/nektos/act + @act -C ../../. -P ubuntu-20.04=catthehacker/ubuntu:act-20.04 -j unit-test-service-sidecar diff --git a/services/service-sidecar/VERSION b/services/service-sidecar/VERSION new file mode 100644 index 00000000000..8acdd82b765 --- /dev/null +++ b/services/service-sidecar/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/services/service-sidecar/docker/boot.sh b/services/service-sidecar/docker/boot.sh new file mode 100755 index 00000000000..c8c39dd6f84 --- /dev/null +++ b/services/service-sidecar/docker/boot.sh @@ -0,0 +1,36 @@ +#!/bin/sh +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + +INFO="INFO: [$(basename "$0")] " + +# BOOTING application --------------------------------------------- +echo "$INFO" "Booting in ${SC_BOOT_MODE} mode ..." +echo "$INFO" "User :$(id "$(whoami)")" +echo "$INFO" "Workdir : $(pwd)" + +if [ "${SC_BUILD_TARGET}" = "development" ]; then + echo "$INFO" "Environment :" + printenv | sed 's/=/: /' | sed 's/^/ /' | sort + echo "$INFO" "Python :" + python --version | sed 's/^/ /' + command -v python | sed 's/^/ /' + + cd services/service-sidecar || exit 1 + pip --quiet --no-cache-dir install -r requirements/dev.txt + cd - || exit 1 + echo "$INFO" "PIP :" + pip list | sed 's/^/ /' +fi + +# RUNNING application ---------------------------------------- +if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ] +then + # NOTE: ptvsd is programmatically enabled inside of the service + # this way we can have reload in place as well + exec uvicorn sidecar.app:app --reload --host 0.0.0.0 +else + exec simcore_service_service_sidecar_startup +fi diff --git a/services/service-sidecar/docker/entrypoint.sh b/services/service-sidecar/docker/entrypoint.sh new file mode 100755 index 00000000000..84bbce1d50b --- /dev/null +++ b/services/service-sidecar/docker/entrypoint.sh @@ -0,0 +1,95 @@ +#!/bin/sh +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + +INFO="INFO: [$(basename "$0")] " +WARNING="WARNING: [$(basename "$0")] " +ERROR="ERROR: [$(basename "$0")] " + +# This entrypoint script: +# +# - Executes *inside* of the container upon start as --user [default root] +# - Notice that the container *starts* as --user [default root] but +# *runs* as non-root user [scu] +# +echo "$INFO" "Entrypoint for stage ${SC_BUILD_TARGET} ..." +echo "$INFO" "User :$(id "$(whoami)")" +echo "$INFO" "Workdir : $(pwd)" +echo "$INFO" "User : $(id scu)" +echo "$INFO" "python : $(command -v python)" +echo "$INFO" "pip : $(command -v pip)" + +USERNAME=scu +GROUPNAME=scu + +if [ "${SC_BUILD_TARGET}" = "development" ]; then + echo "$INFO" "development mode detected..." + # NOTE: expects docker run ... -v $(pwd):$DEVEL_MOUNT + DEVEL_MOUNT=/devel/services/service-sidecar + + stat $DEVEL_MOUNT >/dev/null 2>&1 || + (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) + + echo "$INFO" "setting correct user id/group id..." + HOST_USERID=$(stat --format=%u "${DEVEL_MOUNT}") + HOST_GROUPID=$(stat --format=%g "${DEVEL_MOUNT}") + CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) + if [ "$HOST_USERID" -eq 0 ]; then + echo "$WARNING" "Folder mounted owned by root user... adding $SC_USER_NAME to root..." + adduser "$SC_USER_NAME" root + else + echo "$INFO" "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." + # take host's credentials in $SC_USER_NAME + if [ -z "$CONT_GROUPNAME" ]; then + echo "$WARNING" "Creating new group grp$SC_USER_NAME" + CONT_GROUPNAME=grp$SC_USER_NAME + addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" + else + echo "$INFO" "group already exists" + fi + echo "$INFO" "Adding $SC_USER_NAME to group $CONT_GROUPNAME..." + adduser "$SC_USER_NAME" "$CONT_GROUPNAME" + + echo "$WARNING" "Changing ownership [this could take some time]" + echo "$INFO" "Changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" + usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" + + echo "$INFO" "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; + # change user property of files already around + echo "$INFO" "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; + fi +fi + +if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ]; then + # NOTE: production does NOT pre-installs ptvsd + pip install --no-cache-dir ptvsd +fi + +# Appends docker group if socket is mounted +DOCKER_MOUNT=/var/run/docker.sock +if stat $DOCKER_MOUNT > /dev/null 2>&1 +then + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker + + if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 + then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." + # if group already exists in container, then reuse name + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" + fi + adduser "$SC_USER_NAME" "$GROUPNAME" +fi + + +echo "$INFO Starting $* ..." +echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" +echo " local dir : $(ls -al)" + +exec gosu "$SC_USER_NAME" "$@" diff --git a/services/service-sidecar/docker/healthcheck.py b/services/service-sidecar/docker/healthcheck.py new file mode 100644 index 00000000000..23a3ba3ec11 --- /dev/null +++ b/services/service-sidecar/docker/healthcheck.py @@ -0,0 +1,40 @@ +#!/bin/python +""" Healthcheck script to run inside docker + +Example of usage in a Dockerfile +``` + COPY --chown=scu:scu docker/healthcheck.py docker/healthcheck.py + HEALTHCHECK --interval=30s \ + --timeout=30s \ + --start-period=1s \ + --retries=3 \ + CMD python3 docker/healthcheck.py http://localhost:8000/ +``` + +Q&A: + 1. why not to use curl instead of a python script? + - SEE https://blog.sixeyed.com/docker-healthchecks-why-not-to-use-curl-or-iwr/ +""" + +import os +import sys +from urllib.request import urlopen + +SUCCESS, UNHEALTHY = 0, 1 + +# Disabled if boots with debugger +ok = os.environ.get("SC_BOOT_MODE").lower() == "debug" + +# Queries host +ok = ( + ok + or urlopen( + "{host}{baseurl}".format( + host=sys.argv[1], baseurl=os.environ.get("SIMCORE_NODE_BASEPATH", "") + ) # adds a base-path if defined in environ + ).getcode() + == 200 +) + + +sys.exit(SUCCESS if ok else UNHEALTHY) diff --git a/services/service-sidecar/requirements/Makefile b/services/service-sidecar/requirements/Makefile new file mode 100644 index 00000000000..3f25442b790 --- /dev/null +++ b/services/service-sidecar/requirements/Makefile @@ -0,0 +1,6 @@ +# +# Targets to pip-compile requirements +# +include ../../../requirements/base.Makefile + +# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt diff --git a/services/service-sidecar/requirements/_base.in b/services/service-sidecar/requirements/_base.in new file mode 100644 index 00000000000..2f5e51ba0b3 --- /dev/null +++ b/services/service-sidecar/requirements/_base.in @@ -0,0 +1,19 @@ +# +# Specifies third-party dependencies for 'services/service-sidecar/src' +# +# NOTE: ALL version constraints MUST be commented +-c ../../../requirements/constraints.txt +# NOTE: These input-requirements under packages are tested using latest updates +-r ../../../packages/models-library/requirements/_base.in +-r ../../../packages/postgres-database/requirements/_base.in + +fastapi +pydantic +uvicorn + +docker-compose +aiodocker +aiofiles +PyYAML +async-timeout +async_generator diff --git a/services/service-sidecar/requirements/_base.txt b/services/service-sidecar/requirements/_base.txt new file mode 100644 index 00000000000..3a19f47282e --- /dev/null +++ b/services/service-sidecar/requirements/_base.txt @@ -0,0 +1,167 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/_base.txt requirements/_base.in +# +aiodocker==0.19.1 + # via -r requirements/_base.in +aiofiles==0.6.0 + # via -r requirements/_base.in +aiohttp==3.7.4.post0 + # via aiodocker +async-generator==1.10 + # via -r requirements/_base.in +async-timeout==3.0.1 + # via + # -r requirements/_base.in + # aiohttp +attrs==20.3.0 + # via + # aiohttp + # jsonschema +bcrypt==3.2.0 + # via paramiko +cached-property==1.5.2 + # via docker-compose +certifi==2020.12.5 + # via requests +cffi==1.14.5 + # via + # bcrypt + # cryptography + # pynacl +chardet==4.0.0 + # via + # aiohttp + # requests +click==7.1.2 + # via uvicorn +cryptography==3.4.7 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # paramiko +dataclasses==0.8 + # via pydantic +distro==1.5.0 + # via docker-compose +dnspython==2.1.0 + # via email-validator +docker-compose==1.27.4 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_base.in +docker[ssh]==4.4.4 + # via docker-compose +dockerpty==0.4.1 + # via docker-compose +docopt==0.6.2 + # via docker-compose +email-validator==1.1.2 + # via pydantic +fastapi==0.63.0 + # via -r requirements/_base.in +greenlet==1.0.0 + # via sqlalchemy +h11==0.12.0 + # via uvicorn +idna-ssl==1.1.0 + # via aiohttp +idna==2.10 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # email-validator + # idna-ssl + # requests + # yarl +importlib-metadata==3.10.1 + # via + # jsonschema + # sqlalchemy +jsonschema==3.2.0 + # via docker-compose +multidict==5.1.0 + # via + # aiohttp + # yarl +paramiko==2.7.2 + # via docker +psycopg2-binary==2.8.6 + # via sqlalchemy +pycparser==2.20 + # via cffi +pydantic[email]==1.8.1 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/_base.in + # fastapi +pynacl==1.4.0 + # via paramiko +pyrsistent==0.17.3 + # via jsonschema +python-dotenv==0.17.0 + # via docker-compose +pyyaml==5.4.1 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_base.in + # docker-compose +requests==2.25.1 + # via + # docker + # docker-compose +six==1.15.0 + # via + # bcrypt + # docker + # dockerpty + # jsonschema + # pynacl + # websocket-client +sqlalchemy[postgresql_psycopg2binary]==1.4.7 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/postgres-database/requirements/_base.in +starlette==0.13.6 + # via fastapi +texttable==1.6.3 + # via docker-compose +typing-extensions==3.7.4.3 + # via + # aiodocker + # aiohttp + # importlib-metadata + # pydantic + # uvicorn + # yarl +urllib3==1.26.4 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # requests +uvicorn==0.13.4 + # via -r requirements/_base.in +websocket-client==0.58.0 + # via + # docker + # docker-compose +yarl==1.6.3 + # via + # -r requirements/../../../packages/postgres-database/requirements/_base.in + # aiohttp +zipp==3.4.1 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/services/service-sidecar/requirements/_test.in b/services/service-sidecar/requirements/_test.in new file mode 100644 index 00000000000..213d9496589 --- /dev/null +++ b/services/service-sidecar/requirements/_test.in @@ -0,0 +1,8 @@ +-c ../../../requirements/constraints.txt + +pytest +pytest-cov +pytest-asyncio +pytest-mock +async-asgi-testclient +faker \ No newline at end of file diff --git a/services/service-sidecar/requirements/_test.txt b/services/service-sidecar/requirements/_test.txt new file mode 100644 index 00000000000..a742f7be08d --- /dev/null +++ b/services/service-sidecar/requirements/_test.txt @@ -0,0 +1,68 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/_test.txt requirements/_test.in +# +async-asgi-testclient==1.4.6 + # via -r requirements/_test.in +attrs==20.3.0 + # via pytest +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +coverage==5.5 + # via pytest-cov +faker==8.0.0 + # via -r requirements/_test.in +idna==2.10 + # via + # -c requirements/../../../requirements/constraints.txt + # requests +importlib-metadata==3.10.1 + # via + # pluggy + # pytest +iniconfig==1.1.1 + # via pytest +multidict==5.1.0 + # via async-asgi-testclient +packaging==20.9 + # via pytest +pluggy==0.13.1 + # via pytest +py==1.10.0 + # via pytest +pyparsing==2.4.7 + # via packaging +pytest-asyncio==0.14.0 + # via -r requirements/_test.in +pytest-cov==2.11.1 + # via -r requirements/_test.in +pytest-mock==3.5.1 + # via -r requirements/_test.in +pytest==6.2.3 + # via + # -r requirements/_test.in + # pytest-asyncio + # pytest-cov + # pytest-mock +python-dateutil==2.8.1 + # via faker +requests==2.25.1 + # via async-asgi-testclient +six==1.15.0 + # via python-dateutil +text-unidecode==1.3 + # via faker +toml==0.10.2 + # via pytest +typing-extensions==3.7.4.3 + # via importlib-metadata +urllib3==1.26.4 + # via + # -c requirements/../../../requirements/constraints.txt + # requests +zipp==3.4.1 + # via importlib-metadata diff --git a/services/service-sidecar/requirements/_tools.in b/services/service-sidecar/requirements/_tools.in new file mode 100644 index 00000000000..451a9517204 --- /dev/null +++ b/services/service-sidecar/requirements/_tools.in @@ -0,0 +1,12 @@ +-c ../../../requirements/constraints.txt + +-c _base.txt +-c _test.txt + +-r ../../../requirements/devenv.txt + +# basic dev tools +mypy +pylint +black +isort \ No newline at end of file diff --git a/services/service-sidecar/requirements/_tools.txt b/services/service-sidecar/requirements/_tools.txt new file mode 100644 index 00000000000..af67058507f --- /dev/null +++ b/services/service-sidecar/requirements/_tools.txt @@ -0,0 +1,118 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/_tools.txt requirements/_tools.in +# +appdirs==1.4.4 + # via + # black + # virtualenv +astroid==2.5.3 + # via pylint +black==20.8b1 + # via + # -r requirements/../../../requirements/devenv.txt + # -r requirements/_tools.in +bump2version==1.0.1 + # via -r requirements/../../../requirements/devenv.txt +cfgv==3.2.0 + # via pre-commit +click==7.1.2 + # via + # -c requirements/_base.txt + # black + # pip-tools +dataclasses==0.8 + # via + # -c requirements/_base.txt + # black +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +identify==2.2.3 + # via pre-commit +importlib-metadata==3.10.1 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # pep517 + # pre-commit + # virtualenv +importlib-resources==5.1.2 + # via + # pre-commit + # virtualenv +isort==5.8.0 + # via + # -r requirements/../../../requirements/devenv.txt + # -r requirements/_tools.in + # pylint +lazy-object-proxy==1.6.0 + # via astroid +mccabe==0.6.1 + # via pylint +mypy-extensions==0.4.3 + # via + # black + # mypy +mypy==0.812 + # via -r requirements/_tools.in +nodeenv==1.6.0 + # via pre-commit +pathspec==0.8.1 + # via black +pep517==0.10.0 + # via pip-tools +pip-tools==6.0.1 + # via -r requirements/../../../requirements/devenv.txt +pre-commit==2.12.0 + # via -r requirements/../../../requirements/devenv.txt +pylint==2.7.4 + # via -r requirements/_tools.in +pyyaml==5.4.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # pre-commit +regex==2021.4.4 + # via black +six==1.15.0 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # virtualenv +toml==0.10.2 + # via + # -c requirements/_test.txt + # black + # pep517 + # pre-commit + # pylint +typed-ast==1.4.3 + # via + # astroid + # black + # mypy +typing-extensions==3.7.4.3 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # black + # importlib-metadata + # mypy +virtualenv==20.4.3 + # via pre-commit +wrapt==1.12.1 + # via astroid +zipp==3.4.1 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # importlib-metadata + # importlib-resources + # pep517 + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/services/service-sidecar/requirements/ci.txt b/services/service-sidecar/requirements/ci.txt new file mode 100644 index 00000000000..98aa7025ecf --- /dev/null +++ b/services/service-sidecar/requirements/ci.txt @@ -0,0 +1,18 @@ +# Shortcut to install all packages for the contigous integration (CI) of 'services/api-server' +# +# - As ci.txt but w/ tests +# +# Usage: +# pip install -r requirements/ci.txt +# + +# installs base + tests requirements +-r _test.txt + +# installs this repo's packages +../../packages/models-library/ +../../packages/postgres-database/ +../../packages/pytest-simcore/ + +# installs current package +. diff --git a/services/service-sidecar/requirements/dev.txt b/services/service-sidecar/requirements/dev.txt new file mode 100644 index 00000000000..1d7c29d2587 --- /dev/null +++ b/services/service-sidecar/requirements/dev.txt @@ -0,0 +1,20 @@ +# Shortcut to install all packages needed to develop 'services/api-server' +# +# - As ci.txt but with current and repo packages in develop (edit) mode +# +# Usage: +# pip install -r requirements/dev.txt +# + +# installs base + tests requirements +-r _base.txt +-r _test.txt + +# installs this repo's packages +-e ../../packages/models-library +-e ../../packages/postgres-database/ +-e ../../packages/pytest-simcore/ + + +# installs current package +-e . diff --git a/services/service-sidecar/requirements/prod.txt b/services/service-sidecar/requirements/prod.txt new file mode 100644 index 00000000000..b5b75c7750a --- /dev/null +++ b/services/service-sidecar/requirements/prod.txt @@ -0,0 +1,17 @@ +# Shortcut to install 'services/api-server' for production +# +# - As ci.txt but w/o tests +# +# Usage: +# pip install -r requirements/prod.txt +# + +# installs base requirements +-r _base.txt + +# installs this repo's packages +../../packages/models-library/ +../../packages/postgres-database/ + +# installs current package +. diff --git a/services/service-sidecar/setup.py b/services/service-sidecar/setup.py new file mode 100644 index 00000000000..ef6326c4cb9 --- /dev/null +++ b/services/service-sidecar/setup.py @@ -0,0 +1,42 @@ +import os +import re +from pathlib import Path + +from setuptools import find_packages, setup + +current_dir = Path(os.path.dirname(os.path.realpath(__file__))) + + +def read_reqs(reqs_path: Path): + return re.findall(r"(^[^#-][\w]+[-~>=<.\w]+)", reqs_path.read_text(), re.MULTILINE) + + +# ----------------------------------------------------------------- +# Hard requirements on third-parties and latest for in-repo packages +install_requires = read_reqs(current_dir / "requirements" / "_base.txt") + [ + "simcore-models-library", + "simcore-postgres-database", +] + +tests_require = read_reqs(current_dir / "requirements" / "_test.txt") + +current_version = (current_dir / "VERSION").read_text().strip() + +setup( + name="simcore_service_service_sidecar", + version=current_version, + packages=find_packages(where="src"), + package_dir={ + "": "src", + }, + include_package_data=True, + python_requires=">=3.6", + install_requires=install_requires, + tests_require=tests_require, + setup_requires=["setuptools_scm"], + entry_points={ + "console_scripts": [ + "simcore_service_service_sidecar_startup = simcore_service_service_sidecar.main:main", + ], + }, +) diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/__init__.py b/services/service-sidecar/src/simcore_service_service_sidecar/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/__init__.py b/services/service-sidecar/src/simcore_service_service_sidecar/api/__init__.py new file mode 100644 index 00000000000..94ff56968e4 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/api/__init__.py @@ -0,0 +1 @@ +from ._routing import main_router diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/_routing.py b/services/service-sidecar/src/simcore_service_service_sidecar/api/_routing.py new file mode 100644 index 00000000000..225504128e9 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/api/_routing.py @@ -0,0 +1,23 @@ +# module acting as root for all routes + +from fastapi import APIRouter + +from .compose import compose_router +from .container import container_router +from .containers import containers_router +from .health import health_router +from .push import push_router +from .retrive import retrive_router +from .state import state_router + +# setup and register all routes here form different modules +main_router = APIRouter() +main_router.include_router(health_router) +main_router.include_router(compose_router) +main_router.include_router(containers_router) +main_router.include_router(container_router) +main_router.include_router(state_router) +main_router.include_router(retrive_router) +main_router.include_router(push_router) + +__all__ = ["main_router"] diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/compose.py b/services/service-sidecar/src/simcore_service_service_sidecar/api/compose.py new file mode 100644 index 00000000000..2c8dd4becb9 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/api/compose.py @@ -0,0 +1,167 @@ +import logging +import traceback +from typing import Optional + +from fastapi import APIRouter, Request, Response +from fastapi.responses import PlainTextResponse + +from ..settings import ServiceSidecarSettings +from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command +from ..storage import SharedStore +from ..utils import InvalidComposeSpec + +logger = logging.getLogger(__name__) +compose_router = APIRouter() + + +@compose_router.post( + "/compose:store", response_class=PlainTextResponse, responses={204: {"model": None}} +) +async def store_docker_compose_spec_for_later_usage( + request: Request, response: Response +) -> Optional[str]: + """ Expects the docker-compose spec as raw-body utf-8 encoded text """ + body_as_text = (await request.body()).decode("utf-8") + + shared_store: SharedStore = request.app.state.shared_store + + try: + shared_store.put_spec(body_as_text) + except InvalidComposeSpec as e: + logger.warning("Error detected %s", traceback.format_exc()) + response.status_code = 400 + return str(e) + + response.status_code = 204 + return None + + +@compose_router.post("/compose:preload", response_class=PlainTextResponse) +async def create_docker_compose_configuration_containers_without_starting( + request: Request, response: Response, command_timeout: float +) -> str: + """ Expects the docker-compose spec as raw-body utf-8 encoded text """ + body_as_text = (await request.body()).decode("utf-8") + + settings: ServiceSidecarSettings = request.app.state.settings + shared_store: SharedStore = request.app.state.shared_store + + try: + shared_store.put_spec(body_as_text) + except InvalidComposeSpec as e: + logger.warning("Error detected %s", traceback.format_exc()) + response.status_code = 400 + return str(e) + + # --no-build might be a security risk building is disabled + command = "docker-compose -p {project} -f {file_path} up --no-build --no-start" + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=shared_store.get_spec(), + command=command, + command_timeout=command_timeout, + ) + + response.status_code = 200 if finished_without_errors else 400 + return stdout + + +@compose_router.post("/compose", response_class=PlainTextResponse) +async def start_or_update_docker_compose_configuration( + request: Request, response: Response, command_timeout: float +) -> str: + """ Expects the docker-compose spec as raw-body utf-8 encoded text """ + settings: ServiceSidecarSettings = request.app.state.settings + shared_store: SharedStore = request.app.state.shared_store + + # --no-build might be a security risk building is disabled + command = "docker-compose -p {project} -f {file_path} up --no-build -d" + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=shared_store.get_spec(), + command=command, + command_timeout=command_timeout, + ) + + response.status_code = 200 if finished_without_errors else 400 + return stdout + + +@compose_router.get("/compose:pull", response_class=PlainTextResponse) +async def pull_docker_required_docker_images( + request: Request, response: Response, command_timeout: float +) -> str: + """ Expects the docker-compose spec as raw-body utf-8 encoded text """ + shared_store: SharedStore = request.app.state.shared_store + settings: ServiceSidecarSettings = request.app.state.settings + + stored_compose_content = shared_store.get_spec() + if stored_compose_content is None: + response.status_code = 400 + return "No started spec to stop was found" + + command = "docker-compose -p {project} -f {file_path} pull --include-deps" + + try: + # mark as pulling images + shared_store.set_is_pulling_containsers() + + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=stored_compose_content, + command=command, + command_timeout=command_timeout, + ) + finally: + # remove mark + shared_store.unset_is_pulling_containsers() + + response.status_code = 200 if finished_without_errors else 400 + return stdout + + +@compose_router.put("/compose:stop", response_class=PlainTextResponse) +async def stop_containers_without_removing_them( + request: Request, response: Response, command_timeout: float +) -> str: + """Stops the previously started service + and returns the docker-compose output""" + shared_store: SharedStore = request.app.state.shared_store + settings: ServiceSidecarSettings = request.app.state.settings + + stored_compose_content = shared_store.get_spec() + if stored_compose_content is None: + response.status_code = 400 + return "No started spec to stop was found" + + command = ( + "docker-compose -p {project} -f {file_path} stop -t {stop_and_remove_timeout}" + ) + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=stored_compose_content, + command=command, + command_timeout=command_timeout, + ) + + response.status_code = 200 if finished_without_errors else 400 + return stdout + + +@compose_router.delete("/compose", response_class=PlainTextResponse) +async def remove_docker_compose_configuration( + request: Request, response: Response, command_timeout: float +) -> str: + """Removes the previously started service + and returns the docker-compose output""" + finished_without_errors, stdout = await remove_the_compose_spec( + shared_store=request.app.state.shared_store, + settings=request.app.state.settings, + command_timeout=command_timeout, + ) + + response.status_code = 200 if finished_without_errors else 400 + return stdout + + +__all__ = ["compose_router"] diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/container.py b/services/service-sidecar/src/simcore_service_service_sidecar/api/container.py new file mode 100644 index 00000000000..3182335af6d --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/api/container.py @@ -0,0 +1,97 @@ +from typing import Any, Dict, Union + +import aiodocker +from fastapi import APIRouter, Query, Request, Response + +from ..storage import SharedStore + +container_router = APIRouter() + + +@container_router.get("/container/logs") +async def get_container_logs( + # pylint: disable=unused-argument + request: Request, + response: Response, + container: str, + since: int = Query( + 0, + title="Timstamp", + description="Only return logs since this time, as a UNIX timestamp", + ), + until: int = Query( + 0, + title="Timstamp", + description="Only return logs before this time, as a UNIX timestamp", + ), + timestamps: bool = Query( + False, + title="Display timestamps", + description="Enabling this parameter will include timestamps in logs", + ), +) -> Union[str, Dict[str, Any]]: + """ Returns the logs of a given container if found """ + shared_store: SharedStore = request.app.state.shared_store + + if container not in shared_store.get_container_names(): + response.status_code = 400 + return dict(error=f"No container '{container}' was started") + + docker = aiodocker.Docker() + + try: + container_instance = await docker.containers.get(container) + + args = dict(stdout=True, stderr=True) + if timestamps: + args["timestamps"] = True + + return await container_instance.log(**args) + except aiodocker.exceptions.DockerError as e: + response.status_code = 400 + return dict(error=e.message) + + +@container_router.get("/container/inspect") +async def container_inspect( + request: Request, response: Response, container: str +) -> Dict[str, Any]: + """ Returns information about the container, like docker inspect command """ + shared_store: SharedStore = request.app.state.shared_store + + if container not in shared_store.get_container_names(): + response.status_code = 400 + return dict(error=f"No container '{container}' was started") + + docker = aiodocker.Docker() + + try: + container_instance = await docker.containers.get(container) + return await container_instance.show() + except aiodocker.exceptions.DockerError as e: + response.status_code = 400 + return dict(error=e.message) + + +@container_router.delete("/container/remove") +async def container_remove( + request: Request, response: Response, container: str +) -> Union[bool, Dict[str, Any]]: + shared_store: SharedStore = request.app.state.shared_store + + if container not in shared_store.get_container_names(): + response.status_code = 400 + return dict(error=f"No container '{container}' was started") + + docker = aiodocker.Docker() + + try: + container_instance = await docker.containers.get(container) + await container_instance.delete() + return True + except aiodocker.exceptions.DockerError as e: + response.status_code = 400 + return dict(error=e.message) + + +__all__ = ["container_router"] diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/containers.py b/services/service-sidecar/src/simcore_service_service_sidecar/api/containers.py new file mode 100644 index 00000000000..e22f2adba92 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/api/containers.py @@ -0,0 +1,76 @@ +from typing import Any, Dict, List + +import aiodocker +from fastapi import APIRouter, Request, Response + +containers_router = APIRouter() + + +@containers_router.get("/containers") +async def get_spawned_container_names(request: Request) -> List[str]: + """ Returns a list of containers created using docker-compose """ + return request.app.state.shared_store.get_container_names() + + +@containers_router.get("/containers:inspect") +async def containers_inspect(request: Request, response: Response) -> Dict[str, Any]: + """ Returns information about the container, like docker inspect command """ + docker = aiodocker.Docker() + + container_names = request.app.state.shared_store.get_container_names() + container_names = container_names if container_names else {} + + results = {} + + for container in container_names: + try: + container_instance = await docker.containers.get(container) + results[container] = await container_instance.show() + except aiodocker.exceptions.DockerError as e: + response.status_code = 400 + return dict(error=e.message) + + return results + + +@containers_router.get("/containers:docker-status") +async def containers_docker_status( + request: Request, response: Response +) -> Dict[str, Any]: + """ Returns the status of the containers """ + + def assemble_entry(status: str, error: str = "") -> Dict[str, str]: + return {"Status": status, "Error": error} + + docker = aiodocker.Docker() + + shared_store = request.app.state.shared_store + container_names = shared_store.get_container_names() + container_names = container_names if container_names else {} + + # if containers are being pulled, return pulling (fake status) + if shared_store.is_pulling_containsers: + # pulling is a fake state use to share more information with the frontend + return {x: assemble_entry(status="pulling") for x in container_names} + + results = {} + + for container in container_names: + try: + container_instance = await docker.containers.get(container) + container_inspect = await container_instance.show() + container_state = container_inspect.get("State", {}) + + # pending is another fake state use to share more information with the frontend + results[container] = { + "Status": container_state.get("Status", "pending"), + "Error": container_state.get("Error", ""), + } + except aiodocker.exceptions.DockerError as e: + response.status_code = 400 + return dict(error=e.message) + + return results + + +__all__ = ["containers_router"] diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/health.py b/services/service-sidecar/src/simcore_service_service_sidecar/api/health.py new file mode 100644 index 00000000000..f5dad519250 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/api/health.py @@ -0,0 +1,13 @@ +from fastapi import APIRouter, Request + +from ..models import ApplicationHealth + +health_router = APIRouter() + + +@health_router.get("/health", response_model=ApplicationHealth) +async def health_endpoint(request: Request) -> ApplicationHealth: + return request.app.state.application_health + + +__all__ = ["health_router"] diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/push.py b/services/service-sidecar/src/simcore_service_service_sidecar/api/push.py new file mode 100644 index 00000000000..02c7328e281 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/api/push.py @@ -0,0 +1,18 @@ +# acts as mock for now + +import logging + +from fastapi import APIRouter + +logger = logging.getLogger(__name__) + +push_router = APIRouter() + + +@push_router.post("/push") +async def post_api() -> str: + logger.warning("TODO: still need to implement") + return "" + + +__all__ = ["push_router"] diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/retrive.py b/services/service-sidecar/src/simcore_service_service_sidecar/api/retrive.py new file mode 100644 index 00000000000..9e441ccd6c2 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/api/retrive.py @@ -0,0 +1,24 @@ +# acts as mock for now + +import logging + +from fastapi import APIRouter + +logger = logging.getLogger(__name__) + +retrive_router = APIRouter() + + +@retrive_router.get("/retrive") +async def get_api() -> str: + logger.warning("TODO: still need to implement") + return "" + + +@retrive_router.post("/retrive") +async def post_api() -> str: + logger.warning("TODO: still need to implement") + return "" + + +__all__ = ["retrive_router"] diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/state.py b/services/service-sidecar/src/simcore_service_service_sidecar/api/state.py new file mode 100644 index 00000000000..47f110cabff --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/api/state.py @@ -0,0 +1,24 @@ +# acts as mock for now + +import logging + +from fastapi import APIRouter + +logger = logging.getLogger(__name__) + +state_router = APIRouter() + + +@state_router.get("/state") +async def get_api() -> str: + logger.warning("TODO: still need to implement") + return "" + + +@state_router.post("/state") +async def post_api() -> str: + logger.warning("TODO: still need to implement") + return "" + + +__all__ = ["state_router"] diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/application.py b/services/service-sidecar/src/simcore_service_service_sidecar/application.py new file mode 100644 index 00000000000..bbd75ee44c6 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/application.py @@ -0,0 +1,51 @@ +import logging + +from fastapi import FastAPI + +from .api import main_router +from .models import ApplicationHealth +from .remote_debug import setup as remote_debug_setup +from .settings import ServiceSidecarSettings +from .shared_handlers import on_shutdown_handler +from .storage import SharedStore + +logger = logging.getLogger(__name__) + + +def assemble_application() -> FastAPI: + """ + Creates the application from using the env vars as a context + Also stores inside the state all instances of classes + needed in other requests and used to share data. + """ + + service_sidecar_settings = ServiceSidecarSettings.create() + + logging.basicConfig(level=service_sidecar_settings.loglevel) + logging.root.setLevel(service_sidecar_settings.loglevel) + logger.debug(service_sidecar_settings.json(indent=2)) + + application = FastAPI(debug=service_sidecar_settings.debug) + + # store "settings" and "shared_store" for later usage + application.state.settings = service_sidecar_settings + application.state.shared_store = SharedStore(settings=service_sidecar_settings) + # used to keep track of the health of the application + # also will be used in the /health endpoint + application.state.application_health = ApplicationHealth() + + # enable debug if required + if service_sidecar_settings.is_development_mode: + remote_debug_setup(application) + + # add routing paths + application.include_router(main_router) + + # setting up handler for lifecycle + async def on_shutdown() -> None: + await on_shutdown_handler(application) + logger.info("shutdown cleanup completed") + + application.add_event_handler("shutdown", on_shutdown) + + return application diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/main.py b/services/service-sidecar/src/simcore_service_service_sidecar/main.py new file mode 100644 index 00000000000..cc52402383d --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/main.py @@ -0,0 +1,31 @@ +import sys +from pathlib import Path + +import uvicorn +from fastapi import FastAPI +from simcore_service_service_sidecar.application import assemble_application +from simcore_service_service_sidecar.settings import ServiceSidecarSettings + +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + + +app: FastAPI = assemble_application() + + +def main(): + settings: ServiceSidecarSettings = app.state.settings + + uvicorn.run( + "simcore_service_service_sidecar.main:app", + host=settings.host, + port=settings.port, + reload=settings.is_development_mode, + reload_dirs=[ + current_dir, + ], + log_level=settings.log_level_name.lower(), + ) + + +if __name__ == "__main__": + main() diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/models.py b/services/service-sidecar/src/simcore_service_service_sidecar/models.py new file mode 100644 index 00000000000..7d2080340c9 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/models.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel, Field + + +class ApplicationHealth(BaseModel): + is_healthy: bool = Field( + True, description="returns True if the service sis running correctly" + ) diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/remote_debug.py b/services/service-sidecar/src/simcore_service_service_sidecar/remote_debug.py new file mode 100644 index 00000000000..75e49be73a9 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/remote_debug.py @@ -0,0 +1,33 @@ +""" Setup remote debugger with Python Tools for Visual Studio (PTVSD) + +""" +import logging + +from fastapi import FastAPI + +logger = logging.getLogger(__name__) + + +def setup(app: FastAPI): + settings = app.state.settings + + def on_startup() -> None: + try: + logger.debug("Enabling attach ptvsd ...") + # + # SEE https://github.com/microsoft/ptvsd#enabling-debugging + # + import ptvsd # pylint: disable=import-outside-toplevel + + ptvsd.enable_attach(address=(settings.host, settings.remote_debug_port)) + + except ImportError as err: + raise Exception( + "Cannot enable remote debugging. Please install ptvsd first" + ) from err + + logger.info( + "Remote debugging enabled: listening port %s", settings.remote_debug_port + ) + + app.add_event_handler("startup", on_startup) diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/settings.py b/services/service-sidecar/src/simcore_service_service_sidecar/settings.py new file mode 100644 index 00000000000..8a767f94a3b --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/settings.py @@ -0,0 +1,87 @@ +import logging +from typing import Optional + +from models_library.basic_types import BootModeEnum, PortInt +from pydantic import BaseSettings, Field, PositiveInt, validator + + +class ServiceSidecarSettings(BaseSettings): + @classmethod + def create(cls, **settings_kwargs) -> "ServiceSidecarSettings": + return cls( + **settings_kwargs, + ) + + boot_mode: Optional[BootModeEnum] = Field( + ..., + description="boot mode helps determine if in development mode or normal operation", + env="SC_BOOT_MODE", + ) + + # LOGGING + log_level_name: str = Field("DEBUG", env="LOG_LEVEL") + + @validator("log_level_name") + @classmethod + def match_logging_level(cls, v) -> str: + try: + getattr(logging, v.upper()) + except AttributeError as err: + raise ValueError(f"{v.upper()} is not a valid level") from err + return v.upper() + + # SERVICE SERVER (see : https://www.uvicorn.org/settings/) + host: str = Field( + "0.0.0.0", # nosec + description="host where to bind the application on which to serve", + ) + port: PortInt = Field( + 8000, description="port where the server will be currently serving" + ) + + compose_namespace: str = Field( + ..., + description=( + "To avoid collisions when scheduling on the same node, this " + "will be compsoed by the project_uuid and node_uuid." + ), + ) + + max_combined_container_name_length: PositiveInt = Field( + 63, description="the container name which will be used as hostname" + ) + + stop_and_remove_timeout: PositiveInt = Field( + 5, + description=( + "When receiving SIGTERM the process has 10 seconds to cleanup its children " + "forcing our children to stop in 5 seconds in all cases" + ), + ) + + debug: bool = Field( + False, + description="If set to True the application will boot into debug mode", + env="DEBUG", + ) + + remote_debug_port: PortInt = Field( + 3000, description="ptsvd remote debugger starting port" + ) + + docker_compose_down_timeout: PositiveInt = Field( + ..., description="used during shutdown when containers swapend will be removed" + ) + + @property + def is_development_mode(self): + """If in development mode this will be True""" + return self.boot_mode == BootModeEnum.DEVELOPMENT + + @property + def loglevel(self) -> int: + return getattr(logging, self.log_level_name) + + class Config: + case_sensitive = False + env_prefix = "SERVICE_SIDECAR_" diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/shared_handlers.py b/services/service-sidecar/src/simcore_service_service_sidecar/shared_handlers.py new file mode 100644 index 00000000000..2e41c86f165 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/shared_handlers.py @@ -0,0 +1,64 @@ +import logging +from typing import Optional, Tuple + +from fastapi import FastAPI + +from .settings import ServiceSidecarSettings +from .storage import SharedStore +from .utils import async_command, write_to_tmp_file + +logger = logging.getLogger(__name__) + + +async def write_file_and_run_command( + settings: ServiceSidecarSettings, + file_content: Optional[str], + command: str, + command_timeout: float, +) -> Tuple[bool, str]: + """ The command which accepts {file_path} as an argument for string formatting """ + + # pylint: disable=not-async-context-manager + async with write_to_tmp_file(file_content) as file_path: + formatted_command = command.format( + file_path=file_path, + project=settings.compose_namespace, + stop_and_remove_timeout=settings.stop_and_remove_timeout, + ) + logger.debug("Will run command\n'%s':\n%s", formatted_command, file_content) + return await async_command(formatted_command, command_timeout) + + +async def remove_the_compose_spec( + shared_store: SharedStore, settings: ServiceSidecarSettings, command_timeout: float +) -> Tuple[bool, str]: + + stored_compose_content = shared_store.get_spec() + if stored_compose_content is None: + return True, "No started spec to remove was found" + + command = ( + "docker-compose -p {project} -f {file_path} " + "down --remove-orphans -t {stop_and_remove_timeout}" + ) + result = await write_file_and_run_command( + settings=settings, + file_content=stored_compose_content, + command=command, + command_timeout=command_timeout, + ) + shared_store.put_spec(None) # removing compose-file spec + return result + + +async def on_shutdown_handler(app: FastAPI) -> None: + logging.info("Going to remove spawned containers") + shared_store: SharedStore = app.state.shared_store + settings: ServiceSidecarSettings = app.state.settings + + result = await remove_the_compose_spec( + shared_store=shared_store, + settings=settings, + command_timeout=settings.docker_compose_down_timeout, + ) + logging.info("Container removal did_succeed=%s\n%s", result[0], result[1]) diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/storage.py b/services/service-sidecar/src/simcore_service_service_sidecar/storage.py new file mode 100644 index 00000000000..3ab22024512 --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/storage.py @@ -0,0 +1,51 @@ +from typing import Any, Dict, List, Optional + +from .settings import ServiceSidecarSettings +from .utils import assemble_container_names, validate_compose_spec + + +class SharedStore: + """Define custom storage abstraction for easy future extension""" + + __slots__ = ("_storage", "_settings", "_is_pulling_containsers") + + _K_COMPOSE_SPEC = "compose_spec" + _K_CONTAINER_NAMES = "container_names" + + def __set_as_compose_spec_none(self): + self._storage[self._K_COMPOSE_SPEC] = None + self._storage[self._K_CONTAINER_NAMES] = [] + + def __init__(self, settings: ServiceSidecarSettings): + self._storage: Dict[str, Any] = {} + self._settings: ServiceSidecarSettings = settings + self._is_pulling_containsers: bool = False + self.__set_as_compose_spec_none() + + def put_spec(self, compose_file_content: Optional[str]) -> None: + if compose_file_content is None: + self.__set_as_compose_spec_none() + return + + self._storage[self._K_COMPOSE_SPEC] = validate_compose_spec( + settings=self._settings, compose_file_content=compose_file_content + ) + self._storage[self._K_CONTAINER_NAMES] = assemble_container_names( + self._storage[self._K_COMPOSE_SPEC] + ) + + def get_spec(self) -> Optional[str]: + return self._storage.get(self._K_COMPOSE_SPEC) + + def get_container_names(self) -> List[str]: + return self._storage[self._K_CONTAINER_NAMES] + + @property + def is_pulling_containsers(self) -> bool: + return self._is_pulling_containsers + + def set_is_pulling_containsers(self) -> None: + self._is_pulling_containsers = True + + def unset_is_pulling_containsers(self) -> None: + self._is_pulling_containsers = False diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/utils.py b/services/service-sidecar/src/simcore_service_service_sidecar/utils.py new file mode 100644 index 00000000000..44f0b179fce --- /dev/null +++ b/services/service-sidecar/src/simcore_service_service_sidecar/utils.py @@ -0,0 +1,282 @@ +import asyncio +import json +import logging +import os +import re +import tempfile +import traceback +from pathlib import Path +from typing import Any, Dict, Generator, List, Tuple + +import aiofiles +import yaml +from async_generator import asynccontextmanager +from async_timeout import timeout + +from .settings import ServiceSidecarSettings + +TEMPLATE_SEARCH_PATTERN = r"%%(.*?)%%" + +logger = logging.getLogger(__name__) + + +class InvalidComposeSpec(Exception): + """Exception used to signal incorrect docker-compose configuration file""" + + +@asynccontextmanager +async def write_to_tmp_file(file_contents): + """Disposes of file on exit""" + # pylint: disable=protected-access,stop-iteration-return + file_path = Path("/") / f"tmp/{next(tempfile._get_candidate_names())}" + async with aiofiles.open(file_path, mode="w") as tmp_file: + await tmp_file.write(file_contents) + try: + yield file_path + finally: + await aiofiles.os.remove(file_path) + + +async def async_command(command, command_timeout: float) -> Tuple[bool, str]: + """Returns if the command exited correctly and the stdout of the command """ + proc = await asyncio.create_subprocess_shell( + command, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + + # because the Processes returned by create_subprocess_shell it is not possible to + # have a timeout otherwise nor to stream the response from the process. + try: + async with timeout(command_timeout): + stdout, _ = await proc.communicate() + except asyncio.TimeoutError: + message = ( + f"{traceback.format_exc()}\nTimed out after {command_timeout} " + f"seconds while running {command}" + ) + logger.warning(message) + return False, message + + decoded_stdout = stdout.decode() + logger.info("'%s' result:\n%s", command, decoded_stdout) + finished_without_errors = proc.returncode == 0 + + return finished_without_errors, decoded_stdout + + +def _assemble_container_name( + settings: ServiceSidecarSettings, + service_key: str, + user_given_container_name: str, + index: int, +) -> str: + strings_to_use = [ + settings.compose_namespace, + str(index), + user_given_container_name, + service_key, + ] + + container_name = "-".join([x for x in strings_to_use if len(x) > 0])[ + : settings.max_combined_container_name_length + ] + return container_name.replace("_", "-") + + +def _get_forwarded_env_vars(container_key: str) -> List[str]: + """retruns env vars targeted to each container in the compose spec""" + results = [ + # some services expect it, using it as empty + "SIMCORE_NODE_BASEPATH=", + ] + for key in os.environ.keys(): + if key.startswith("FORWARD_ENV_"): + new_entry_key = key.replace("FORWARD_ENV_", "") + + # parsing `VAR={"destination_container": "destination_container", "env_var": "PAYLOAD"}` + new_entry_payload = json.loads(os.environ[key]) + if new_entry_payload["destination_container"] != container_key: + continue + + new_entry_value = new_entry_payload["env_var"] + new_entry = f"{new_entry_key}={new_entry_value}" + results.append(new_entry) + return results + + +def _extract_templated_entries(text: str) -> List[str]: + return re.findall(TEMPLATE_SEARCH_PATTERN, text) + + +def _apply_templating_directives( + stringified_compose_spec: str, + services: Dict[str, Any], + spec_services_to_container_name: Dict[str, str], +) -> str: + """ + Some custom rules are supported for replacing `container_name` + with the following syntax `%%container_name.SERVICE_KEY_NAME%%`, + where `SERVICE_KEY_NAME` targets a container in the compose spec + + If the directive cannot be applied it will just be left untouched + """ + matches = set(_extract_templated_entries(stringified_compose_spec)) + for match in matches: + parts = match.split(".") + + if len(parts) != 2: + continue # templating will be skipped + + target_property = parts[0] + services_key = parts[1] + if target_property != "container_name": + continue # also ignore if the container_name is not the directive to replace + + remapped_service_key = spec_services_to_container_name[services_key] + replace_with = services.get(remapped_service_key, {}).get( + "container_name", None + ) + if replace_with is None: + continue # also skip here if nothing was found + + match_pattern = f"%%{match}%%" + stringified_compose_spec = stringified_compose_spec.replace( + match_pattern, replace_with + ) + + return stringified_compose_spec + + +def _merge_env_vars( + compose_spec_env_vars: List[str], settings_env_vars: List[str] +) -> List[str]: + def _gen_parts_env_vars( + env_vars: List[str], + ) -> Generator[Tuple[str, str], None, None]: + for env_var in env_vars: + key, value = env_var.split("=") + yield key, value + + # pylint: disable=unnecessary-comprehension + dict_spec_env_vars = {k: v for k, v in _gen_parts_env_vars(compose_spec_env_vars)} + dict_settings_env_vars = {k: v for k, v in _gen_parts_env_vars(settings_env_vars)} + + # overwrite spec vars with vars from settings + for key, value in dict_settings_env_vars.items(): + dict_spec_env_vars[key] = value + + # returns a single list of vars + return [f"{k}={v}" for k, v in dict_spec_env_vars.items()] + + +def _inject_backend_networking( + parsed_compose_spec: Dict[str, Any], network_name: str = "__backend__" +) -> None: + """ + Put all containers in the compose spec in the same network. + The `network_name` must only be unique inside the user defined spec; + docker-compose will add some prefix to it. + """ + + networks = parsed_compose_spec.get("networks", {}) + networks[network_name] = None + + for service_content in parsed_compose_spec["services"].values(): + service_networks = service_content.get("networks", []) + service_networks.append(network_name) + service_content["networks"] = service_networks + + parsed_compose_spec["networks"] = networks + + +def validate_compose_spec( + settings: ServiceSidecarSettings, compose_file_content: str +) -> str: + """ + Checks the following: + - proper yaml format + - no "container_name" service property allowed, because it can + spawn 2 cotainers with the same name + """ + + try: + parsed_compose_spec = yaml.safe_load(compose_file_content) + except yaml.YAMLError as e: + raise InvalidComposeSpec( + f"{str(e)}\n{compose_file_content}\nProvided yaml is not valid!" + ) from e + + if parsed_compose_spec is None or not isinstance(parsed_compose_spec, dict): + raise InvalidComposeSpec(f"{compose_file_content}\nProvided yaml is not valid!") + + if not {"version", "services"}.issubset(set(parsed_compose_spec.keys())): + raise InvalidComposeSpec(f"{compose_file_content}\nProvided yaml is not valid!") + + version = parsed_compose_spec["version"] + if version.startswith("1"): + raise InvalidComposeSpec(f"Provided spec version '{version}' is not supported") + + spec_services_to_container_name: Dict[str, str] = {} + + spec_services = parsed_compose_spec["services"] + for index, service in enumerate(spec_services): + service_content = spec_services[service] + + # assemble and inject the container name + user_given_container_name = service_content.get("container_name", "") + container_name = _assemble_container_name( + settings, service, user_given_container_name, index + ) + service_content["container_name"] = container_name + spec_services_to_container_name[service] = container_name + + # inject forwarded environment variables + environment_entries = service_content.get("environment", []) + service_settings_env_vars = _get_forwarded_env_vars(service) + service_content["environment"] = _merge_env_vars( + compose_spec_env_vars=environment_entries, + settings_env_vars=service_settings_env_vars, + ) + + # if more then one container is defined, add an "backend" network + if len(spec_services) > 1: + _inject_backend_networking(parsed_compose_spec) + + # replace service_key with the container_name int the dict + for service_key in list(spec_services.keys()): + container_name_service_key = spec_services_to_container_name[service_key] + service_data = spec_services.pop(service_key) + + depends_on = service_data.get("depends_on", None) + if depends_on is not None: + service_data["depends_on"] = [ + # replaces with the container name + # if not found it leaves the old value + spec_services_to_container_name.get(x, x) + for x in depends_on + ] + + spec_services[container_name_service_key] = service_data + # TODO: replace names in depends_on keys + + # transform back to string and return + validated_compose_file_content = yaml.safe_dump(parsed_compose_spec) + + compose_spec = _apply_templating_directives( + stringified_compose_spec=validated_compose_file_content, + services=spec_services, + spec_services_to_container_name=spec_services_to_container_name, + ) + + return compose_spec + + +def assemble_container_names(validated_compose_content: str) -> List[str]: + """returns the list of container names from a validated compose_spec""" + parsed_compose_spec = yaml.safe_load(validated_compose_content) + return [ + service_data["container_name"] + for service_data in parsed_compose_spec["services"].values() + ] diff --git a/services/service-sidecar/tests/conftest.py b/services/service-sidecar/tests/conftest.py new file mode 100644 index 00000000000..ddfc89b4e97 --- /dev/null +++ b/services/service-sidecar/tests/conftest.py @@ -0,0 +1,102 @@ +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name + +import os +import subprocess +import sys +from typing import AsyncGenerator +from unittest import mock + +import aiodocker +import pytest +from async_asgi_testclient import TestClient +from fastapi import FastAPI +from simcore_service_service_sidecar.application import assemble_application +from simcore_service_service_sidecar.settings import ServiceSidecarSettings +from simcore_service_service_sidecar.shared_handlers import write_file_and_run_command +from simcore_service_service_sidecar.storage import SharedStore + + +@pytest.fixture(scope="module", autouse=True) +def app() -> FastAPI: + with mock.patch.dict( + os.environ, + { + "SC_BOOT_MODE": "production", + "SERVICE_SIDECAR_compose_namespace": "test-space", + "SERVICE_SIDECAR_docker_compose_down_timeout": "15", + }, + ): + return assemble_application() + + +@pytest.fixture +async def test_client(app: FastAPI) -> TestClient: + async with TestClient(app) as client: + yield client + + +@pytest.fixture(autouse=True) +async def cleanup_containers(app: FastAPI) -> AsyncGenerator[None, None]: + yield + # run docker compose down here + + shared_store: SharedStore = app.state.shared_store + stored_compose_content = shared_store.get_spec() + + if stored_compose_content is None: + # if no compose-spec is stored skip this operation + return + + settings: ServiceSidecarSettings = app.state.settings + command = ( + "docker-compose -p {project} -f {file_path} " + "down --remove-orphans -t {stop_and_remove_timeout}" + ) + await write_file_and_run_command( + settings=settings, + file_content=stored_compose_content, + command=command, + command_timeout=5.0, + ) + + +@pytest.fixture(autouse=True) +async def monkey_patch_asyncio_subprocess(mocker) -> None: + # TODO: The below bug is not allowing me to fully test, + # mocking and waiting for an update + # https://bugs.python.org/issue35621 + # this issue was patched in 3.8, no need + if sys.version_info.major == 3 and sys.version_info.minor >= 8: + raise RuntimeError( + "Issue no longer present in this version of python, " + "please remote this mock on python >= 3.8" + ) + + async def create_subprocess_exec(*command, **extra_params): + class MockResponse: + def __init__(self, command, **kwargs): + self.proc = subprocess.Popen(command, **extra_params) + + async def communicate(self): + return self.proc.communicate() + + @property + def returncode(self): + return self.proc.returncode + + mock_response = MockResponse(command, **extra_params) + + return mock_response + + mocker.patch("asyncio.create_subprocess_exec", side_effect=create_subprocess_exec) + + +@pytest.fixture +def mock_containers_get(mocker) -> None: + async def mock_get(*args, **kwargs): + raise aiodocker.exceptions.DockerError( + status="mock", data=dict(message="aiodocker_mocked_error") + ) + + mocker.patch("aiodocker.containers.DockerContainers.get", side_effect=mock_get) diff --git a/services/service-sidecar/tests/unit/test_api_compose.py b/services/service-sidecar/tests/unit/test_api_compose.py new file mode 100644 index 00000000000..6a8fa0d80f3 --- /dev/null +++ b/services/service-sidecar/tests/unit/test_api_compose.py @@ -0,0 +1,149 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +import json +from typing import Any, Dict + +import pytest +from async_asgi_testclient import TestClient +from faker import Faker + +DEFAULT_COMMAND_TIMEOUT = 10.0 + + +@pytest.fixture +def compose_spec() -> str: + return json.dumps( + { + "version": "3", + "services": {"nginx": {"image": "busybox"}}, + } + ) + + +@pytest.mark.asyncio +async def test_store_compose_spec( + test_client: TestClient, compose_spec: Dict[str, Any] +): + response = await test_client.post("/compose:store", data=compose_spec) + assert response.status_code == 204, response.text + assert response.text == "" + + +@pytest.mark.asyncio +async def test_store_compose_spec_not_provided(test_client: TestClient): + response = await test_client.post("/compose:store") + assert response.status_code == 400, response.text + assert response.text == "\nProvided yaml is not valid!" + + +@pytest.mark.asyncio +async def test_store_compose_spec_invalid(test_client: TestClient): + invalid_compose_spec = Faker().text() + response = await test_client.post("/compose:store", data=invalid_compose_spec) + assert response.status_code == 400, response.text + assert response.text.endswith("\nProvided yaml is not valid!") + # 28+ characters means the compos spec is also present in the error message + assert len(response.text) > 28 + + +@pytest.mark.asyncio +async def test_preload(test_client: TestClient, compose_spec: Dict[str, Any]): + response = await test_client.post( + "/compose:preload", query_string=dict(command_timeout=5.0), data=compose_spec + ) + assert response.status_code == 200, response.text + + +@pytest.mark.asyncio +async def test_preload_compose_spec_not_provided(test_client: TestClient): + + response = await test_client.post( + "/compose:preload", query_string=dict(command_timeout=5.0) + ) + assert response.status_code == 400, response.text + assert response.text == "\nProvided yaml is not valid!" + + +@pytest.mark.asyncio +async def test_compuse_up(test_client: TestClient, compose_spec: Dict[str, Any]): + # store spec first + response = await test_client.post("/compose:store", data=compose_spec) + assert response.status_code == 204, response.text + assert response.text == "" + + # pull images for spec + response = await test_client.post( + "/compose", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + ) + assert response.status_code == 200, response.text + + +@pytest.mark.asyncio +async def test_pull(test_client: TestClient, compose_spec: Dict[str, Any]): + # store spec first + response = await test_client.post("/compose:store", data=compose_spec) + assert response.status_code == 204, response.text + assert response.text == "" + + # pull images for spec + response = await test_client.get( + "/compose:pull", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + ) + assert response.status_code == 200, response.text + + +@pytest.mark.asyncio +async def test_pull_missing_spec(test_client: TestClient, compose_spec: Dict[str, Any]): + response = await test_client.get( + "/compose:pull", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + ) + assert response.status_code == 400, response.text + assert response.text == "No started spec to stop was found" + + +@pytest.mark.asyncio +async def test_compuse_stop_after_running( + test_client: TestClient, compose_spec: Dict[str, Any] +): + # store spec first + response = await test_client.post("/compose:store", data=compose_spec) + assert response.status_code == 204, response.text + assert response.text == "" + + # pull images for spec + response = await test_client.post( + "/compose", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + ) + assert response.status_code == 200, response.text + + response = await test_client.put( + "/compose:stop", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + ) + assert response.status_code == 200, response.text + + +@pytest.mark.asyncio +async def test_compuse_delete_after_stopping( + test_client: TestClient, compose_spec: Dict[str, Any] +): + # store spec first + response = await test_client.post("/compose:store", data=compose_spec) + assert response.status_code == 204, response.text + assert response.text == "" + + # pull images for spec + response = await test_client.post( + "/compose", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + ) + assert response.status_code == 200, response.text + + response = await test_client.put( + "/compose:stop", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + ) + assert response.status_code == 200, response.text + + response = await test_client.delete( + "/compose", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + ) + assert response.status_code == 200, response.text diff --git a/services/service-sidecar/tests/unit/test_api_container.py b/services/service-sidecar/tests/unit/test_api_container.py new file mode 100644 index 00000000000..24b9d11ca25 --- /dev/null +++ b/services/service-sidecar/tests/unit/test_api_container.py @@ -0,0 +1,149 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +import json +from typing import Any, Dict, List + +import pytest +from async_asgi_testclient import TestClient +from simcore_service_service_sidecar.storage import SharedStore + + +@pytest.fixture +def compose_spec() -> str: + return json.dumps( + { + "version": "3", + "services": { + "first-box": {"image": "busybox"}, + "second-box": {"image": "busybox"}, + }, + } + ) + + +@pytest.fixture +async def started_containers( + test_client: TestClient, compose_spec: Dict[str, Any] +) -> List[str]: + # store spec first + response = await test_client.post("/compose:store", data=compose_spec) + assert response.status_code == 204, response.text + assert response.text == "" + + # pull images for spec + response = await test_client.post( + "/compose", query_string=dict(command_timeout=10.0) + ) + assert response.status_code == 200, response.text + + shared_store: SharedStore = test_client.application.state.shared_store + container_names = shared_store.get_container_names() + assert len(container_names) == 2 + + return container_names + + +@pytest.fixture +def not_started_containers() -> List[str]: + return [f"missing-container-{i}" for i in range(5)] + + +@pytest.mark.asyncio +async def test_container_inspect_logs_remove( + test_client: TestClient, started_containers: List[str] +): + for container in started_containers: + # get container logs + response = await test_client.get( + "/container/logs", query_string=dict(container=container) + ) + assert response.status_code == 200, response.text + + # inspect container + response = await test_client.get( + "/container/inspect", query_string=dict(container=container) + ) + assert response.status_code == 200, response.text + parsed_response = response.json() + assert parsed_response["Name"] == f"/{container}" + + # delete container + response = await test_client.delete( + "/container/remove", query_string=dict(container=container) + ) + assert response.status_code == 200, response.text + + +@pytest.mark.asyncio +async def test_container_logs_with_timestamps( + test_client: TestClient, started_containers: List[str] +): + for container in started_containers: + # get container logs + response = await test_client.get( + "/container/logs", query_string=dict(container=container, timestamps=True) + ) + assert response.status_code == 200, response.text + + +@pytest.mark.asyncio +async def test_container_missing_container( + test_client: TestClient, not_started_containers: List[str] +): + def _expected_error_string(container: str) -> Dict[str, str]: + return dict(error=f"No container '{container}' was started") + + for container in not_started_containers: + # get container logs + response = await test_client.get( + "/container/logs", query_string=dict(container=container) + ) + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string(container) + + # inspect container + response = await test_client.get( + "/container/inspect", query_string=dict(container=container) + ) + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string(container) + + # delete container + response = await test_client.delete( + "/container/remove", query_string=dict(container=container) + ) + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string(container) + + +@pytest.mark.asyncio +async def test_container_docker_error( + test_client: TestClient, + started_containers: List[str], + mock_containers_get: None, +): + def _expected_error_string() -> Dict[str, str]: + return dict(error="aiodocker_mocked_error") + + for container in started_containers: + # get container logs + response = await test_client.get( + "/container/logs", query_string=dict(container=container) + ) + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string() + + # inspect container + response = await test_client.get( + "/container/inspect", query_string=dict(container=container) + ) + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string() + + # delete container + response = await test_client.delete( + "/container/remove", query_string=dict(container=container) + ) + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string() diff --git a/services/service-sidecar/tests/unit/test_api_containers.py b/services/service-sidecar/tests/unit/test_api_containers.py new file mode 100644 index 00000000000..a0bb07e4b9f --- /dev/null +++ b/services/service-sidecar/tests/unit/test_api_containers.py @@ -0,0 +1,134 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +import json +from contextlib import contextmanager +from typing import Any, Dict, Generator, List + +import pytest +from async_asgi_testclient import TestClient +from simcore_service_service_sidecar.storage import SharedStore + + +@pytest.fixture +def compose_spec() -> str: + return json.dumps( + { + "version": "3", + "services": { + "first-box": {"image": "busybox"}, + "second-box": {"image": "busybox"}, + }, + } + ) + + +@pytest.fixture +async def started_containers( + test_client: TestClient, compose_spec: Dict[str, Any] +) -> List[str]: + # store spec first + response = await test_client.post("/compose:store", data=compose_spec) + assert response.status_code == 204, response.text + assert response.text == "" + + # pull images for spec + response = await test_client.post( + "/compose", query_string=dict(command_timeout=10.0) + ) + assert response.status_code == 200, response.text + + shared_store: SharedStore = test_client.application.state.shared_store + container_names = shared_store.get_container_names() + assert len(container_names) == 2 + + return container_names + + +@pytest.mark.asyncio +async def test_containers_get(test_client: TestClient, started_containers: List[str]): + response = await test_client.get("/containers") + assert response.status_code == 200, response.text + assert set(json.loads(response.text)) == set(started_containers) + + +@pytest.mark.asyncio +async def test_containers_inspect( + test_client: TestClient, started_containers: List[str] +): + response = await test_client.get( + "/containers:inspect", query_string=dict(container_names=started_containers) + ) + assert response.status_code == 200, response.text + assert set(json.loads(response.text).keys()) == set(started_containers) + + +@pytest.mark.asyncio +async def test_containers_inspect_docker_error( + test_client: TestClient, started_containers: List[str], mock_containers_get: None +): + response = await test_client.get( + "/containers:inspect", query_string=dict(container_names=started_containers) + ) + assert response.status_code == 400, response.text + + +def assert_keys_exist(result: Dict[str, Any]) -> bool: + for entry in result.values(): + assert "Status" in entry + assert "Error" in entry + return True + + +@pytest.mark.asyncio +async def test_containers_docker_status( + test_client: TestClient, started_containers: List[str] +): + response = await test_client.get( + "/containers:docker-status", + query_string=dict(container_names=started_containers), + ) + assert response.status_code == 200, response.text + decoded_response = json.loads(response.text) + assert set(decoded_response) == set(started_containers) + assert assert_keys_exist(decoded_response) is True + + +@pytest.mark.asyncio +async def test_containers_docker_status_pulling_containers( + test_client: TestClient, started_containers: List[str] +): + @contextmanager + def mark_pulling(shared_store: SharedStore) -> Generator[None, None, None]: + try: + shared_store.set_is_pulling_containsers() + yield + finally: + shared_store.unset_is_pulling_containsers() + + shared_store: SharedStore = test_client.application.state.shared_store + + with mark_pulling(shared_store): + assert shared_store.is_pulling_containsers is True + + response = await test_client.get( + "/containers:docker-status", + query_string=dict(container_names=started_containers), + ) + assert response.status_code == 200, response.text + decoded_response = json.loads(response.text) + assert assert_keys_exist(decoded_response) is True + + for entry in decoded_response.values(): + assert entry["Status"] == "pulling" + + +@pytest.mark.asyncio +async def test_containers_docker_status_docker_error( + test_client: TestClient, started_containers: List[str], mock_containers_get: None +): + response = await test_client.get( + "/containers:docker-status", + query_string=dict(container_names=started_containers), + ) + assert response.status_code == 400, response.text diff --git a/services/service-sidecar/tests/unit/test_api_health.py b/services/service-sidecar/tests/unit/test_api_health.py new file mode 100644 index 00000000000..d714e07f214 --- /dev/null +++ b/services/service-sidecar/tests/unit/test_api_health.py @@ -0,0 +1,10 @@ +import pytest +from async_asgi_testclient import TestClient +from simcore_service_service_sidecar.models import ApplicationHealth + + +@pytest.mark.asyncio +async def test_is_healthy(test_client: TestClient): + response = await test_client.get("/health") + assert response.status_code == 200, response + assert response.json() == ApplicationHealth().dict() diff --git a/services/service-sidecar/tests/unit/test_api_push_retrive_state.py b/services/service-sidecar/tests/unit/test_api_push_retrive_state.py new file mode 100644 index 00000000000..5d1444686a1 --- /dev/null +++ b/services/service-sidecar/tests/unit/test_api_push_retrive_state.py @@ -0,0 +1,34 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + + +import json + +import pytest +from async_asgi_testclient import TestClient +from async_asgi_testclient.response import Response + + +def assert_200_empty(response: Response) -> bool: + assert response.status_code == 200, response.text + assert json.loads(response.text) == "" + return True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route,method", + [ + # push api module + ("/push", "POST"), + # retrive api module + ("/retrive", "GET"), + ("/retrive", "POST"), + # state api module + ("/state", "GET"), + ("/state", "POST"), + ], +) +async def test_mocked_modules(test_client: TestClient, route: str, method: str): + response = await test_client.open(route, method=method) + assert assert_200_empty(response) is True diff --git a/services/service-sidecar/tox.ini b/services/service-sidecar/tox.ini new file mode 100644 index 00000000000..0051d0094a9 --- /dev/null +++ b/services/service-sidecar/tox.ini @@ -0,0 +1,57 @@ +# content of: tox.ini , put in same dir as setup.py +[tox] +isolated_build = True +envlist = clean, codestyle, py36, report +skipsdist=True + + +[testenv] +# install pytest in the virtualenv where commands will be executed +deps = + -r requirements/dev.txt +commands = + pytest -vvv -s --cov=simcore_service_service_sidecar --cov-append --cov-report=term-missing --capture=sys tests/unit + +depends = + {py36}: clean + report: py36 + +# will fail if any of isort, black, pylint or mypy fail +[testenv:codestyle-ci] +basepython = python3.6 +deps = + -r requirements/_tools.txt +commands = + isort --check setup.py src/simcore_service_service_sidecar tests + black --check src/simcore_service_service_sidecar tests/ + pylint --rcfile=../../.pylintrc src/simcore_service_service_sidecar tests/ + mypy src/simcore_service_service_sidecar tests/ --ignore-missing-imports + + +# used for development +[testenv:codestyle] +basepython = python3.6 +deps = + -r requirements/_tools.txt +commands = + isort setup.py src/simcore_service_service_sidecar tests + black src/simcore_service_service_sidecar tests/ + pylint --rcfile=../../.pylintrc src/simcore_service_service_sidecar tests/ + mypy src/simcore_service_service_sidecar tests/ --ignore-missing-imports + +[testenv:report] +basepython = python3.6 +deps = + coverage +skip_install = true +exclude_lines = + pragma: no cover +commands = + coverage report + # TODO: upload the test report somewere + +[testenv:clean] +basepython = python3.6 +deps = coverage +skip_install = true +commands = coverage erase From 7133a7b130d981b0ba5bcdf1df84ffcc3c8a39b2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 09:11:47 +0200 Subject: [PATCH 002/102] added task to run unittests --- .github/workflows/ci-testing-deploy.yml | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 85ef4782979..8da6417f879 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -313,6 +313,53 @@ jobs: name: codeclimate-${{ github.job }}-coverage path: codeclimate.${{ github.job }}_coverage.json + unit-test-dynamic-sidecar: + name: "[unit] dynamic-sidecar" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: [3.6] + os: [ubuntu-20.04] + fail-fast: false + steps: + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + - name: setup python environment + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - uses: actions/cache@v2 + name: getting cached data + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-dynamic-sidecar-${{ hashFiles('services/service-sidecar/requirements/ci.txt') }} + restore-keys: | + ${{ runner.os }}-pip-dynamic-sidecar- + ${{ runner.os }}-pip- + ${{ runner.os }}- + - name: install + run: ./ci/github/unit-testing/dynamic-sidecar.bash install + - name: test + run: ./ci/github/unit-testing/dynamic-sidecar.bash test + - uses: codecov/codecov-action@v1 + with: + flags: unittests #optional + - name: prepare codeclimate coverage file + run: | + curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter + chmod +x ./cc-test-reporter && ./cc-test-reporter --version + ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml + - name: upload codeclimate coverage + uses: actions/upload-artifact@v2 + with: + name: codeclimate-${{ github.job }}-coverage + path: codeclimate.${{ github.job }}_coverage.json + unit-test-frontend: name: "[unit] frontend" runs-on: ${{ matrix.os }} From 6d00094160bb3ea763253f61a10bdfaeeaff51c6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 09:11:57 +0200 Subject: [PATCH 003/102] added unittests bash commands --- ci/github/unit-testing/dynamic-sidecar.bash | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100755 ci/github/unit-testing/dynamic-sidecar.bash diff --git a/ci/github/unit-testing/dynamic-sidecar.bash b/ci/github/unit-testing/dynamic-sidecar.bash new file mode 100755 index 00000000000..cfb7c968944 --- /dev/null +++ b/ci/github/unit-testing/dynamic-sidecar.bash @@ -0,0 +1,29 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +install() { + bash ci/helpers/ensure_python_pip.bash; + pushd services/service-sidecar; pip3 install -r requirements/ci.txt; popd; + pip list -v +} + +test() { + pytest --cov=simcore_service_service_sidecar --durations=10 --cov-append \ + --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \ + -v -m "not travis" services/service-sidecar/tests/unit +} + +# Check if the function exists (bash specific) +if declare -f "$1" > /dev/null +then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi From 279f5854dbfe2b9a75d123b5c98d1316c7d51d1f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 09:12:07 +0200 Subject: [PATCH 004/102] added command to run unittests locally --- services/service-sidecar/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/service-sidecar/Makefile b/services/service-sidecar/Makefile index 3c9002043ba..ecb91f2e0d0 100644 --- a/services/service-sidecar/Makefile +++ b/services/service-sidecar/Makefile @@ -41,4 +41,4 @@ ci-install-requirements: ## runs tests with coverage in a new enviornment .PHONY: run-github-action-locally run-github-action-locally: ## runs the defined github action from the workflow locally # Note: ⚡ act is required https://github.com/nektos/act - @act -C ../../. -P ubuntu-20.04=catthehacker/ubuntu:act-20.04 -j unit-test-service-sidecar + @act -C ../../. -P ubuntu-20.04=catthehacker/ubuntu:act-20.04 -j unit-test-dynamic-sidecar From 892276999f4d38a290f2d7fce2022b80dd764547 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 09:12:23 +0200 Subject: [PATCH 005/102] added entry to build dynamic-sidecar --- services/docker-compose-build.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/services/docker-compose-build.yml b/services/docker-compose-build.yml index 5a5ecbbefc1..7ffb932f5d9 100644 --- a/services/docker-compose-build.yml +++ b/services/docker-compose-build.yml @@ -124,6 +124,22 @@ services: org.label-schema.vcs-url: "${VCS_URL}" org.label-schema.vcs-ref: "${VCS_REF}" + dynamic-sidecar: + image: local/dynamic-sidecar:${BUILD_TARGET:?build_target_required} + build: + context: ../ + dockerfile: services/service-sidecar/Dockerfile + cache_from: + - local/dynamic-sidecar:${BUILD_TARGET:?build_target_required} + - ${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:cache + - ${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest} + target: ${BUILD_TARGET:?build_target_required} + labels: + org.label-schema.schema-version: "1.0" + org.label-schema.build-date: "${BUILD_DATE}" + org.label-schema.vcs-url: "${VCS_URL}" + org.label-schema.vcs-ref: "${VCS_REF}" + storage: image: local/storage:${BUILD_TARGET:?build_target_required} build: From 45229bcadcf74b74031225adf62bb910f834181f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 09:33:14 +0200 Subject: [PATCH 006/102] coverage and deploy now await for dynamic-sidecar --- .github/workflows/ci-testing-deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 8da6417f879..80dcc7ce32c 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -1795,6 +1795,7 @@ jobs: unit-test-director, unit-test-director-v2, unit-test-sidecar, + unit-test-dynamic-sidecar, unit-test-service-integration, unit-test-service-library, unit-test-models-library, @@ -1849,6 +1850,7 @@ jobs: unit-test-director, unit-test-director-v2, unit-test-sidecar, + unit-test-dynamic-sidecar, unit-test-frontend, unit-test-python-linting, unit-test-service-integration, From 11d085ae0cd325d6fafa960a45ebedf33cb0cac7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:04:58 +0200 Subject: [PATCH 007/102] renamed service-sidecar to dynamic-sidecar --- .github/workflows/ci-testing-deploy.yml | 2 +- ci/github/unit-testing/dynamic-sidecar.bash | 6 ++-- services/docker-compose-build.yml | 2 +- .../CHANGELOG.md | 0 .../Dockerfile | 30 +++++++++---------- .../Makefile | 4 +-- .../VERSION | 0 .../docker/boot.sh | 4 +-- .../docker/entrypoint.sh | 2 +- .../docker/healthcheck.py | 0 .../requirements/Makefile | 0 .../requirements/_base.in | 2 +- .../requirements/_base.txt | 0 .../requirements/_test.in | 0 .../requirements/_test.txt | 2 +- .../requirements/_tools.in | 0 .../requirements/_tools.txt | 0 .../requirements/ci.txt | 0 .../requirements/dev.txt | 0 .../requirements/prod.txt | 0 .../setup.py | 4 +-- .../__init__.py | 0 .../api/__init__.py | 0 .../api/_routing.py | 0 .../api/compose.py | 0 .../api/container.py | 0 .../api/containers.py | 0 .../api/health.py | 0 .../api/push.py | 0 .../api/retrive.py | 0 .../api/state.py | 0 .../application.py | 16 +++++----- .../simcore_service_service_sidecar/main.py | 6 ++-- .../simcore_service_service_sidecar/models.py | 0 .../remote_debug.py | 0 .../settings.py | 2 +- .../shared_handlers.py | 0 .../storage.py | 0 .../simcore_service_service_sidecar/utils.py | 0 .../tests/conftest.py | 12 ++++---- .../tests/unit/test_api_compose.py | 0 .../tests/unit/test_api_container.py | 2 +- .../tests/unit/test_api_containers.py | 2 +- .../tests/unit/test_api_health.py | 2 +- .../tests/unit/test_api_push_retrive_state.py | 0 .../tox.ini | 20 ++++++------- 46 files changed, 60 insertions(+), 60 deletions(-) rename services/{service-sidecar => dynamic-sidecar}/CHANGELOG.md (100%) rename services/{service-sidecar => dynamic-sidecar}/Dockerfile (79%) rename services/{service-sidecar => dynamic-sidecar}/Makefile (94%) rename services/{service-sidecar => dynamic-sidecar}/VERSION (100%) rename services/{service-sidecar => dynamic-sidecar}/docker/boot.sh (91%) rename services/{service-sidecar => dynamic-sidecar}/docker/entrypoint.sh (98%) rename services/{service-sidecar => dynamic-sidecar}/docker/healthcheck.py (100%) rename services/{service-sidecar => dynamic-sidecar}/requirements/Makefile (100%) rename services/{service-sidecar => dynamic-sidecar}/requirements/_base.in (86%) rename services/{service-sidecar => dynamic-sidecar}/requirements/_base.txt (100%) rename services/{service-sidecar => dynamic-sidecar}/requirements/_test.in (100%) rename services/{service-sidecar => dynamic-sidecar}/requirements/_test.txt (99%) rename services/{service-sidecar => dynamic-sidecar}/requirements/_tools.in (100%) rename services/{service-sidecar => dynamic-sidecar}/requirements/_tools.txt (100%) rename services/{service-sidecar => dynamic-sidecar}/requirements/ci.txt (100%) rename services/{service-sidecar => dynamic-sidecar}/requirements/dev.txt (100%) rename services/{service-sidecar => dynamic-sidecar}/requirements/prod.txt (100%) rename services/{service-sidecar => dynamic-sidecar}/setup.py (88%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/__init__.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/api/__init__.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/api/_routing.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/api/compose.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/api/container.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/api/containers.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/api/health.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/api/push.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/api/retrive.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/api/state.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/application.py (70%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/main.py (76%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/models.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/remote_debug.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/settings.py (98%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/shared_handlers.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/storage.py (100%) rename services/{service-sidecar => dynamic-sidecar}/src/simcore_service_service_sidecar/utils.py (100%) rename services/{service-sidecar => dynamic-sidecar}/tests/conftest.py (88%) rename services/{service-sidecar => dynamic-sidecar}/tests/unit/test_api_compose.py (100%) rename services/{service-sidecar => dynamic-sidecar}/tests/unit/test_api_container.py (98%) rename services/{service-sidecar => dynamic-sidecar}/tests/unit/test_api_containers.py (98%) rename services/{service-sidecar => dynamic-sidecar}/tests/unit/test_api_health.py (82%) rename services/{service-sidecar => dynamic-sidecar}/tests/unit/test_api_push_retrive_state.py (100%) rename services/{service-sidecar => dynamic-sidecar}/tox.ini (62%) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 80dcc7ce32c..1545cbc1444 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -337,7 +337,7 @@ jobs: name: getting cached data with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-dynamic-sidecar-${{ hashFiles('services/service-sidecar/requirements/ci.txt') }} + key: ${{ runner.os }}-pip-dynamic-sidecar-${{ hashFiles('services/dynamic-sidecar/requirements/ci.txt') }} restore-keys: | ${{ runner.os }}-pip-dynamic-sidecar- ${{ runner.os }}-pip- diff --git a/ci/github/unit-testing/dynamic-sidecar.bash b/ci/github/unit-testing/dynamic-sidecar.bash index cfb7c968944..eeef5e2386d 100755 --- a/ci/github/unit-testing/dynamic-sidecar.bash +++ b/ci/github/unit-testing/dynamic-sidecar.bash @@ -7,14 +7,14 @@ IFS=$'\n\t' install() { bash ci/helpers/ensure_python_pip.bash; - pushd services/service-sidecar; pip3 install -r requirements/ci.txt; popd; + pushd services/dynamic-sidecar; pip3 install -r requirements/ci.txt; popd; pip list -v } test() { - pytest --cov=simcore_service_service_sidecar --durations=10 --cov-append \ + pytest --cov=simcore_service_dynamic_sidecar --durations=10 --cov-append \ --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \ - -v -m "not travis" services/service-sidecar/tests/unit + -v -m "not travis" services/dynamic-sidecar/tests/unit } # Check if the function exists (bash specific) diff --git a/services/docker-compose-build.yml b/services/docker-compose-build.yml index 7ffb932f5d9..29e0174d579 100644 --- a/services/docker-compose-build.yml +++ b/services/docker-compose-build.yml @@ -128,7 +128,7 @@ services: image: local/dynamic-sidecar:${BUILD_TARGET:?build_target_required} build: context: ../ - dockerfile: services/service-sidecar/Dockerfile + dockerfile: services/dynamic-sidecar/Dockerfile cache_from: - local/dynamic-sidecar:${BUILD_TARGET:?build_target_required} - ${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:cache diff --git a/services/service-sidecar/CHANGELOG.md b/services/dynamic-sidecar/CHANGELOG.md similarity index 100% rename from services/service-sidecar/CHANGELOG.md rename to services/dynamic-sidecar/CHANGELOG.md diff --git a/services/service-sidecar/Dockerfile b/services/dynamic-sidecar/Dockerfile similarity index 79% rename from services/service-sidecar/Dockerfile rename to services/dynamic-sidecar/Dockerfile index 5dc5b1d9c9f..b97a5877bef 100644 --- a/services/service-sidecar/Dockerfile +++ b/services/dynamic-sidecar/Dockerfile @@ -2,9 +2,9 @@ ARG PYTHON_VERSION="3.6.10" FROM python:${PYTHON_VERSION}-slim-buster as base # # USAGE: -# cd sercices/service-sidecar -# docker build -f Dockerfile -t service-sidecar:prod --target production ../../ -# docker run service-sidecar:prod +# cd sercices/dynamic-sidecar +# docker build -f Dockerfile -t dynamic-sidecar:prod --target production ../../ +# docker run dynamic-sidecar:prod # # REQUIRED: context expected at ``osparc-simcore/`` folder because we need access to osparc-simcore/packages @@ -67,23 +67,23 @@ WORKDIR /build # install base 3rd party dependencies # NOTE: copies to /build to avoid overwriting later which would invalidate this layer -COPY --chown=scu:scu services/service-sidecar/requirements/_base.txt . +COPY --chown=scu:scu services/dynamic-sidecar/requirements/_base.txt . RUN pip --no-cache-dir install -r _base.txt # --------------------------Cache stage ------------------- # CI in master buils & pushes this target to speed-up image build # # + /build -# + services/service-sidecar [scu:scu] WORKDIR +# + services/dynamic-sidecar [scu:scu] WORKDIR # FROM build as cache ENV SC_BUILD_TARGET cache COPY --chown=scu:scu packages /build/packages -COPY --chown=scu:scu services/service-sidecar /build/services/service-sidecar +COPY --chown=scu:scu services/dynamic-sidecar /build/services/dynamic-sidecar -WORKDIR /build/services/service-sidecar +WORKDIR /build/services/dynamic-sidecar RUN pip --no-cache-dir install -r requirements/prod.txt &&\ pip --no-cache-dir list -v @@ -94,7 +94,7 @@ RUN pip --no-cache-dir install -r requirements/prod.txt &&\ # Runs as scu (non-root user) # # + /home/scu $HOME = WORKDIR -# + services/service-sidecar [scu:scu] +# + services/dynamic-sidecar [scu:scu] # FROM base as production @@ -109,19 +109,19 @@ WORKDIR /home/scu COPY --chown=scu:scu --from=cache ${VIRTUAL_ENV} ${VIRTUAL_ENV} # Copies booting scripts -COPY --chown=scu:scu services/service-sidecar/docker services/service-sidecar/docker -RUN chmod +x services/service-sidecar/docker/*.sh +COPY --chown=scu:scu services/dynamic-sidecar/docker services/dynamic-sidecar/docker +RUN chmod +x services/dynamic-sidecar/docker/*.sh HEALTHCHECK --interval=30s \ --timeout=20s \ --start-period=30s \ --retries=3 \ - CMD ["python3", "services/service-sidecar/docker/healthcheck.py", "http://localhost:8000/"] + CMD ["python3", "services/dynamic-sidecar/docker/healthcheck.py", "http://localhost:8000/"] EXPOSE 8000 -ENTRYPOINT [ "/bin/sh", "services/service-sidecar/docker/entrypoint.sh" ] -CMD ["/bin/sh", "services/service-sidecar/docker/boot.sh"] +ENTRYPOINT [ "/bin/sh", "services/dynamic-sidecar/docker/entrypoint.sh" ] +CMD ["/bin/sh", "services/dynamic-sidecar/docker/boot.sh"] # --------------------------Development stage ------------------- @@ -143,5 +143,5 @@ RUN chown -R scu:scu ${VIRTUAL_ENV} EXPOSE 8000 EXPOSE 3000 -ENTRYPOINT ["/bin/sh", "services/service-sidecar/docker/entrypoint.sh"] -CMD ["/bin/sh", "services/service-sidecar/docker/boot.sh"] +ENTRYPOINT ["/bin/sh", "services/dynamic-sidecar/docker/entrypoint.sh"] +CMD ["/bin/sh", "services/dynamic-sidecar/docker/boot.sh"] diff --git a/services/service-sidecar/Makefile b/services/dynamic-sidecar/Makefile similarity index 94% rename from services/service-sidecar/Makefile rename to services/dynamic-sidecar/Makefile index ecb91f2e0d0..ff3d276e928 100644 --- a/services/service-sidecar/Makefile +++ b/services/dynamic-sidecar/Makefile @@ -22,8 +22,8 @@ install-dev-dependencies: _ensure-in-venv ## install depenencies for development dev-run: _ensure-in-venv ## starts the container on its own @docker run -it --rm \ -p 8000:8000 \ - -v $(CURDIR):/devel/services/service-sidecar \ - local/service-sidecar:development + -v $(CURDIR):/devel/services/dynamic-sidecar \ + local/dynamic-sidecar:development .PHONY: ci-tox-codestyle ci-tox-codestyle: ## runs codestyle checks diff --git a/services/service-sidecar/VERSION b/services/dynamic-sidecar/VERSION similarity index 100% rename from services/service-sidecar/VERSION rename to services/dynamic-sidecar/VERSION diff --git a/services/service-sidecar/docker/boot.sh b/services/dynamic-sidecar/docker/boot.sh similarity index 91% rename from services/service-sidecar/docker/boot.sh rename to services/dynamic-sidecar/docker/boot.sh index c8c39dd6f84..c53a8500c02 100755 --- a/services/service-sidecar/docker/boot.sh +++ b/services/dynamic-sidecar/docker/boot.sh @@ -18,7 +18,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/service-sidecar || exit 1 + cd services/dynamic-sidecar || exit 1 pip --quiet --no-cache-dir install -r requirements/dev.txt cd - || exit 1 echo "$INFO" "PIP :" @@ -32,5 +32,5 @@ then # this way we can have reload in place as well exec uvicorn sidecar.app:app --reload --host 0.0.0.0 else - exec simcore_service_service_sidecar_startup + exec simcore_service_dynamic_sidecar_startup fi diff --git a/services/service-sidecar/docker/entrypoint.sh b/services/dynamic-sidecar/docker/entrypoint.sh similarity index 98% rename from services/service-sidecar/docker/entrypoint.sh rename to services/dynamic-sidecar/docker/entrypoint.sh index 84bbce1d50b..a7050f48c1d 100755 --- a/services/service-sidecar/docker/entrypoint.sh +++ b/services/dynamic-sidecar/docker/entrypoint.sh @@ -27,7 +27,7 @@ GROUPNAME=scu if [ "${SC_BUILD_TARGET}" = "development" ]; then echo "$INFO" "development mode detected..." # NOTE: expects docker run ... -v $(pwd):$DEVEL_MOUNT - DEVEL_MOUNT=/devel/services/service-sidecar + DEVEL_MOUNT=/devel/services/dynamic-sidecar stat $DEVEL_MOUNT >/dev/null 2>&1 || (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) diff --git a/services/service-sidecar/docker/healthcheck.py b/services/dynamic-sidecar/docker/healthcheck.py similarity index 100% rename from services/service-sidecar/docker/healthcheck.py rename to services/dynamic-sidecar/docker/healthcheck.py diff --git a/services/service-sidecar/requirements/Makefile b/services/dynamic-sidecar/requirements/Makefile similarity index 100% rename from services/service-sidecar/requirements/Makefile rename to services/dynamic-sidecar/requirements/Makefile diff --git a/services/service-sidecar/requirements/_base.in b/services/dynamic-sidecar/requirements/_base.in similarity index 86% rename from services/service-sidecar/requirements/_base.in rename to services/dynamic-sidecar/requirements/_base.in index 2f5e51ba0b3..c79cdedf515 100644 --- a/services/service-sidecar/requirements/_base.in +++ b/services/dynamic-sidecar/requirements/_base.in @@ -1,5 +1,5 @@ # -# Specifies third-party dependencies for 'services/service-sidecar/src' +# Specifies third-party dependencies for 'services/dynamic-sidecar/src' # # NOTE: ALL version constraints MUST be commented -c ../../../requirements/constraints.txt diff --git a/services/service-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt similarity index 100% rename from services/service-sidecar/requirements/_base.txt rename to services/dynamic-sidecar/requirements/_base.txt diff --git a/services/service-sidecar/requirements/_test.in b/services/dynamic-sidecar/requirements/_test.in similarity index 100% rename from services/service-sidecar/requirements/_test.in rename to services/dynamic-sidecar/requirements/_test.in diff --git a/services/service-sidecar/requirements/_test.txt b/services/dynamic-sidecar/requirements/_test.txt similarity index 99% rename from services/service-sidecar/requirements/_test.txt rename to services/dynamic-sidecar/requirements/_test.txt index a742f7be08d..0b841a507fe 100644 --- a/services/service-sidecar/requirements/_test.txt +++ b/services/dynamic-sidecar/requirements/_test.txt @@ -14,7 +14,7 @@ chardet==4.0.0 # via requests coverage==5.5 # via pytest-cov -faker==8.0.0 +faker==8.1.0 # via -r requirements/_test.in idna==2.10 # via diff --git a/services/service-sidecar/requirements/_tools.in b/services/dynamic-sidecar/requirements/_tools.in similarity index 100% rename from services/service-sidecar/requirements/_tools.in rename to services/dynamic-sidecar/requirements/_tools.in diff --git a/services/service-sidecar/requirements/_tools.txt b/services/dynamic-sidecar/requirements/_tools.txt similarity index 100% rename from services/service-sidecar/requirements/_tools.txt rename to services/dynamic-sidecar/requirements/_tools.txt diff --git a/services/service-sidecar/requirements/ci.txt b/services/dynamic-sidecar/requirements/ci.txt similarity index 100% rename from services/service-sidecar/requirements/ci.txt rename to services/dynamic-sidecar/requirements/ci.txt diff --git a/services/service-sidecar/requirements/dev.txt b/services/dynamic-sidecar/requirements/dev.txt similarity index 100% rename from services/service-sidecar/requirements/dev.txt rename to services/dynamic-sidecar/requirements/dev.txt diff --git a/services/service-sidecar/requirements/prod.txt b/services/dynamic-sidecar/requirements/prod.txt similarity index 100% rename from services/service-sidecar/requirements/prod.txt rename to services/dynamic-sidecar/requirements/prod.txt diff --git a/services/service-sidecar/setup.py b/services/dynamic-sidecar/setup.py similarity index 88% rename from services/service-sidecar/setup.py rename to services/dynamic-sidecar/setup.py index ef6326c4cb9..b04ec058b98 100644 --- a/services/service-sidecar/setup.py +++ b/services/dynamic-sidecar/setup.py @@ -23,7 +23,7 @@ def read_reqs(reqs_path: Path): current_version = (current_dir / "VERSION").read_text().strip() setup( - name="simcore_service_service_sidecar", + name="simcore_service_dynamic_sidecar", version=current_version, packages=find_packages(where="src"), package_dir={ @@ -36,7 +36,7 @@ def read_reqs(reqs_path: Path): setup_requires=["setuptools_scm"], entry_points={ "console_scripts": [ - "simcore_service_service_sidecar_startup = simcore_service_service_sidecar.main:main", + "simcore_service_dynamic_sidecar_startup = simcore_service_dynamic_sidecar.main:main", ], }, ) diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/__init__.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/__init__.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/__init__.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/__init__.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/__init__.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/__init__.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/api/__init__.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/api/__init__.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/_routing.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/_routing.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/api/_routing.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/api/_routing.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/compose.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/api/compose.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/api/compose.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/container.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/container.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/api/container.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/api/container.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/containers.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/api/containers.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/api/containers.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/health.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/api/health.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/api/health.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/push.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/push.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/api/push.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/api/push.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/retrive.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/retrive.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/api/retrive.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/api/retrive.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/api/state.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/state.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/api/state.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/api/state.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/application.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/application.py similarity index 70% rename from services/service-sidecar/src/simcore_service_service_sidecar/application.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/application.py index bbd75ee44c6..2e707674b28 100644 --- a/services/service-sidecar/src/simcore_service_service_sidecar/application.py +++ b/services/dynamic-sidecar/src/simcore_service_service_sidecar/application.py @@ -19,23 +19,23 @@ def assemble_application() -> FastAPI: needed in other requests and used to share data. """ - service_sidecar_settings = ServiceSidecarSettings.create() + dynamic_sidecar_settings = ServiceSidecarSettings.create() - logging.basicConfig(level=service_sidecar_settings.loglevel) - logging.root.setLevel(service_sidecar_settings.loglevel) - logger.debug(service_sidecar_settings.json(indent=2)) + logging.basicConfig(level=dynamic_sidecar_settings.loglevel) + logging.root.setLevel(dynamic_sidecar_settings.loglevel) + logger.debug(dynamic_sidecar_settings.json(indent=2)) - application = FastAPI(debug=service_sidecar_settings.debug) + application = FastAPI(debug=dynamic_sidecar_settings.debug) # store "settings" and "shared_store" for later usage - application.state.settings = service_sidecar_settings - application.state.shared_store = SharedStore(settings=service_sidecar_settings) + application.state.settings = dynamic_sidecar_settings + application.state.shared_store = SharedStore(settings=dynamic_sidecar_settings) # used to keep track of the health of the application # also will be used in the /health endpoint application.state.application_health = ApplicationHealth() # enable debug if required - if service_sidecar_settings.is_development_mode: + if dynamic_sidecar_settings.is_development_mode: remote_debug_setup(application) # add routing paths diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/main.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/main.py similarity index 76% rename from services/service-sidecar/src/simcore_service_service_sidecar/main.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/main.py index cc52402383d..7247d675c90 100644 --- a/services/service-sidecar/src/simcore_service_service_sidecar/main.py +++ b/services/dynamic-sidecar/src/simcore_service_service_sidecar/main.py @@ -3,8 +3,8 @@ import uvicorn from fastapi import FastAPI -from simcore_service_service_sidecar.application import assemble_application -from simcore_service_service_sidecar.settings import ServiceSidecarSettings +from simcore_service_dynamic_sidecar.application import assemble_application +from simcore_service_dynamic_sidecar.settings import ServiceSidecarSettings current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -16,7 +16,7 @@ def main(): settings: ServiceSidecarSettings = app.state.settings uvicorn.run( - "simcore_service_service_sidecar.main:app", + "simcore_service_dynamic_sidecar.main:app", host=settings.host, port=settings.port, reload=settings.is_development_mode, diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/models.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/models.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/models.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/models.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/remote_debug.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/remote_debug.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/remote_debug.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/remote_debug.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/settings.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/settings.py similarity index 98% rename from services/service-sidecar/src/simcore_service_service_sidecar/settings.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/settings.py index 8a767f94a3b..0b841e3fa68 100644 --- a/services/service-sidecar/src/simcore_service_service_sidecar/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_service_sidecar/settings.py @@ -84,4 +84,4 @@ def loglevel(self) -> int: class Config: case_sensitive = False - env_prefix = "SERVICE_SIDECAR_" + env_prefix = "DYNAMIC_SIDECAR_" diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/shared_handlers.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/shared_handlers.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/shared_handlers.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/shared_handlers.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/storage.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/storage.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/storage.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/storage.py diff --git a/services/service-sidecar/src/simcore_service_service_sidecar/utils.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/utils.py similarity index 100% rename from services/service-sidecar/src/simcore_service_service_sidecar/utils.py rename to services/dynamic-sidecar/src/simcore_service_service_sidecar/utils.py diff --git a/services/service-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py similarity index 88% rename from services/service-sidecar/tests/conftest.py rename to services/dynamic-sidecar/tests/conftest.py index ddfc89b4e97..a54e1873e94 100644 --- a/services/service-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -11,10 +11,10 @@ import pytest from async_asgi_testclient import TestClient from fastapi import FastAPI -from simcore_service_service_sidecar.application import assemble_application -from simcore_service_service_sidecar.settings import ServiceSidecarSettings -from simcore_service_service_sidecar.shared_handlers import write_file_and_run_command -from simcore_service_service_sidecar.storage import SharedStore +from simcore_service_dynamic_sidecar.application import assemble_application +from simcore_service_dynamic_sidecar.settings import ServiceSidecarSettings +from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command +from simcore_service_dynamic_sidecar.storage import SharedStore @pytest.fixture(scope="module", autouse=True) @@ -23,8 +23,8 @@ def app() -> FastAPI: os.environ, { "SC_BOOT_MODE": "production", - "SERVICE_SIDECAR_compose_namespace": "test-space", - "SERVICE_SIDECAR_docker_compose_down_timeout": "15", + "DYNAMIC_SIDECAR_compose_namespace": "test-space", + "DYNAMIC_SIDECAR_docker_compose_down_timeout": "15", }, ): return assemble_application() diff --git a/services/service-sidecar/tests/unit/test_api_compose.py b/services/dynamic-sidecar/tests/unit/test_api_compose.py similarity index 100% rename from services/service-sidecar/tests/unit/test_api_compose.py rename to services/dynamic-sidecar/tests/unit/test_api_compose.py diff --git a/services/service-sidecar/tests/unit/test_api_container.py b/services/dynamic-sidecar/tests/unit/test_api_container.py similarity index 98% rename from services/service-sidecar/tests/unit/test_api_container.py rename to services/dynamic-sidecar/tests/unit/test_api_container.py index 24b9d11ca25..ea224d8468d 100644 --- a/services/service-sidecar/tests/unit/test_api_container.py +++ b/services/dynamic-sidecar/tests/unit/test_api_container.py @@ -6,7 +6,7 @@ import pytest from async_asgi_testclient import TestClient -from simcore_service_service_sidecar.storage import SharedStore +from simcore_service_dynamic_sidecar.storage import SharedStore @pytest.fixture diff --git a/services/service-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py similarity index 98% rename from services/service-sidecar/tests/unit/test_api_containers.py rename to services/dynamic-sidecar/tests/unit/test_api_containers.py index a0bb07e4b9f..2622e5ba808 100644 --- a/services/service-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -7,7 +7,7 @@ import pytest from async_asgi_testclient import TestClient -from simcore_service_service_sidecar.storage import SharedStore +from simcore_service_dynamic_sidecar.storage import SharedStore @pytest.fixture diff --git a/services/service-sidecar/tests/unit/test_api_health.py b/services/dynamic-sidecar/tests/unit/test_api_health.py similarity index 82% rename from services/service-sidecar/tests/unit/test_api_health.py rename to services/dynamic-sidecar/tests/unit/test_api_health.py index d714e07f214..6aa860322e2 100644 --- a/services/service-sidecar/tests/unit/test_api_health.py +++ b/services/dynamic-sidecar/tests/unit/test_api_health.py @@ -1,6 +1,6 @@ import pytest from async_asgi_testclient import TestClient -from simcore_service_service_sidecar.models import ApplicationHealth +from simcore_service_dynamic_sidecar.models import ApplicationHealth @pytest.mark.asyncio diff --git a/services/service-sidecar/tests/unit/test_api_push_retrive_state.py b/services/dynamic-sidecar/tests/unit/test_api_push_retrive_state.py similarity index 100% rename from services/service-sidecar/tests/unit/test_api_push_retrive_state.py rename to services/dynamic-sidecar/tests/unit/test_api_push_retrive_state.py diff --git a/services/service-sidecar/tox.ini b/services/dynamic-sidecar/tox.ini similarity index 62% rename from services/service-sidecar/tox.ini rename to services/dynamic-sidecar/tox.ini index 0051d0094a9..973c61ba2a1 100644 --- a/services/service-sidecar/tox.ini +++ b/services/dynamic-sidecar/tox.ini @@ -8,9 +8,9 @@ skipsdist=True [testenv] # install pytest in the virtualenv where commands will be executed deps = - -r requirements/dev.txt + -r requirements/ci.txt commands = - pytest -vvv -s --cov=simcore_service_service_sidecar --cov-append --cov-report=term-missing --capture=sys tests/unit + pytest -vvv -s --cov=simcore_service_dynamic_sidecar --cov-append --cov-report=term-missing --capture=sys tests/unit depends = {py36}: clean @@ -22,10 +22,10 @@ basepython = python3.6 deps = -r requirements/_tools.txt commands = - isort --check setup.py src/simcore_service_service_sidecar tests - black --check src/simcore_service_service_sidecar tests/ - pylint --rcfile=../../.pylintrc src/simcore_service_service_sidecar tests/ - mypy src/simcore_service_service_sidecar tests/ --ignore-missing-imports + isort --check setup.py src/simcore_service_dynamic_sidecar tests + black --check src/simcore_service_dynamic_sidecar tests/ + pylint --rcfile=../../.pylintrc src/simcore_service_dynamic_sidecar tests/ + mypy src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports # used for development @@ -34,10 +34,10 @@ basepython = python3.6 deps = -r requirements/_tools.txt commands = - isort setup.py src/simcore_service_service_sidecar tests - black src/simcore_service_service_sidecar tests/ - pylint --rcfile=../../.pylintrc src/simcore_service_service_sidecar tests/ - mypy src/simcore_service_service_sidecar tests/ --ignore-missing-imports + isort setup.py src/simcore_service_dynamic_sidecar tests + black src/simcore_service_dynamic_sidecar tests/ + pylint --rcfile=../../.pylintrc src/simcore_service_dynamic_sidecar tests/ + mypy src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports [testenv:report] basepython = python3.6 From cf64a12f8c4272dbd8d424b69fdc8e3aef595835 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:24:08 +0200 Subject: [PATCH 008/102] renaming missing folder --- .../__init__.py | 0 .../api/__init__.py | 1 + .../api/_routing.py | 23 ++ .../api/compose.py | 167 +++++++++++ .../api/container.py | 97 ++++++ .../api/containers.py | 76 +++++ .../api/health.py | 13 + .../api/push.py | 18 ++ .../api/retrive.py | 24 ++ .../api/state.py | 24 ++ .../application.py | 51 ++++ .../simcore_service_dynamic_sidecar/main.py | 31 ++ .../simcore_service_dynamic_sidecar/models.py | 7 + .../remote_debug.py | 33 ++ .../settings.py | 87 ++++++ .../shared_handlers.py | 64 ++++ .../storage.py | 51 ++++ .../simcore_service_dynamic_sidecar/utils.py | 282 ++++++++++++++++++ 18 files changed, 1049 insertions(+) create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/__init__.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/__init__.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/push.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/retrive.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/state.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/remote_debug.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/storage.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/__init__.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/__init__.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/__init__.py new file mode 100644 index 00000000000..94ff56968e4 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/__init__.py @@ -0,0 +1 @@ +from ._routing import main_router diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py new file mode 100644 index 00000000000..225504128e9 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py @@ -0,0 +1,23 @@ +# module acting as root for all routes + +from fastapi import APIRouter + +from .compose import compose_router +from .container import container_router +from .containers import containers_router +from .health import health_router +from .push import push_router +from .retrive import retrive_router +from .state import state_router + +# setup and register all routes here form different modules +main_router = APIRouter() +main_router.include_router(health_router) +main_router.include_router(compose_router) +main_router.include_router(containers_router) +main_router.include_router(container_router) +main_router.include_router(state_router) +main_router.include_router(retrive_router) +main_router.include_router(push_router) + +__all__ = ["main_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py new file mode 100644 index 00000000000..2c8dd4becb9 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -0,0 +1,167 @@ +import logging +import traceback +from typing import Optional + +from fastapi import APIRouter, Request, Response +from fastapi.responses import PlainTextResponse + +from ..settings import ServiceSidecarSettings +from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command +from ..storage import SharedStore +from ..utils import InvalidComposeSpec + +logger = logging.getLogger(__name__) +compose_router = APIRouter() + + +@compose_router.post( + "/compose:store", response_class=PlainTextResponse, responses={204: {"model": None}} +) +async def store_docker_compose_spec_for_later_usage( + request: Request, response: Response +) -> Optional[str]: + """ Expects the docker-compose spec as raw-body utf-8 encoded text """ + body_as_text = (await request.body()).decode("utf-8") + + shared_store: SharedStore = request.app.state.shared_store + + try: + shared_store.put_spec(body_as_text) + except InvalidComposeSpec as e: + logger.warning("Error detected %s", traceback.format_exc()) + response.status_code = 400 + return str(e) + + response.status_code = 204 + return None + + +@compose_router.post("/compose:preload", response_class=PlainTextResponse) +async def create_docker_compose_configuration_containers_without_starting( + request: Request, response: Response, command_timeout: float +) -> str: + """ Expects the docker-compose spec as raw-body utf-8 encoded text """ + body_as_text = (await request.body()).decode("utf-8") + + settings: ServiceSidecarSettings = request.app.state.settings + shared_store: SharedStore = request.app.state.shared_store + + try: + shared_store.put_spec(body_as_text) + except InvalidComposeSpec as e: + logger.warning("Error detected %s", traceback.format_exc()) + response.status_code = 400 + return str(e) + + # --no-build might be a security risk building is disabled + command = "docker-compose -p {project} -f {file_path} up --no-build --no-start" + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=shared_store.get_spec(), + command=command, + command_timeout=command_timeout, + ) + + response.status_code = 200 if finished_without_errors else 400 + return stdout + + +@compose_router.post("/compose", response_class=PlainTextResponse) +async def start_or_update_docker_compose_configuration( + request: Request, response: Response, command_timeout: float +) -> str: + """ Expects the docker-compose spec as raw-body utf-8 encoded text """ + settings: ServiceSidecarSettings = request.app.state.settings + shared_store: SharedStore = request.app.state.shared_store + + # --no-build might be a security risk building is disabled + command = "docker-compose -p {project} -f {file_path} up --no-build -d" + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=shared_store.get_spec(), + command=command, + command_timeout=command_timeout, + ) + + response.status_code = 200 if finished_without_errors else 400 + return stdout + + +@compose_router.get("/compose:pull", response_class=PlainTextResponse) +async def pull_docker_required_docker_images( + request: Request, response: Response, command_timeout: float +) -> str: + """ Expects the docker-compose spec as raw-body utf-8 encoded text """ + shared_store: SharedStore = request.app.state.shared_store + settings: ServiceSidecarSettings = request.app.state.settings + + stored_compose_content = shared_store.get_spec() + if stored_compose_content is None: + response.status_code = 400 + return "No started spec to stop was found" + + command = "docker-compose -p {project} -f {file_path} pull --include-deps" + + try: + # mark as pulling images + shared_store.set_is_pulling_containsers() + + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=stored_compose_content, + command=command, + command_timeout=command_timeout, + ) + finally: + # remove mark + shared_store.unset_is_pulling_containsers() + + response.status_code = 200 if finished_without_errors else 400 + return stdout + + +@compose_router.put("/compose:stop", response_class=PlainTextResponse) +async def stop_containers_without_removing_them( + request: Request, response: Response, command_timeout: float +) -> str: + """Stops the previously started service + and returns the docker-compose output""" + shared_store: SharedStore = request.app.state.shared_store + settings: ServiceSidecarSettings = request.app.state.settings + + stored_compose_content = shared_store.get_spec() + if stored_compose_content is None: + response.status_code = 400 + return "No started spec to stop was found" + + command = ( + "docker-compose -p {project} -f {file_path} stop -t {stop_and_remove_timeout}" + ) + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=stored_compose_content, + command=command, + command_timeout=command_timeout, + ) + + response.status_code = 200 if finished_without_errors else 400 + return stdout + + +@compose_router.delete("/compose", response_class=PlainTextResponse) +async def remove_docker_compose_configuration( + request: Request, response: Response, command_timeout: float +) -> str: + """Removes the previously started service + and returns the docker-compose output""" + finished_without_errors, stdout = await remove_the_compose_spec( + shared_store=request.app.state.shared_store, + settings=request.app.state.settings, + command_timeout=command_timeout, + ) + + response.status_code = 200 if finished_without_errors else 400 + return stdout + + +__all__ = ["compose_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py new file mode 100644 index 00000000000..3182335af6d --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py @@ -0,0 +1,97 @@ +from typing import Any, Dict, Union + +import aiodocker +from fastapi import APIRouter, Query, Request, Response + +from ..storage import SharedStore + +container_router = APIRouter() + + +@container_router.get("/container/logs") +async def get_container_logs( + # pylint: disable=unused-argument + request: Request, + response: Response, + container: str, + since: int = Query( + 0, + title="Timstamp", + description="Only return logs since this time, as a UNIX timestamp", + ), + until: int = Query( + 0, + title="Timstamp", + description="Only return logs before this time, as a UNIX timestamp", + ), + timestamps: bool = Query( + False, + title="Display timestamps", + description="Enabling this parameter will include timestamps in logs", + ), +) -> Union[str, Dict[str, Any]]: + """ Returns the logs of a given container if found """ + shared_store: SharedStore = request.app.state.shared_store + + if container not in shared_store.get_container_names(): + response.status_code = 400 + return dict(error=f"No container '{container}' was started") + + docker = aiodocker.Docker() + + try: + container_instance = await docker.containers.get(container) + + args = dict(stdout=True, stderr=True) + if timestamps: + args["timestamps"] = True + + return await container_instance.log(**args) + except aiodocker.exceptions.DockerError as e: + response.status_code = 400 + return dict(error=e.message) + + +@container_router.get("/container/inspect") +async def container_inspect( + request: Request, response: Response, container: str +) -> Dict[str, Any]: + """ Returns information about the container, like docker inspect command """ + shared_store: SharedStore = request.app.state.shared_store + + if container not in shared_store.get_container_names(): + response.status_code = 400 + return dict(error=f"No container '{container}' was started") + + docker = aiodocker.Docker() + + try: + container_instance = await docker.containers.get(container) + return await container_instance.show() + except aiodocker.exceptions.DockerError as e: + response.status_code = 400 + return dict(error=e.message) + + +@container_router.delete("/container/remove") +async def container_remove( + request: Request, response: Response, container: str +) -> Union[bool, Dict[str, Any]]: + shared_store: SharedStore = request.app.state.shared_store + + if container not in shared_store.get_container_names(): + response.status_code = 400 + return dict(error=f"No container '{container}' was started") + + docker = aiodocker.Docker() + + try: + container_instance = await docker.containers.get(container) + await container_instance.delete() + return True + except aiodocker.exceptions.DockerError as e: + response.status_code = 400 + return dict(error=e.message) + + +__all__ = ["container_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py new file mode 100644 index 00000000000..e22f2adba92 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -0,0 +1,76 @@ +from typing import Any, Dict, List + +import aiodocker +from fastapi import APIRouter, Request, Response + +containers_router = APIRouter() + + +@containers_router.get("/containers") +async def get_spawned_container_names(request: Request) -> List[str]: + """ Returns a list of containers created using docker-compose """ + return request.app.state.shared_store.get_container_names() + + +@containers_router.get("/containers:inspect") +async def containers_inspect(request: Request, response: Response) -> Dict[str, Any]: + """ Returns information about the container, like docker inspect command """ + docker = aiodocker.Docker() + + container_names = request.app.state.shared_store.get_container_names() + container_names = container_names if container_names else {} + + results = {} + + for container in container_names: + try: + container_instance = await docker.containers.get(container) + results[container] = await container_instance.show() + except aiodocker.exceptions.DockerError as e: + response.status_code = 400 + return dict(error=e.message) + + return results + + +@containers_router.get("/containers:docker-status") +async def containers_docker_status( + request: Request, response: Response +) -> Dict[str, Any]: + """ Returns the status of the containers """ + + def assemble_entry(status: str, error: str = "") -> Dict[str, str]: + return {"Status": status, "Error": error} + + docker = aiodocker.Docker() + + shared_store = request.app.state.shared_store + container_names = shared_store.get_container_names() + container_names = container_names if container_names else {} + + # if containers are being pulled, return pulling (fake status) + if shared_store.is_pulling_containsers: + # pulling is a fake state use to share more information with the frontend + return {x: assemble_entry(status="pulling") for x in container_names} + + results = {} + + for container in container_names: + try: + container_instance = await docker.containers.get(container) + container_inspect = await container_instance.show() + container_state = container_inspect.get("State", {}) + + # pending is another fake state use to share more information with the frontend + results[container] = { + "Status": container_state.get("Status", "pending"), + "Error": container_state.get("Error", ""), + } + except aiodocker.exceptions.DockerError as e: + response.status_code = 400 + return dict(error=e.message) + + return results + + +__all__ = ["containers_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py new file mode 100644 index 00000000000..f5dad519250 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py @@ -0,0 +1,13 @@ +from fastapi import APIRouter, Request + +from ..models import ApplicationHealth + +health_router = APIRouter() + + +@health_router.get("/health", response_model=ApplicationHealth) +async def health_endpoint(request: Request) -> ApplicationHealth: + return request.app.state.application_health + + +__all__ = ["health_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/push.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/push.py new file mode 100644 index 00000000000..02c7328e281 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/push.py @@ -0,0 +1,18 @@ +# acts as mock for now + +import logging + +from fastapi import APIRouter + +logger = logging.getLogger(__name__) + +push_router = APIRouter() + + +@push_router.post("/push") +async def post_api() -> str: + logger.warning("TODO: still need to implement") + return "" + + +__all__ = ["push_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/retrive.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/retrive.py new file mode 100644 index 00000000000..9e441ccd6c2 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/retrive.py @@ -0,0 +1,24 @@ +# acts as mock for now + +import logging + +from fastapi import APIRouter + +logger = logging.getLogger(__name__) + +retrive_router = APIRouter() + + +@retrive_router.get("/retrive") +async def get_api() -> str: + logger.warning("TODO: still need to implement") + return "" + + +@retrive_router.post("/retrive") +async def post_api() -> str: + logger.warning("TODO: still need to implement") + return "" + + +__all__ = ["retrive_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/state.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/state.py new file mode 100644 index 00000000000..47f110cabff --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/state.py @@ -0,0 +1,24 @@ +# acts as mock for now + +import logging + +from fastapi import APIRouter + +logger = logging.getLogger(__name__) + +state_router = APIRouter() + + +@state_router.get("/state") +async def get_api() -> str: + logger.warning("TODO: still need to implement") + return "" + + +@state_router.post("/state") +async def post_api() -> str: + logger.warning("TODO: still need to implement") + return "" + + +__all__ = ["state_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py new file mode 100644 index 00000000000..2e707674b28 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py @@ -0,0 +1,51 @@ +import logging + +from fastapi import FastAPI + +from .api import main_router +from .models import ApplicationHealth +from .remote_debug import setup as remote_debug_setup +from .settings import ServiceSidecarSettings +from .shared_handlers import on_shutdown_handler +from .storage import SharedStore + +logger = logging.getLogger(__name__) + + +def assemble_application() -> FastAPI: + """ + Creates the application from using the env vars as a context + Also stores inside the state all instances of classes + needed in other requests and used to share data. + """ + + dynamic_sidecar_settings = ServiceSidecarSettings.create() + + logging.basicConfig(level=dynamic_sidecar_settings.loglevel) + logging.root.setLevel(dynamic_sidecar_settings.loglevel) + logger.debug(dynamic_sidecar_settings.json(indent=2)) + + application = FastAPI(debug=dynamic_sidecar_settings.debug) + + # store "settings" and "shared_store" for later usage + application.state.settings = dynamic_sidecar_settings + application.state.shared_store = SharedStore(settings=dynamic_sidecar_settings) + # used to keep track of the health of the application + # also will be used in the /health endpoint + application.state.application_health = ApplicationHealth() + + # enable debug if required + if dynamic_sidecar_settings.is_development_mode: + remote_debug_setup(application) + + # add routing paths + application.include_router(main_router) + + # setting up handler for lifecycle + async def on_shutdown() -> None: + await on_shutdown_handler(application) + logger.info("shutdown cleanup completed") + + application.add_event_handler("shutdown", on_shutdown) + + return application diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py new file mode 100644 index 00000000000..7247d675c90 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py @@ -0,0 +1,31 @@ +import sys +from pathlib import Path + +import uvicorn +from fastapi import FastAPI +from simcore_service_dynamic_sidecar.application import assemble_application +from simcore_service_dynamic_sidecar.settings import ServiceSidecarSettings + +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + + +app: FastAPI = assemble_application() + + +def main(): + settings: ServiceSidecarSettings = app.state.settings + + uvicorn.run( + "simcore_service_dynamic_sidecar.main:app", + host=settings.host, + port=settings.port, + reload=settings.is_development_mode, + reload_dirs=[ + current_dir, + ], + log_level=settings.log_level_name.lower(), + ) + + +if __name__ == "__main__": + main() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models.py new file mode 100644 index 00000000000..7d2080340c9 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel, Field + + +class ApplicationHealth(BaseModel): + is_healthy: bool = Field( + True, description="returns True if the service sis running correctly" + ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/remote_debug.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/remote_debug.py new file mode 100644 index 00000000000..75e49be73a9 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/remote_debug.py @@ -0,0 +1,33 @@ +""" Setup remote debugger with Python Tools for Visual Studio (PTVSD) + +""" +import logging + +from fastapi import FastAPI + +logger = logging.getLogger(__name__) + + +def setup(app: FastAPI): + settings = app.state.settings + + def on_startup() -> None: + try: + logger.debug("Enabling attach ptvsd ...") + # + # SEE https://github.com/microsoft/ptvsd#enabling-debugging + # + import ptvsd # pylint: disable=import-outside-toplevel + + ptvsd.enable_attach(address=(settings.host, settings.remote_debug_port)) + + except ImportError as err: + raise Exception( + "Cannot enable remote debugging. Please install ptvsd first" + ) from err + + logger.info( + "Remote debugging enabled: listening port %s", settings.remote_debug_port + ) + + app.add_event_handler("startup", on_startup) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py new file mode 100644 index 00000000000..0b841e3fa68 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py @@ -0,0 +1,87 @@ +import logging +from typing import Optional + +from models_library.basic_types import BootModeEnum, PortInt +from pydantic import BaseSettings, Field, PositiveInt, validator + + +class ServiceSidecarSettings(BaseSettings): + @classmethod + def create(cls, **settings_kwargs) -> "ServiceSidecarSettings": + return cls( + **settings_kwargs, + ) + + boot_mode: Optional[BootModeEnum] = Field( + ..., + description="boot mode helps determine if in development mode or normal operation", + env="SC_BOOT_MODE", + ) + + # LOGGING + log_level_name: str = Field("DEBUG", env="LOG_LEVEL") + + @validator("log_level_name") + @classmethod + def match_logging_level(cls, v) -> str: + try: + getattr(logging, v.upper()) + except AttributeError as err: + raise ValueError(f"{v.upper()} is not a valid level") from err + return v.upper() + + # SERVICE SERVER (see : https://www.uvicorn.org/settings/) + host: str = Field( + "0.0.0.0", # nosec + description="host where to bind the application on which to serve", + ) + port: PortInt = Field( + 8000, description="port where the server will be currently serving" + ) + + compose_namespace: str = Field( + ..., + description=( + "To avoid collisions when scheduling on the same node, this " + "will be compsoed by the project_uuid and node_uuid." + ), + ) + + max_combined_container_name_length: PositiveInt = Field( + 63, description="the container name which will be used as hostname" + ) + + stop_and_remove_timeout: PositiveInt = Field( + 5, + description=( + "When receiving SIGTERM the process has 10 seconds to cleanup its children " + "forcing our children to stop in 5 seconds in all cases" + ), + ) + + debug: bool = Field( + False, + description="If set to True the application will boot into debug mode", + env="DEBUG", + ) + + remote_debug_port: PortInt = Field( + 3000, description="ptsvd remote debugger starting port" + ) + + docker_compose_down_timeout: PositiveInt = Field( + ..., description="used during shutdown when containers swapend will be removed" + ) + + @property + def is_development_mode(self): + """If in development mode this will be True""" + return self.boot_mode == BootModeEnum.DEVELOPMENT + + @property + def loglevel(self) -> int: + return getattr(logging, self.log_level_name) + + class Config: + case_sensitive = False + env_prefix = "DYNAMIC_SIDECAR_" diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py new file mode 100644 index 00000000000..2e41c86f165 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py @@ -0,0 +1,64 @@ +import logging +from typing import Optional, Tuple + +from fastapi import FastAPI + +from .settings import ServiceSidecarSettings +from .storage import SharedStore +from .utils import async_command, write_to_tmp_file + +logger = logging.getLogger(__name__) + + +async def write_file_and_run_command( + settings: ServiceSidecarSettings, + file_content: Optional[str], + command: str, + command_timeout: float, +) -> Tuple[bool, str]: + """ The command which accepts {file_path} as an argument for string formatting """ + + # pylint: disable=not-async-context-manager + async with write_to_tmp_file(file_content) as file_path: + formatted_command = command.format( + file_path=file_path, + project=settings.compose_namespace, + stop_and_remove_timeout=settings.stop_and_remove_timeout, + ) + logger.debug("Will run command\n'%s':\n%s", formatted_command, file_content) + return await async_command(formatted_command, command_timeout) + + +async def remove_the_compose_spec( + shared_store: SharedStore, settings: ServiceSidecarSettings, command_timeout: float +) -> Tuple[bool, str]: + + stored_compose_content = shared_store.get_spec() + if stored_compose_content is None: + return True, "No started spec to remove was found" + + command = ( + "docker-compose -p {project} -f {file_path} " + "down --remove-orphans -t {stop_and_remove_timeout}" + ) + result = await write_file_and_run_command( + settings=settings, + file_content=stored_compose_content, + command=command, + command_timeout=command_timeout, + ) + shared_store.put_spec(None) # removing compose-file spec + return result + + +async def on_shutdown_handler(app: FastAPI) -> None: + logging.info("Going to remove spawned containers") + shared_store: SharedStore = app.state.shared_store + settings: ServiceSidecarSettings = app.state.settings + + result = await remove_the_compose_spec( + shared_store=shared_store, + settings=settings, + command_timeout=settings.docker_compose_down_timeout, + ) + logging.info("Container removal did_succeed=%s\n%s", result[0], result[1]) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/storage.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/storage.py new file mode 100644 index 00000000000..3ab22024512 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/storage.py @@ -0,0 +1,51 @@ +from typing import Any, Dict, List, Optional + +from .settings import ServiceSidecarSettings +from .utils import assemble_container_names, validate_compose_spec + + +class SharedStore: + """Define custom storage abstraction for easy future extension""" + + __slots__ = ("_storage", "_settings", "_is_pulling_containsers") + + _K_COMPOSE_SPEC = "compose_spec" + _K_CONTAINER_NAMES = "container_names" + + def __set_as_compose_spec_none(self): + self._storage[self._K_COMPOSE_SPEC] = None + self._storage[self._K_CONTAINER_NAMES] = [] + + def __init__(self, settings: ServiceSidecarSettings): + self._storage: Dict[str, Any] = {} + self._settings: ServiceSidecarSettings = settings + self._is_pulling_containsers: bool = False + self.__set_as_compose_spec_none() + + def put_spec(self, compose_file_content: Optional[str]) -> None: + if compose_file_content is None: + self.__set_as_compose_spec_none() + return + + self._storage[self._K_COMPOSE_SPEC] = validate_compose_spec( + settings=self._settings, compose_file_content=compose_file_content + ) + self._storage[self._K_CONTAINER_NAMES] = assemble_container_names( + self._storage[self._K_COMPOSE_SPEC] + ) + + def get_spec(self) -> Optional[str]: + return self._storage.get(self._K_COMPOSE_SPEC) + + def get_container_names(self) -> List[str]: + return self._storage[self._K_CONTAINER_NAMES] + + @property + def is_pulling_containsers(self) -> bool: + return self._is_pulling_containsers + + def set_is_pulling_containsers(self) -> None: + self._is_pulling_containsers = True + + def unset_is_pulling_containsers(self) -> None: + self._is_pulling_containsers = False diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py new file mode 100644 index 00000000000..44f0b179fce --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py @@ -0,0 +1,282 @@ +import asyncio +import json +import logging +import os +import re +import tempfile +import traceback +from pathlib import Path +from typing import Any, Dict, Generator, List, Tuple + +import aiofiles +import yaml +from async_generator import asynccontextmanager +from async_timeout import timeout + +from .settings import ServiceSidecarSettings + +TEMPLATE_SEARCH_PATTERN = r"%%(.*?)%%" + +logger = logging.getLogger(__name__) + + +class InvalidComposeSpec(Exception): + """Exception used to signal incorrect docker-compose configuration file""" + + +@asynccontextmanager +async def write_to_tmp_file(file_contents): + """Disposes of file on exit""" + # pylint: disable=protected-access,stop-iteration-return + file_path = Path("/") / f"tmp/{next(tempfile._get_candidate_names())}" + async with aiofiles.open(file_path, mode="w") as tmp_file: + await tmp_file.write(file_contents) + try: + yield file_path + finally: + await aiofiles.os.remove(file_path) + + +async def async_command(command, command_timeout: float) -> Tuple[bool, str]: + """Returns if the command exited correctly and the stdout of the command """ + proc = await asyncio.create_subprocess_shell( + command, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + + # because the Processes returned by create_subprocess_shell it is not possible to + # have a timeout otherwise nor to stream the response from the process. + try: + async with timeout(command_timeout): + stdout, _ = await proc.communicate() + except asyncio.TimeoutError: + message = ( + f"{traceback.format_exc()}\nTimed out after {command_timeout} " + f"seconds while running {command}" + ) + logger.warning(message) + return False, message + + decoded_stdout = stdout.decode() + logger.info("'%s' result:\n%s", command, decoded_stdout) + finished_without_errors = proc.returncode == 0 + + return finished_without_errors, decoded_stdout + + +def _assemble_container_name( + settings: ServiceSidecarSettings, + service_key: str, + user_given_container_name: str, + index: int, +) -> str: + strings_to_use = [ + settings.compose_namespace, + str(index), + user_given_container_name, + service_key, + ] + + container_name = "-".join([x for x in strings_to_use if len(x) > 0])[ + : settings.max_combined_container_name_length + ] + return container_name.replace("_", "-") + + +def _get_forwarded_env_vars(container_key: str) -> List[str]: + """retruns env vars targeted to each container in the compose spec""" + results = [ + # some services expect it, using it as empty + "SIMCORE_NODE_BASEPATH=", + ] + for key in os.environ.keys(): + if key.startswith("FORWARD_ENV_"): + new_entry_key = key.replace("FORWARD_ENV_", "") + + # parsing `VAR={"destination_container": "destination_container", "env_var": "PAYLOAD"}` + new_entry_payload = json.loads(os.environ[key]) + if new_entry_payload["destination_container"] != container_key: + continue + + new_entry_value = new_entry_payload["env_var"] + new_entry = f"{new_entry_key}={new_entry_value}" + results.append(new_entry) + return results + + +def _extract_templated_entries(text: str) -> List[str]: + return re.findall(TEMPLATE_SEARCH_PATTERN, text) + + +def _apply_templating_directives( + stringified_compose_spec: str, + services: Dict[str, Any], + spec_services_to_container_name: Dict[str, str], +) -> str: + """ + Some custom rules are supported for replacing `container_name` + with the following syntax `%%container_name.SERVICE_KEY_NAME%%`, + where `SERVICE_KEY_NAME` targets a container in the compose spec + + If the directive cannot be applied it will just be left untouched + """ + matches = set(_extract_templated_entries(stringified_compose_spec)) + for match in matches: + parts = match.split(".") + + if len(parts) != 2: + continue # templating will be skipped + + target_property = parts[0] + services_key = parts[1] + if target_property != "container_name": + continue # also ignore if the container_name is not the directive to replace + + remapped_service_key = spec_services_to_container_name[services_key] + replace_with = services.get(remapped_service_key, {}).get( + "container_name", None + ) + if replace_with is None: + continue # also skip here if nothing was found + + match_pattern = f"%%{match}%%" + stringified_compose_spec = stringified_compose_spec.replace( + match_pattern, replace_with + ) + + return stringified_compose_spec + + +def _merge_env_vars( + compose_spec_env_vars: List[str], settings_env_vars: List[str] +) -> List[str]: + def _gen_parts_env_vars( + env_vars: List[str], + ) -> Generator[Tuple[str, str], None, None]: + for env_var in env_vars: + key, value = env_var.split("=") + yield key, value + + # pylint: disable=unnecessary-comprehension + dict_spec_env_vars = {k: v for k, v in _gen_parts_env_vars(compose_spec_env_vars)} + dict_settings_env_vars = {k: v for k, v in _gen_parts_env_vars(settings_env_vars)} + + # overwrite spec vars with vars from settings + for key, value in dict_settings_env_vars.items(): + dict_spec_env_vars[key] = value + + # returns a single list of vars + return [f"{k}={v}" for k, v in dict_spec_env_vars.items()] + + +def _inject_backend_networking( + parsed_compose_spec: Dict[str, Any], network_name: str = "__backend__" +) -> None: + """ + Put all containers in the compose spec in the same network. + The `network_name` must only be unique inside the user defined spec; + docker-compose will add some prefix to it. + """ + + networks = parsed_compose_spec.get("networks", {}) + networks[network_name] = None + + for service_content in parsed_compose_spec["services"].values(): + service_networks = service_content.get("networks", []) + service_networks.append(network_name) + service_content["networks"] = service_networks + + parsed_compose_spec["networks"] = networks + + +def validate_compose_spec( + settings: ServiceSidecarSettings, compose_file_content: str +) -> str: + """ + Checks the following: + - proper yaml format + - no "container_name" service property allowed, because it can + spawn 2 cotainers with the same name + """ + + try: + parsed_compose_spec = yaml.safe_load(compose_file_content) + except yaml.YAMLError as e: + raise InvalidComposeSpec( + f"{str(e)}\n{compose_file_content}\nProvided yaml is not valid!" + ) from e + + if parsed_compose_spec is None or not isinstance(parsed_compose_spec, dict): + raise InvalidComposeSpec(f"{compose_file_content}\nProvided yaml is not valid!") + + if not {"version", "services"}.issubset(set(parsed_compose_spec.keys())): + raise InvalidComposeSpec(f"{compose_file_content}\nProvided yaml is not valid!") + + version = parsed_compose_spec["version"] + if version.startswith("1"): + raise InvalidComposeSpec(f"Provided spec version '{version}' is not supported") + + spec_services_to_container_name: Dict[str, str] = {} + + spec_services = parsed_compose_spec["services"] + for index, service in enumerate(spec_services): + service_content = spec_services[service] + + # assemble and inject the container name + user_given_container_name = service_content.get("container_name", "") + container_name = _assemble_container_name( + settings, service, user_given_container_name, index + ) + service_content["container_name"] = container_name + spec_services_to_container_name[service] = container_name + + # inject forwarded environment variables + environment_entries = service_content.get("environment", []) + service_settings_env_vars = _get_forwarded_env_vars(service) + service_content["environment"] = _merge_env_vars( + compose_spec_env_vars=environment_entries, + settings_env_vars=service_settings_env_vars, + ) + + # if more then one container is defined, add an "backend" network + if len(spec_services) > 1: + _inject_backend_networking(parsed_compose_spec) + + # replace service_key with the container_name int the dict + for service_key in list(spec_services.keys()): + container_name_service_key = spec_services_to_container_name[service_key] + service_data = spec_services.pop(service_key) + + depends_on = service_data.get("depends_on", None) + if depends_on is not None: + service_data["depends_on"] = [ + # replaces with the container name + # if not found it leaves the old value + spec_services_to_container_name.get(x, x) + for x in depends_on + ] + + spec_services[container_name_service_key] = service_data + # TODO: replace names in depends_on keys + + # transform back to string and return + validated_compose_file_content = yaml.safe_dump(parsed_compose_spec) + + compose_spec = _apply_templating_directives( + stringified_compose_spec=validated_compose_file_content, + services=spec_services, + spec_services_to_container_name=spec_services_to_container_name, + ) + + return compose_spec + + +def assemble_container_names(validated_compose_content: str) -> List[str]: + """returns the list of container names from a validated compose_spec""" + parsed_compose_spec = yaml.safe_load(validated_compose_content) + return [ + service_data["container_name"] + for service_data in parsed_compose_spec["services"].values() + ] From 2ee8f644fffa1cb28ff27cb28bc2acaab2722944 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:24:16 +0200 Subject: [PATCH 009/102] renaming missing fodler part 2 --- .../__init__.py | 0 .../api/__init__.py | 1 - .../api/_routing.py | 23 -- .../api/compose.py | 167 ----------- .../api/container.py | 97 ------ .../api/containers.py | 76 ----- .../api/health.py | 13 - .../api/push.py | 18 -- .../api/retrive.py | 24 -- .../api/state.py | 24 -- .../application.py | 51 ---- .../simcore_service_service_sidecar/main.py | 31 -- .../simcore_service_service_sidecar/models.py | 7 - .../remote_debug.py | 33 -- .../settings.py | 87 ------ .../shared_handlers.py | 64 ---- .../storage.py | 51 ---- .../simcore_service_service_sidecar/utils.py | 282 ------------------ 18 files changed, 1049 deletions(-) delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/__init__.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/api/__init__.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/api/_routing.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/api/compose.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/api/container.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/api/containers.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/api/health.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/api/push.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/api/retrive.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/api/state.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/application.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/main.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/models.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/remote_debug.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/settings.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/shared_handlers.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/storage.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_service_sidecar/utils.py diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/__init__.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/__init__.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/__init__.py deleted file mode 100644 index 94ff56968e4..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from ._routing import main_router diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/_routing.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/_routing.py deleted file mode 100644 index 225504128e9..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/_routing.py +++ /dev/null @@ -1,23 +0,0 @@ -# module acting as root for all routes - -from fastapi import APIRouter - -from .compose import compose_router -from .container import container_router -from .containers import containers_router -from .health import health_router -from .push import push_router -from .retrive import retrive_router -from .state import state_router - -# setup and register all routes here form different modules -main_router = APIRouter() -main_router.include_router(health_router) -main_router.include_router(compose_router) -main_router.include_router(containers_router) -main_router.include_router(container_router) -main_router.include_router(state_router) -main_router.include_router(retrive_router) -main_router.include_router(push_router) - -__all__ = ["main_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/compose.py deleted file mode 100644 index 2c8dd4becb9..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/compose.py +++ /dev/null @@ -1,167 +0,0 @@ -import logging -import traceback -from typing import Optional - -from fastapi import APIRouter, Request, Response -from fastapi.responses import PlainTextResponse - -from ..settings import ServiceSidecarSettings -from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command -from ..storage import SharedStore -from ..utils import InvalidComposeSpec - -logger = logging.getLogger(__name__) -compose_router = APIRouter() - - -@compose_router.post( - "/compose:store", response_class=PlainTextResponse, responses={204: {"model": None}} -) -async def store_docker_compose_spec_for_later_usage( - request: Request, response: Response -) -> Optional[str]: - """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - body_as_text = (await request.body()).decode("utf-8") - - shared_store: SharedStore = request.app.state.shared_store - - try: - shared_store.put_spec(body_as_text) - except InvalidComposeSpec as e: - logger.warning("Error detected %s", traceback.format_exc()) - response.status_code = 400 - return str(e) - - response.status_code = 204 - return None - - -@compose_router.post("/compose:preload", response_class=PlainTextResponse) -async def create_docker_compose_configuration_containers_without_starting( - request: Request, response: Response, command_timeout: float -) -> str: - """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - body_as_text = (await request.body()).decode("utf-8") - - settings: ServiceSidecarSettings = request.app.state.settings - shared_store: SharedStore = request.app.state.shared_store - - try: - shared_store.put_spec(body_as_text) - except InvalidComposeSpec as e: - logger.warning("Error detected %s", traceback.format_exc()) - response.status_code = 400 - return str(e) - - # --no-build might be a security risk building is disabled - command = "docker-compose -p {project} -f {file_path} up --no-build --no-start" - finished_without_errors, stdout = await write_file_and_run_command( - settings=settings, - file_content=shared_store.get_spec(), - command=command, - command_timeout=command_timeout, - ) - - response.status_code = 200 if finished_without_errors else 400 - return stdout - - -@compose_router.post("/compose", response_class=PlainTextResponse) -async def start_or_update_docker_compose_configuration( - request: Request, response: Response, command_timeout: float -) -> str: - """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - settings: ServiceSidecarSettings = request.app.state.settings - shared_store: SharedStore = request.app.state.shared_store - - # --no-build might be a security risk building is disabled - command = "docker-compose -p {project} -f {file_path} up --no-build -d" - finished_without_errors, stdout = await write_file_and_run_command( - settings=settings, - file_content=shared_store.get_spec(), - command=command, - command_timeout=command_timeout, - ) - - response.status_code = 200 if finished_without_errors else 400 - return stdout - - -@compose_router.get("/compose:pull", response_class=PlainTextResponse) -async def pull_docker_required_docker_images( - request: Request, response: Response, command_timeout: float -) -> str: - """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - shared_store: SharedStore = request.app.state.shared_store - settings: ServiceSidecarSettings = request.app.state.settings - - stored_compose_content = shared_store.get_spec() - if stored_compose_content is None: - response.status_code = 400 - return "No started spec to stop was found" - - command = "docker-compose -p {project} -f {file_path} pull --include-deps" - - try: - # mark as pulling images - shared_store.set_is_pulling_containsers() - - finished_without_errors, stdout = await write_file_and_run_command( - settings=settings, - file_content=stored_compose_content, - command=command, - command_timeout=command_timeout, - ) - finally: - # remove mark - shared_store.unset_is_pulling_containsers() - - response.status_code = 200 if finished_without_errors else 400 - return stdout - - -@compose_router.put("/compose:stop", response_class=PlainTextResponse) -async def stop_containers_without_removing_them( - request: Request, response: Response, command_timeout: float -) -> str: - """Stops the previously started service - and returns the docker-compose output""" - shared_store: SharedStore = request.app.state.shared_store - settings: ServiceSidecarSettings = request.app.state.settings - - stored_compose_content = shared_store.get_spec() - if stored_compose_content is None: - response.status_code = 400 - return "No started spec to stop was found" - - command = ( - "docker-compose -p {project} -f {file_path} stop -t {stop_and_remove_timeout}" - ) - finished_without_errors, stdout = await write_file_and_run_command( - settings=settings, - file_content=stored_compose_content, - command=command, - command_timeout=command_timeout, - ) - - response.status_code = 200 if finished_without_errors else 400 - return stdout - - -@compose_router.delete("/compose", response_class=PlainTextResponse) -async def remove_docker_compose_configuration( - request: Request, response: Response, command_timeout: float -) -> str: - """Removes the previously started service - and returns the docker-compose output""" - finished_without_errors, stdout = await remove_the_compose_spec( - shared_store=request.app.state.shared_store, - settings=request.app.state.settings, - command_timeout=command_timeout, - ) - - response.status_code = 200 if finished_without_errors else 400 - return stdout - - -__all__ = ["compose_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/container.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/container.py deleted file mode 100644 index 3182335af6d..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/container.py +++ /dev/null @@ -1,97 +0,0 @@ -from typing import Any, Dict, Union - -import aiodocker -from fastapi import APIRouter, Query, Request, Response - -from ..storage import SharedStore - -container_router = APIRouter() - - -@container_router.get("/container/logs") -async def get_container_logs( - # pylint: disable=unused-argument - request: Request, - response: Response, - container: str, - since: int = Query( - 0, - title="Timstamp", - description="Only return logs since this time, as a UNIX timestamp", - ), - until: int = Query( - 0, - title="Timstamp", - description="Only return logs before this time, as a UNIX timestamp", - ), - timestamps: bool = Query( - False, - title="Display timestamps", - description="Enabling this parameter will include timestamps in logs", - ), -) -> Union[str, Dict[str, Any]]: - """ Returns the logs of a given container if found """ - shared_store: SharedStore = request.app.state.shared_store - - if container not in shared_store.get_container_names(): - response.status_code = 400 - return dict(error=f"No container '{container}' was started") - - docker = aiodocker.Docker() - - try: - container_instance = await docker.containers.get(container) - - args = dict(stdout=True, stderr=True) - if timestamps: - args["timestamps"] = True - - return await container_instance.log(**args) - except aiodocker.exceptions.DockerError as e: - response.status_code = 400 - return dict(error=e.message) - - -@container_router.get("/container/inspect") -async def container_inspect( - request: Request, response: Response, container: str -) -> Dict[str, Any]: - """ Returns information about the container, like docker inspect command """ - shared_store: SharedStore = request.app.state.shared_store - - if container not in shared_store.get_container_names(): - response.status_code = 400 - return dict(error=f"No container '{container}' was started") - - docker = aiodocker.Docker() - - try: - container_instance = await docker.containers.get(container) - return await container_instance.show() - except aiodocker.exceptions.DockerError as e: - response.status_code = 400 - return dict(error=e.message) - - -@container_router.delete("/container/remove") -async def container_remove( - request: Request, response: Response, container: str -) -> Union[bool, Dict[str, Any]]: - shared_store: SharedStore = request.app.state.shared_store - - if container not in shared_store.get_container_names(): - response.status_code = 400 - return dict(error=f"No container '{container}' was started") - - docker = aiodocker.Docker() - - try: - container_instance = await docker.containers.get(container) - await container_instance.delete() - return True - except aiodocker.exceptions.DockerError as e: - response.status_code = 400 - return dict(error=e.message) - - -__all__ = ["container_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/containers.py deleted file mode 100644 index e22f2adba92..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/containers.py +++ /dev/null @@ -1,76 +0,0 @@ -from typing import Any, Dict, List - -import aiodocker -from fastapi import APIRouter, Request, Response - -containers_router = APIRouter() - - -@containers_router.get("/containers") -async def get_spawned_container_names(request: Request) -> List[str]: - """ Returns a list of containers created using docker-compose """ - return request.app.state.shared_store.get_container_names() - - -@containers_router.get("/containers:inspect") -async def containers_inspect(request: Request, response: Response) -> Dict[str, Any]: - """ Returns information about the container, like docker inspect command """ - docker = aiodocker.Docker() - - container_names = request.app.state.shared_store.get_container_names() - container_names = container_names if container_names else {} - - results = {} - - for container in container_names: - try: - container_instance = await docker.containers.get(container) - results[container] = await container_instance.show() - except aiodocker.exceptions.DockerError as e: - response.status_code = 400 - return dict(error=e.message) - - return results - - -@containers_router.get("/containers:docker-status") -async def containers_docker_status( - request: Request, response: Response -) -> Dict[str, Any]: - """ Returns the status of the containers """ - - def assemble_entry(status: str, error: str = "") -> Dict[str, str]: - return {"Status": status, "Error": error} - - docker = aiodocker.Docker() - - shared_store = request.app.state.shared_store - container_names = shared_store.get_container_names() - container_names = container_names if container_names else {} - - # if containers are being pulled, return pulling (fake status) - if shared_store.is_pulling_containsers: - # pulling is a fake state use to share more information with the frontend - return {x: assemble_entry(status="pulling") for x in container_names} - - results = {} - - for container in container_names: - try: - container_instance = await docker.containers.get(container) - container_inspect = await container_instance.show() - container_state = container_inspect.get("State", {}) - - # pending is another fake state use to share more information with the frontend - results[container] = { - "Status": container_state.get("Status", "pending"), - "Error": container_state.get("Error", ""), - } - except aiodocker.exceptions.DockerError as e: - response.status_code = 400 - return dict(error=e.message) - - return results - - -__all__ = ["containers_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/health.py deleted file mode 100644 index f5dad519250..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/health.py +++ /dev/null @@ -1,13 +0,0 @@ -from fastapi import APIRouter, Request - -from ..models import ApplicationHealth - -health_router = APIRouter() - - -@health_router.get("/health", response_model=ApplicationHealth) -async def health_endpoint(request: Request) -> ApplicationHealth: - return request.app.state.application_health - - -__all__ = ["health_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/push.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/push.py deleted file mode 100644 index 02c7328e281..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/push.py +++ /dev/null @@ -1,18 +0,0 @@ -# acts as mock for now - -import logging - -from fastapi import APIRouter - -logger = logging.getLogger(__name__) - -push_router = APIRouter() - - -@push_router.post("/push") -async def post_api() -> str: - logger.warning("TODO: still need to implement") - return "" - - -__all__ = ["push_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/retrive.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/retrive.py deleted file mode 100644 index 9e441ccd6c2..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/retrive.py +++ /dev/null @@ -1,24 +0,0 @@ -# acts as mock for now - -import logging - -from fastapi import APIRouter - -logger = logging.getLogger(__name__) - -retrive_router = APIRouter() - - -@retrive_router.get("/retrive") -async def get_api() -> str: - logger.warning("TODO: still need to implement") - return "" - - -@retrive_router.post("/retrive") -async def post_api() -> str: - logger.warning("TODO: still need to implement") - return "" - - -__all__ = ["retrive_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/state.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/state.py deleted file mode 100644 index 47f110cabff..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/api/state.py +++ /dev/null @@ -1,24 +0,0 @@ -# acts as mock for now - -import logging - -from fastapi import APIRouter - -logger = logging.getLogger(__name__) - -state_router = APIRouter() - - -@state_router.get("/state") -async def get_api() -> str: - logger.warning("TODO: still need to implement") - return "" - - -@state_router.post("/state") -async def post_api() -> str: - logger.warning("TODO: still need to implement") - return "" - - -__all__ = ["state_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/application.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/application.py deleted file mode 100644 index 2e707674b28..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/application.py +++ /dev/null @@ -1,51 +0,0 @@ -import logging - -from fastapi import FastAPI - -from .api import main_router -from .models import ApplicationHealth -from .remote_debug import setup as remote_debug_setup -from .settings import ServiceSidecarSettings -from .shared_handlers import on_shutdown_handler -from .storage import SharedStore - -logger = logging.getLogger(__name__) - - -def assemble_application() -> FastAPI: - """ - Creates the application from using the env vars as a context - Also stores inside the state all instances of classes - needed in other requests and used to share data. - """ - - dynamic_sidecar_settings = ServiceSidecarSettings.create() - - logging.basicConfig(level=dynamic_sidecar_settings.loglevel) - logging.root.setLevel(dynamic_sidecar_settings.loglevel) - logger.debug(dynamic_sidecar_settings.json(indent=2)) - - application = FastAPI(debug=dynamic_sidecar_settings.debug) - - # store "settings" and "shared_store" for later usage - application.state.settings = dynamic_sidecar_settings - application.state.shared_store = SharedStore(settings=dynamic_sidecar_settings) - # used to keep track of the health of the application - # also will be used in the /health endpoint - application.state.application_health = ApplicationHealth() - - # enable debug if required - if dynamic_sidecar_settings.is_development_mode: - remote_debug_setup(application) - - # add routing paths - application.include_router(main_router) - - # setting up handler for lifecycle - async def on_shutdown() -> None: - await on_shutdown_handler(application) - logger.info("shutdown cleanup completed") - - application.add_event_handler("shutdown", on_shutdown) - - return application diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/main.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/main.py deleted file mode 100644 index 7247d675c90..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/main.py +++ /dev/null @@ -1,31 +0,0 @@ -import sys -from pathlib import Path - -import uvicorn -from fastapi import FastAPI -from simcore_service_dynamic_sidecar.application import assemble_application -from simcore_service_dynamic_sidecar.settings import ServiceSidecarSettings - -current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent - - -app: FastAPI = assemble_application() - - -def main(): - settings: ServiceSidecarSettings = app.state.settings - - uvicorn.run( - "simcore_service_dynamic_sidecar.main:app", - host=settings.host, - port=settings.port, - reload=settings.is_development_mode, - reload_dirs=[ - current_dir, - ], - log_level=settings.log_level_name.lower(), - ) - - -if __name__ == "__main__": - main() diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/models.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/models.py deleted file mode 100644 index 7d2080340c9..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/models.py +++ /dev/null @@ -1,7 +0,0 @@ -from pydantic import BaseModel, Field - - -class ApplicationHealth(BaseModel): - is_healthy: bool = Field( - True, description="returns True if the service sis running correctly" - ) diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/remote_debug.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/remote_debug.py deleted file mode 100644 index 75e49be73a9..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/remote_debug.py +++ /dev/null @@ -1,33 +0,0 @@ -""" Setup remote debugger with Python Tools for Visual Studio (PTVSD) - -""" -import logging - -from fastapi import FastAPI - -logger = logging.getLogger(__name__) - - -def setup(app: FastAPI): - settings = app.state.settings - - def on_startup() -> None: - try: - logger.debug("Enabling attach ptvsd ...") - # - # SEE https://github.com/microsoft/ptvsd#enabling-debugging - # - import ptvsd # pylint: disable=import-outside-toplevel - - ptvsd.enable_attach(address=(settings.host, settings.remote_debug_port)) - - except ImportError as err: - raise Exception( - "Cannot enable remote debugging. Please install ptvsd first" - ) from err - - logger.info( - "Remote debugging enabled: listening port %s", settings.remote_debug_port - ) - - app.add_event_handler("startup", on_startup) diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/settings.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/settings.py deleted file mode 100644 index 0b841e3fa68..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/settings.py +++ /dev/null @@ -1,87 +0,0 @@ -import logging -from typing import Optional - -from models_library.basic_types import BootModeEnum, PortInt -from pydantic import BaseSettings, Field, PositiveInt, validator - - -class ServiceSidecarSettings(BaseSettings): - @classmethod - def create(cls, **settings_kwargs) -> "ServiceSidecarSettings": - return cls( - **settings_kwargs, - ) - - boot_mode: Optional[BootModeEnum] = Field( - ..., - description="boot mode helps determine if in development mode or normal operation", - env="SC_BOOT_MODE", - ) - - # LOGGING - log_level_name: str = Field("DEBUG", env="LOG_LEVEL") - - @validator("log_level_name") - @classmethod - def match_logging_level(cls, v) -> str: - try: - getattr(logging, v.upper()) - except AttributeError as err: - raise ValueError(f"{v.upper()} is not a valid level") from err - return v.upper() - - # SERVICE SERVER (see : https://www.uvicorn.org/settings/) - host: str = Field( - "0.0.0.0", # nosec - description="host where to bind the application on which to serve", - ) - port: PortInt = Field( - 8000, description="port where the server will be currently serving" - ) - - compose_namespace: str = Field( - ..., - description=( - "To avoid collisions when scheduling on the same node, this " - "will be compsoed by the project_uuid and node_uuid." - ), - ) - - max_combined_container_name_length: PositiveInt = Field( - 63, description="the container name which will be used as hostname" - ) - - stop_and_remove_timeout: PositiveInt = Field( - 5, - description=( - "When receiving SIGTERM the process has 10 seconds to cleanup its children " - "forcing our children to stop in 5 seconds in all cases" - ), - ) - - debug: bool = Field( - False, - description="If set to True the application will boot into debug mode", - env="DEBUG", - ) - - remote_debug_port: PortInt = Field( - 3000, description="ptsvd remote debugger starting port" - ) - - docker_compose_down_timeout: PositiveInt = Field( - ..., description="used during shutdown when containers swapend will be removed" - ) - - @property - def is_development_mode(self): - """If in development mode this will be True""" - return self.boot_mode == BootModeEnum.DEVELOPMENT - - @property - def loglevel(self) -> int: - return getattr(logging, self.log_level_name) - - class Config: - case_sensitive = False - env_prefix = "DYNAMIC_SIDECAR_" diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/shared_handlers.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/shared_handlers.py deleted file mode 100644 index 2e41c86f165..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/shared_handlers.py +++ /dev/null @@ -1,64 +0,0 @@ -import logging -from typing import Optional, Tuple - -from fastapi import FastAPI - -from .settings import ServiceSidecarSettings -from .storage import SharedStore -from .utils import async_command, write_to_tmp_file - -logger = logging.getLogger(__name__) - - -async def write_file_and_run_command( - settings: ServiceSidecarSettings, - file_content: Optional[str], - command: str, - command_timeout: float, -) -> Tuple[bool, str]: - """ The command which accepts {file_path} as an argument for string formatting """ - - # pylint: disable=not-async-context-manager - async with write_to_tmp_file(file_content) as file_path: - formatted_command = command.format( - file_path=file_path, - project=settings.compose_namespace, - stop_and_remove_timeout=settings.stop_and_remove_timeout, - ) - logger.debug("Will run command\n'%s':\n%s", formatted_command, file_content) - return await async_command(formatted_command, command_timeout) - - -async def remove_the_compose_spec( - shared_store: SharedStore, settings: ServiceSidecarSettings, command_timeout: float -) -> Tuple[bool, str]: - - stored_compose_content = shared_store.get_spec() - if stored_compose_content is None: - return True, "No started spec to remove was found" - - command = ( - "docker-compose -p {project} -f {file_path} " - "down --remove-orphans -t {stop_and_remove_timeout}" - ) - result = await write_file_and_run_command( - settings=settings, - file_content=stored_compose_content, - command=command, - command_timeout=command_timeout, - ) - shared_store.put_spec(None) # removing compose-file spec - return result - - -async def on_shutdown_handler(app: FastAPI) -> None: - logging.info("Going to remove spawned containers") - shared_store: SharedStore = app.state.shared_store - settings: ServiceSidecarSettings = app.state.settings - - result = await remove_the_compose_spec( - shared_store=shared_store, - settings=settings, - command_timeout=settings.docker_compose_down_timeout, - ) - logging.info("Container removal did_succeed=%s\n%s", result[0], result[1]) diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/storage.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/storage.py deleted file mode 100644 index 3ab22024512..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/storage.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import Any, Dict, List, Optional - -from .settings import ServiceSidecarSettings -from .utils import assemble_container_names, validate_compose_spec - - -class SharedStore: - """Define custom storage abstraction for easy future extension""" - - __slots__ = ("_storage", "_settings", "_is_pulling_containsers") - - _K_COMPOSE_SPEC = "compose_spec" - _K_CONTAINER_NAMES = "container_names" - - def __set_as_compose_spec_none(self): - self._storage[self._K_COMPOSE_SPEC] = None - self._storage[self._K_CONTAINER_NAMES] = [] - - def __init__(self, settings: ServiceSidecarSettings): - self._storage: Dict[str, Any] = {} - self._settings: ServiceSidecarSettings = settings - self._is_pulling_containsers: bool = False - self.__set_as_compose_spec_none() - - def put_spec(self, compose_file_content: Optional[str]) -> None: - if compose_file_content is None: - self.__set_as_compose_spec_none() - return - - self._storage[self._K_COMPOSE_SPEC] = validate_compose_spec( - settings=self._settings, compose_file_content=compose_file_content - ) - self._storage[self._K_CONTAINER_NAMES] = assemble_container_names( - self._storage[self._K_COMPOSE_SPEC] - ) - - def get_spec(self) -> Optional[str]: - return self._storage.get(self._K_COMPOSE_SPEC) - - def get_container_names(self) -> List[str]: - return self._storage[self._K_CONTAINER_NAMES] - - @property - def is_pulling_containsers(self) -> bool: - return self._is_pulling_containsers - - def set_is_pulling_containsers(self) -> None: - self._is_pulling_containsers = True - - def unset_is_pulling_containsers(self) -> None: - self._is_pulling_containsers = False diff --git a/services/dynamic-sidecar/src/simcore_service_service_sidecar/utils.py b/services/dynamic-sidecar/src/simcore_service_service_sidecar/utils.py deleted file mode 100644 index 44f0b179fce..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_service_sidecar/utils.py +++ /dev/null @@ -1,282 +0,0 @@ -import asyncio -import json -import logging -import os -import re -import tempfile -import traceback -from pathlib import Path -from typing import Any, Dict, Generator, List, Tuple - -import aiofiles -import yaml -from async_generator import asynccontextmanager -from async_timeout import timeout - -from .settings import ServiceSidecarSettings - -TEMPLATE_SEARCH_PATTERN = r"%%(.*?)%%" - -logger = logging.getLogger(__name__) - - -class InvalidComposeSpec(Exception): - """Exception used to signal incorrect docker-compose configuration file""" - - -@asynccontextmanager -async def write_to_tmp_file(file_contents): - """Disposes of file on exit""" - # pylint: disable=protected-access,stop-iteration-return - file_path = Path("/") / f"tmp/{next(tempfile._get_candidate_names())}" - async with aiofiles.open(file_path, mode="w") as tmp_file: - await tmp_file.write(file_contents) - try: - yield file_path - finally: - await aiofiles.os.remove(file_path) - - -async def async_command(command, command_timeout: float) -> Tuple[bool, str]: - """Returns if the command exited correctly and the stdout of the command """ - proc = await asyncio.create_subprocess_shell( - command, - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.STDOUT, - ) - - # because the Processes returned by create_subprocess_shell it is not possible to - # have a timeout otherwise nor to stream the response from the process. - try: - async with timeout(command_timeout): - stdout, _ = await proc.communicate() - except asyncio.TimeoutError: - message = ( - f"{traceback.format_exc()}\nTimed out after {command_timeout} " - f"seconds while running {command}" - ) - logger.warning(message) - return False, message - - decoded_stdout = stdout.decode() - logger.info("'%s' result:\n%s", command, decoded_stdout) - finished_without_errors = proc.returncode == 0 - - return finished_without_errors, decoded_stdout - - -def _assemble_container_name( - settings: ServiceSidecarSettings, - service_key: str, - user_given_container_name: str, - index: int, -) -> str: - strings_to_use = [ - settings.compose_namespace, - str(index), - user_given_container_name, - service_key, - ] - - container_name = "-".join([x for x in strings_to_use if len(x) > 0])[ - : settings.max_combined_container_name_length - ] - return container_name.replace("_", "-") - - -def _get_forwarded_env_vars(container_key: str) -> List[str]: - """retruns env vars targeted to each container in the compose spec""" - results = [ - # some services expect it, using it as empty - "SIMCORE_NODE_BASEPATH=", - ] - for key in os.environ.keys(): - if key.startswith("FORWARD_ENV_"): - new_entry_key = key.replace("FORWARD_ENV_", "") - - # parsing `VAR={"destination_container": "destination_container", "env_var": "PAYLOAD"}` - new_entry_payload = json.loads(os.environ[key]) - if new_entry_payload["destination_container"] != container_key: - continue - - new_entry_value = new_entry_payload["env_var"] - new_entry = f"{new_entry_key}={new_entry_value}" - results.append(new_entry) - return results - - -def _extract_templated_entries(text: str) -> List[str]: - return re.findall(TEMPLATE_SEARCH_PATTERN, text) - - -def _apply_templating_directives( - stringified_compose_spec: str, - services: Dict[str, Any], - spec_services_to_container_name: Dict[str, str], -) -> str: - """ - Some custom rules are supported for replacing `container_name` - with the following syntax `%%container_name.SERVICE_KEY_NAME%%`, - where `SERVICE_KEY_NAME` targets a container in the compose spec - - If the directive cannot be applied it will just be left untouched - """ - matches = set(_extract_templated_entries(stringified_compose_spec)) - for match in matches: - parts = match.split(".") - - if len(parts) != 2: - continue # templating will be skipped - - target_property = parts[0] - services_key = parts[1] - if target_property != "container_name": - continue # also ignore if the container_name is not the directive to replace - - remapped_service_key = spec_services_to_container_name[services_key] - replace_with = services.get(remapped_service_key, {}).get( - "container_name", None - ) - if replace_with is None: - continue # also skip here if nothing was found - - match_pattern = f"%%{match}%%" - stringified_compose_spec = stringified_compose_spec.replace( - match_pattern, replace_with - ) - - return stringified_compose_spec - - -def _merge_env_vars( - compose_spec_env_vars: List[str], settings_env_vars: List[str] -) -> List[str]: - def _gen_parts_env_vars( - env_vars: List[str], - ) -> Generator[Tuple[str, str], None, None]: - for env_var in env_vars: - key, value = env_var.split("=") - yield key, value - - # pylint: disable=unnecessary-comprehension - dict_spec_env_vars = {k: v for k, v in _gen_parts_env_vars(compose_spec_env_vars)} - dict_settings_env_vars = {k: v for k, v in _gen_parts_env_vars(settings_env_vars)} - - # overwrite spec vars with vars from settings - for key, value in dict_settings_env_vars.items(): - dict_spec_env_vars[key] = value - - # returns a single list of vars - return [f"{k}={v}" for k, v in dict_spec_env_vars.items()] - - -def _inject_backend_networking( - parsed_compose_spec: Dict[str, Any], network_name: str = "__backend__" -) -> None: - """ - Put all containers in the compose spec in the same network. - The `network_name` must only be unique inside the user defined spec; - docker-compose will add some prefix to it. - """ - - networks = parsed_compose_spec.get("networks", {}) - networks[network_name] = None - - for service_content in parsed_compose_spec["services"].values(): - service_networks = service_content.get("networks", []) - service_networks.append(network_name) - service_content["networks"] = service_networks - - parsed_compose_spec["networks"] = networks - - -def validate_compose_spec( - settings: ServiceSidecarSettings, compose_file_content: str -) -> str: - """ - Checks the following: - - proper yaml format - - no "container_name" service property allowed, because it can - spawn 2 cotainers with the same name - """ - - try: - parsed_compose_spec = yaml.safe_load(compose_file_content) - except yaml.YAMLError as e: - raise InvalidComposeSpec( - f"{str(e)}\n{compose_file_content}\nProvided yaml is not valid!" - ) from e - - if parsed_compose_spec is None or not isinstance(parsed_compose_spec, dict): - raise InvalidComposeSpec(f"{compose_file_content}\nProvided yaml is not valid!") - - if not {"version", "services"}.issubset(set(parsed_compose_spec.keys())): - raise InvalidComposeSpec(f"{compose_file_content}\nProvided yaml is not valid!") - - version = parsed_compose_spec["version"] - if version.startswith("1"): - raise InvalidComposeSpec(f"Provided spec version '{version}' is not supported") - - spec_services_to_container_name: Dict[str, str] = {} - - spec_services = parsed_compose_spec["services"] - for index, service in enumerate(spec_services): - service_content = spec_services[service] - - # assemble and inject the container name - user_given_container_name = service_content.get("container_name", "") - container_name = _assemble_container_name( - settings, service, user_given_container_name, index - ) - service_content["container_name"] = container_name - spec_services_to_container_name[service] = container_name - - # inject forwarded environment variables - environment_entries = service_content.get("environment", []) - service_settings_env_vars = _get_forwarded_env_vars(service) - service_content["environment"] = _merge_env_vars( - compose_spec_env_vars=environment_entries, - settings_env_vars=service_settings_env_vars, - ) - - # if more then one container is defined, add an "backend" network - if len(spec_services) > 1: - _inject_backend_networking(parsed_compose_spec) - - # replace service_key with the container_name int the dict - for service_key in list(spec_services.keys()): - container_name_service_key = spec_services_to_container_name[service_key] - service_data = spec_services.pop(service_key) - - depends_on = service_data.get("depends_on", None) - if depends_on is not None: - service_data["depends_on"] = [ - # replaces with the container name - # if not found it leaves the old value - spec_services_to_container_name.get(x, x) - for x in depends_on - ] - - spec_services[container_name_service_key] = service_data - # TODO: replace names in depends_on keys - - # transform back to string and return - validated_compose_file_content = yaml.safe_dump(parsed_compose_spec) - - compose_spec = _apply_templating_directives( - stringified_compose_spec=validated_compose_file_content, - services=spec_services, - spec_services_to_container_name=spec_services_to_container_name, - ) - - return compose_spec - - -def assemble_container_names(validated_compose_content: str) -> List[str]: - """returns the list of container names from a validated compose_spec""" - parsed_compose_spec = yaml.safe_load(validated_compose_content) - return [ - service_data["container_name"] - for service_data in parsed_compose_spec["services"].values() - ] From 1288c9bbeea02068dcac4eba01b85b2d6e3c733b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:24:30 +0200 Subject: [PATCH 010/102] updating makefile --- services/dynamic-sidecar/Makefile | 23 +++++-------------- services/dynamic-sidecar/requirements/dev.txt | 1 + 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/services/dynamic-sidecar/Makefile b/services/dynamic-sidecar/Makefile index ff3d276e928..993705b7b69 100644 --- a/services/dynamic-sidecar/Makefile +++ b/services/dynamic-sidecar/Makefile @@ -10,13 +10,6 @@ APP_NAME := $(notdir $(CURDIR)) _ensure-in-venv: @python3 -c "import os; os.environ['VIRTUAL_ENV']" || (echo "\n>>>> You are not in a virtualenv. Activate one <<<<\n"; exit 1) -.PHONY: install-dev-dependencies -install-dev-dependencies: _ensure-in-venv ## install depenencies for development - @pip install -r requirements/_base.txt - -.PHONY: install-dev-dependencies -install-dev-dependencies: _ensure-in-venv ## install depenencies for development - @pip install -r requirements/_base.txt .PHONY: dev-run dev-run: _ensure-in-venv ## starts the container on its own @@ -25,18 +18,14 @@ dev-run: _ensure-in-venv ## starts the container on its own -v $(CURDIR):/devel/services/dynamic-sidecar \ local/dynamic-sidecar:development -.PHONY: ci-tox-codestyle -ci-tox-codestyle: ## runs codestyle checks - @tox -r -e codestyle-ci -.PHONY: ci-tox-tests -ci-tox-tests: ## runs tests with coverage in a new enviornment - @tox -r -e py36,report +.PHONY: codestyle +codestyle: ## enforces codestyle and runs pylint and mypy + isort setup.py src/simcore_service_dynamic_sidecar tests + black src/simcore_service_dynamic_sidecar tests/ + pylint --rcfile=../../.pylintrc src/simcore_service_dynamic_sidecar tests/ + mypy src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports -.PHONY: ci-install-requirements -ci-install-requirements: ## runs tests with coverage in a new enviornment - @pip install -r requirements/dev.txt - @pip install tox .PHONY: run-github-action-locally run-github-action-locally: ## runs the defined github action from the workflow locally diff --git a/services/dynamic-sidecar/requirements/dev.txt b/services/dynamic-sidecar/requirements/dev.txt index 1d7c29d2587..b841d0b9093 100644 --- a/services/dynamic-sidecar/requirements/dev.txt +++ b/services/dynamic-sidecar/requirements/dev.txt @@ -9,6 +9,7 @@ # installs base + tests requirements -r _base.txt -r _test.txt +-r _tools.txt # installs this repo's packages -e ../../packages/models-library From ad15255376ef819f607f60f9677f691355bd314b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:24:54 +0200 Subject: [PATCH 011/102] updating tox --- services/dynamic-sidecar/tox.ini | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/services/dynamic-sidecar/tox.ini b/services/dynamic-sidecar/tox.ini index 973c61ba2a1..c2e52d89373 100644 --- a/services/dynamic-sidecar/tox.ini +++ b/services/dynamic-sidecar/tox.ini @@ -16,19 +16,6 @@ depends = {py36}: clean report: py36 -# will fail if any of isort, black, pylint or mypy fail -[testenv:codestyle-ci] -basepython = python3.6 -deps = - -r requirements/_tools.txt -commands = - isort --check setup.py src/simcore_service_dynamic_sidecar tests - black --check src/simcore_service_dynamic_sidecar tests/ - pylint --rcfile=../../.pylintrc src/simcore_service_dynamic_sidecar tests/ - mypy src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports - - -# used for development [testenv:codestyle] basepython = python3.6 deps = From fd8e1203f929a81e305f7f7b20583119027f96b8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:27:09 +0200 Subject: [PATCH 012/102] removing tox --- services/dynamic-sidecar/tox.ini | 44 -------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 services/dynamic-sidecar/tox.ini diff --git a/services/dynamic-sidecar/tox.ini b/services/dynamic-sidecar/tox.ini deleted file mode 100644 index c2e52d89373..00000000000 --- a/services/dynamic-sidecar/tox.ini +++ /dev/null @@ -1,44 +0,0 @@ -# content of: tox.ini , put in same dir as setup.py -[tox] -isolated_build = True -envlist = clean, codestyle, py36, report -skipsdist=True - - -[testenv] -# install pytest in the virtualenv where commands will be executed -deps = - -r requirements/ci.txt -commands = - pytest -vvv -s --cov=simcore_service_dynamic_sidecar --cov-append --cov-report=term-missing --capture=sys tests/unit - -depends = - {py36}: clean - report: py36 - -[testenv:codestyle] -basepython = python3.6 -deps = - -r requirements/_tools.txt -commands = - isort setup.py src/simcore_service_dynamic_sidecar tests - black src/simcore_service_dynamic_sidecar tests/ - pylint --rcfile=../../.pylintrc src/simcore_service_dynamic_sidecar tests/ - mypy src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports - -[testenv:report] -basepython = python3.6 -deps = - coverage -skip_install = true -exclude_lines = - pragma: no cover -commands = - coverage report - # TODO: upload the test report somewere - -[testenv:clean] -basepython = python3.6 -deps = coverage -skip_install = true -commands = coverage erase From 45e43e35bd1d64bc1de05b50664ddf9921275128 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:33:17 +0200 Subject: [PATCH 013/102] added ci codestyle check before tests run --- .github/workflows/ci-testing-deploy.yml | 2 ++ ci/github/unit-testing/dynamic-sidecar.bash | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 1545cbc1444..ec1ef11307a 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -344,6 +344,8 @@ jobs: ${{ runner.os }}- - name: install run: ./ci/github/unit-testing/dynamic-sidecar.bash install + - name: codestyle-ci + run: ./ci/github/unit-testing/dynamic-sidecar.bash codestyle - name: test run: ./ci/github/unit-testing/dynamic-sidecar.bash test - uses: codecov/codecov-action@v1 diff --git a/ci/github/unit-testing/dynamic-sidecar.bash b/ci/github/unit-testing/dynamic-sidecar.bash index eeef5e2386d..21817ec5847 100755 --- a/ci/github/unit-testing/dynamic-sidecar.bash +++ b/ci/github/unit-testing/dynamic-sidecar.bash @@ -7,10 +7,23 @@ IFS=$'\n\t' install() { bash ci/helpers/ensure_python_pip.bash; - pushd services/dynamic-sidecar; pip3 install -r requirements/ci.txt; popd; + pushd services/dynamic-sidecar; pip3 install -r requirements/ci.txt -r requirements/_tools.txt; popd; pip list -v } +codestyle(){ + pushd services/dynamic-sidecar + echo "isort" + isort --check setup.py src/simcore_service_dynamic_sidecar tests + echo "black" + black --check src/simcore_service_dynamic_sidecar tests/ + echo "pylint" + pylint --rcfile=../../.pylintrc src/simcore_service_dynamic_sidecar tests/ + echo "mypy" + mypy src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports + popd +} + test() { pytest --cov=simcore_service_dynamic_sidecar --durations=10 --cov-append \ --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \ From 7160fe0a0cfcfacd8ea30901a7f6ca8f10c8d9b5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:36:23 +0200 Subject: [PATCH 014/102] removed unsued targets --- services/dynamic-sidecar/Makefile | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/services/dynamic-sidecar/Makefile b/services/dynamic-sidecar/Makefile index 993705b7b69..165acda1a36 100644 --- a/services/dynamic-sidecar/Makefile +++ b/services/dynamic-sidecar/Makefile @@ -6,19 +6,6 @@ APP_NAME := $(notdir $(CURDIR)) .DEFAULT_GOAL := help -.PHONY: _ensure-in-venv -_ensure-in-venv: - @python3 -c "import os; os.environ['VIRTUAL_ENV']" || (echo "\n>>>> You are not in a virtualenv. Activate one <<<<\n"; exit 1) - - -.PHONY: dev-run -dev-run: _ensure-in-venv ## starts the container on its own - @docker run -it --rm \ - -p 8000:8000 \ - -v $(CURDIR):/devel/services/dynamic-sidecar \ - local/dynamic-sidecar:development - - .PHONY: codestyle codestyle: ## enforces codestyle and runs pylint and mypy isort setup.py src/simcore_service_dynamic_sidecar tests From 453342afeff694d7e4aca99819d9cbc77e8050f7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:48:29 +0200 Subject: [PATCH 015/102] fixed error message --- .../src/simcore_service_dynamic_sidecar/api/compose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index 2c8dd4becb9..f8e166a5f9d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -98,7 +98,7 @@ async def pull_docker_required_docker_images( stored_compose_content = shared_store.get_spec() if stored_compose_content is None: response.status_code = 400 - return "No started spec to stop was found" + return "No started spec to pull was found" command = "docker-compose -p {project} -f {file_path} pull --include-deps" From a293a1c21337aa1bba09a36a6fd55f4386c156e3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:48:34 +0200 Subject: [PATCH 016/102] added missing test --- services/dynamic-sidecar/tests/unit/test_api_compose.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/services/dynamic-sidecar/tests/unit/test_api_compose.py b/services/dynamic-sidecar/tests/unit/test_api_compose.py index 6a8fa0d80f3..26cb11d458e 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_compose.py +++ b/services/dynamic-sidecar/tests/unit/test_api_compose.py @@ -99,6 +99,15 @@ async def test_pull_missing_spec(test_client: TestClient, compose_spec: Dict[str "/compose:pull", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) ) assert response.status_code == 400, response.text + assert response.text == "No started spec to pull was found" + + +@pytest.mark.asyncio +async def test_stop_missing_spec(test_client: TestClient, compose_spec: Dict[str, Any]): + response = await test_client.put( + "/compose:stop", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + ) + assert response.status_code == 400, response.text assert response.text == "No started spec to stop was found" From 77f550c02ef80c05d37ecbc79d4be6464fba8859 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:51:41 +0200 Subject: [PATCH 017/102] added codeoowner entry --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2ae84c374f3..68b6d88bde0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -22,6 +22,7 @@ Makefile @pcrespov, @sanderegg /scripts/template-projects/ @odeimaiz, @pcrespov /services/api-server/ @pcrespov /services/director*/ @sanderegg, @pcrespov +/services/dynamic-sidecar/ @GitHK /services/catalog/ @pcrespov, @sanderegg /services/migration/ @pcrespov /services/sidecar/ @pcrespov, @mguidon From b883ea57bcf095ca84b7a48c64c5ca6e48a2cfbc Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:58:31 +0200 Subject: [PATCH 018/102] moved mocked fucntions to the same module --- .../api/_routing.py | 8 +--- .../api/mocked.py | 46 +++++++++++++++++++ .../api/push.py | 18 -------- .../api/retrive.py | 24 ---------- .../api/state.py | 24 ---------- 5 files changed, 48 insertions(+), 72 deletions(-) create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/push.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/retrive.py delete mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/state.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py index 225504128e9..09838e1947e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py @@ -6,9 +6,7 @@ from .container import container_router from .containers import containers_router from .health import health_router -from .push import push_router -from .retrive import retrive_router -from .state import state_router +from .mocked import mocked_router # setup and register all routes here form different modules main_router = APIRouter() @@ -16,8 +14,6 @@ main_router.include_router(compose_router) main_router.include_router(containers_router) main_router.include_router(container_router) -main_router.include_router(state_router) -main_router.include_router(retrive_router) -main_router.include_router(push_router) +main_router.include_router(mocked_router) __all__ = ["main_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py new file mode 100644 index 00000000000..338f6b355f7 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py @@ -0,0 +1,46 @@ +""" +All the functions is this module are mocked out +because they are called by the frontend. +Avoids raising errors in the service. +""" + +import logging + +from fastapi import APIRouter + +logger = logging.getLogger(__name__) + +mocked_router = APIRouter() + + +@mocked_router.post("/push") +async def post_push() -> str: + logger.warning("TODO: still need to implement") + return "" + + +@mocked_router.get("/retrive") +async def get_retrive() -> str: + logger.warning("TODO: still need to implement") + return "" + + +@mocked_router.post("/retrive") +async def post_retrive() -> str: + logger.warning("TODO: still need to implement") + return "" + + +@mocked_router.get("/state") +async def get_state() -> str: + logger.warning("TODO: still need to implement") + return "" + + +@mocked_router.post("/state") +async def post_state() -> str: + logger.warning("TODO: still need to implement") + return "" + + +__all__ = ["mocked_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/push.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/push.py deleted file mode 100644 index 02c7328e281..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/push.py +++ /dev/null @@ -1,18 +0,0 @@ -# acts as mock for now - -import logging - -from fastapi import APIRouter - -logger = logging.getLogger(__name__) - -push_router = APIRouter() - - -@push_router.post("/push") -async def post_api() -> str: - logger.warning("TODO: still need to implement") - return "" - - -__all__ = ["push_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/retrive.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/retrive.py deleted file mode 100644 index 9e441ccd6c2..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/retrive.py +++ /dev/null @@ -1,24 +0,0 @@ -# acts as mock for now - -import logging - -from fastapi import APIRouter - -logger = logging.getLogger(__name__) - -retrive_router = APIRouter() - - -@retrive_router.get("/retrive") -async def get_api() -> str: - logger.warning("TODO: still need to implement") - return "" - - -@retrive_router.post("/retrive") -async def post_api() -> str: - logger.warning("TODO: still need to implement") - return "" - - -__all__ = ["retrive_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/state.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/state.py deleted file mode 100644 index 47f110cabff..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/state.py +++ /dev/null @@ -1,24 +0,0 @@ -# acts as mock for now - -import logging - -from fastapi import APIRouter - -logger = logging.getLogger(__name__) - -state_router = APIRouter() - - -@state_router.get("/state") -async def get_api() -> str: - logger.warning("TODO: still need to implement") - return "" - - -@state_router.post("/state") -async def post_api() -> str: - logger.warning("TODO: still need to implement") - return "" - - -__all__ = ["state_router"] From 27546d785ee6755b4dda438805577c124ce8a678 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 10:58:50 +0200 Subject: [PATCH 019/102] renamed module --- .../unit/{test_api_push_retrive_state.py => test_api_mocked.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename services/dynamic-sidecar/tests/unit/{test_api_push_retrive_state.py => test_api_mocked.py} (100%) diff --git a/services/dynamic-sidecar/tests/unit/test_api_push_retrive_state.py b/services/dynamic-sidecar/tests/unit/test_api_mocked.py similarity index 100% rename from services/dynamic-sidecar/tests/unit/test_api_push_retrive_state.py rename to services/dynamic-sidecar/tests/unit/test_api_mocked.py From 31b60eabcd74c5c689c4e88481eb12a61d55d5ba Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 11:00:54 +0200 Subject: [PATCH 020/102] updating log warning messages for mocked calls --- .../src/simcore_service_dynamic_sidecar/api/mocked.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py index 338f6b355f7..8b336c6990e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py @@ -15,31 +15,31 @@ @mocked_router.post("/push") async def post_push() -> str: - logger.warning("TODO: still need to implement") + logger.warning("ignoring call POST /push from frontend") return "" @mocked_router.get("/retrive") async def get_retrive() -> str: - logger.warning("TODO: still need to implement") + logger.warning("ignoring call GET /retrive from frontend") return "" @mocked_router.post("/retrive") async def post_retrive() -> str: - logger.warning("TODO: still need to implement") + logger.warning("ignoring call POST /retrive from frontend") return "" @mocked_router.get("/state") async def get_state() -> str: - logger.warning("TODO: still need to implement") + logger.warning("ignoring call GET /state from frontend") return "" @mocked_router.post("/state") async def post_state() -> str: - logger.warning("TODO: still need to implement") + logger.warning("ignoring call POST /state from frontend") return "" From 4ede09d71f2fe29e47627121ba13d5d1b2562a6e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 12:01:07 +0200 Subject: [PATCH 021/102] added some help --- services/dynamic-sidecar/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/Makefile b/services/dynamic-sidecar/Makefile index 165acda1a36..1a93a707b14 100644 --- a/services/dynamic-sidecar/Makefile +++ b/services/dynamic-sidecar/Makefile @@ -15,6 +15,6 @@ codestyle: ## enforces codestyle and runs pylint and mypy .PHONY: run-github-action-locally -run-github-action-locally: ## runs the defined github action from the workflow locally +run-github-action-locally: ## runs "unit-test-dynamic-sidecar" defined int github workflow locally # Note: ⚡ act is required https://github.com/nektos/act @act -C ../../. -P ubuntu-20.04=catthehacker/ubuntu:act-20.04 -j unit-test-dynamic-sidecar From e18ef583ec7823392843a69c0ed09f4dd25f50e4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 12:01:18 +0200 Subject: [PATCH 022/102] clrifying command usage --- services/dynamic-sidecar/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/Makefile b/services/dynamic-sidecar/Makefile index 1a93a707b14..3c42d266e68 100644 --- a/services/dynamic-sidecar/Makefile +++ b/services/dynamic-sidecar/Makefile @@ -16,5 +16,5 @@ codestyle: ## enforces codestyle and runs pylint and mypy .PHONY: run-github-action-locally run-github-action-locally: ## runs "unit-test-dynamic-sidecar" defined int github workflow locally - # Note: ⚡ act is required https://github.com/nektos/act + # Note: ⚡ act is required https://github.com/nektos/act to emulate the github actons locally @act -C ../../. -P ubuntu-20.04=catthehacker/ubuntu:act-20.04 -j unit-test-dynamic-sidecar From 4ec614792929ebc8bf8eaf6af40676ae598f9b57 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 12:01:25 +0200 Subject: [PATCH 023/102] using verbatim arguments --- .../api/compose.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index f8e166a5f9d..5fc561c1200 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -33,7 +33,6 @@ async def store_docker_compose_spec_for_later_usage( return str(e) response.status_code = 204 - return None @compose_router.post("/compose:preload", response_class=PlainTextResponse) @@ -54,7 +53,10 @@ async def create_docker_compose_configuration_containers_without_starting( return str(e) # --no-build might be a security risk building is disabled - command = "docker-compose -p {project} -f {file_path} up --no-build --no-start" + command = ( + "docker-compose --project-name {project} --file {file_path} " + "up --no-build --no-start" + ) finished_without_errors, stdout = await write_file_and_run_command( settings=settings, file_content=shared_store.get_spec(), @@ -75,7 +77,10 @@ async def start_or_update_docker_compose_configuration( shared_store: SharedStore = request.app.state.shared_store # --no-build might be a security risk building is disabled - command = "docker-compose -p {project} -f {file_path} up --no-build -d" + command = ( + "docker-compose --project-name {project} --file {file_path} " + "up --no-build --detach" + ) finished_without_errors, stdout = await write_file_and_run_command( settings=settings, file_content=shared_store.get_spec(), @@ -100,7 +105,10 @@ async def pull_docker_required_docker_images( response.status_code = 400 return "No started spec to pull was found" - command = "docker-compose -p {project} -f {file_path} pull --include-deps" + command = ( + "docker-compose --project-name {project} --file {file_path} " + "pull --include-deps" + ) try: # mark as pulling images @@ -135,7 +143,8 @@ async def stop_containers_without_removing_them( return "No started spec to stop was found" command = ( - "docker-compose -p {project} -f {file_path} stop -t {stop_and_remove_timeout}" + "docker-compose --project-name {project} --file {file_path} " + "stop --timeout {stop_and_remove_timeout}" ) finished_without_errors, stdout = await write_file_and_run_command( settings=settings, From 08da57c4a74e90b273fe2d808e7f6a92bd15037b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 12:05:12 +0200 Subject: [PATCH 024/102] fixed typo --- .../src/simcore_service_dynamic_sidecar/api/mocked.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py index 8b336c6990e..b09957c5802 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py @@ -19,13 +19,13 @@ async def post_push() -> str: return "" -@mocked_router.get("/retrive") +@mocked_router.get("/retrieve") async def get_retrive() -> str: logger.warning("ignoring call GET /retrive from frontend") return "" -@mocked_router.post("/retrive") +@mocked_router.post("/retrieve") async def post_retrive() -> str: logger.warning("ignoring call POST /retrive from frontend") return "" From 042994958061e42e67b4ca08bf662096255dae01 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 12:05:41 +0200 Subject: [PATCH 025/102] renived tests --- services/dynamic-sidecar/tests/unit/test_api_mocked.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_mocked.py b/services/dynamic-sidecar/tests/unit/test_api_mocked.py index 5d1444686a1..e52ad9b73fe 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_mocked.py +++ b/services/dynamic-sidecar/tests/unit/test_api_mocked.py @@ -22,8 +22,8 @@ def assert_200_empty(response: Response) -> bool: # push api module ("/push", "POST"), # retrive api module - ("/retrive", "GET"), - ("/retrive", "POST"), + ("/retrieve", "GET"), + ("/retrieve", "POST"), # state api module ("/state", "GET"), ("/state", "POST"), From 570067d6319b535666f930872cc8d4709b076e3e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 12:09:13 +0200 Subject: [PATCH 026/102] renaming ServiceSidecarSettings to DynamicSidecarSettings --- .../src/simcore_service_dynamic_sidecar/api/compose.py | 10 +++++----- .../src/simcore_service_dynamic_sidecar/application.py | 4 ++-- .../src/simcore_service_dynamic_sidecar/main.py | 4 ++-- .../src/simcore_service_dynamic_sidecar/settings.py | 4 ++-- .../simcore_service_dynamic_sidecar/shared_handlers.py | 8 ++++---- .../src/simcore_service_dynamic_sidecar/storage.py | 6 +++--- .../src/simcore_service_dynamic_sidecar/utils.py | 6 +++--- services/dynamic-sidecar/tests/conftest.py | 4 ++-- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index 5fc561c1200..e0d620c4836 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -5,7 +5,7 @@ from fastapi import APIRouter, Request, Response from fastapi.responses import PlainTextResponse -from ..settings import ServiceSidecarSettings +from ..settings import DynamicSidecarSettings from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command from ..storage import SharedStore from ..utils import InvalidComposeSpec @@ -42,7 +42,7 @@ async def create_docker_compose_configuration_containers_without_starting( """ Expects the docker-compose spec as raw-body utf-8 encoded text """ body_as_text = (await request.body()).decode("utf-8") - settings: ServiceSidecarSettings = request.app.state.settings + settings: DynamicSidecarSettings = request.app.state.settings shared_store: SharedStore = request.app.state.shared_store try: @@ -73,7 +73,7 @@ async def start_or_update_docker_compose_configuration( request: Request, response: Response, command_timeout: float ) -> str: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - settings: ServiceSidecarSettings = request.app.state.settings + settings: DynamicSidecarSettings = request.app.state.settings shared_store: SharedStore = request.app.state.shared_store # --no-build might be a security risk building is disabled @@ -98,7 +98,7 @@ async def pull_docker_required_docker_images( ) -> str: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ shared_store: SharedStore = request.app.state.shared_store - settings: ServiceSidecarSettings = request.app.state.settings + settings: DynamicSidecarSettings = request.app.state.settings stored_compose_content = shared_store.get_spec() if stored_compose_content is None: @@ -135,7 +135,7 @@ async def stop_containers_without_removing_them( """Stops the previously started service and returns the docker-compose output""" shared_store: SharedStore = request.app.state.shared_store - settings: ServiceSidecarSettings = request.app.state.settings + settings: DynamicSidecarSettings = request.app.state.settings stored_compose_content = shared_store.get_spec() if stored_compose_content is None: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py index 2e707674b28..c7e34aba10d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py @@ -5,7 +5,7 @@ from .api import main_router from .models import ApplicationHealth from .remote_debug import setup as remote_debug_setup -from .settings import ServiceSidecarSettings +from .settings import DynamicSidecarSettings from .shared_handlers import on_shutdown_handler from .storage import SharedStore @@ -19,7 +19,7 @@ def assemble_application() -> FastAPI: needed in other requests and used to share data. """ - dynamic_sidecar_settings = ServiceSidecarSettings.create() + dynamic_sidecar_settings = DynamicSidecarSettings.create() logging.basicConfig(level=dynamic_sidecar_settings.loglevel) logging.root.setLevel(dynamic_sidecar_settings.loglevel) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py index 7247d675c90..6e7c19fb647 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py @@ -4,7 +4,7 @@ import uvicorn from fastapi import FastAPI from simcore_service_dynamic_sidecar.application import assemble_application -from simcore_service_dynamic_sidecar.settings import ServiceSidecarSettings +from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -13,7 +13,7 @@ def main(): - settings: ServiceSidecarSettings = app.state.settings + settings: DynamicSidecarSettings = app.state.settings uvicorn.run( "simcore_service_dynamic_sidecar.main:app", diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py index 0b841e3fa68..4a252432d4c 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py @@ -5,9 +5,9 @@ from pydantic import BaseSettings, Field, PositiveInt, validator -class ServiceSidecarSettings(BaseSettings): +class DynamicSidecarSettings(BaseSettings): @classmethod - def create(cls, **settings_kwargs) -> "ServiceSidecarSettings": + def create(cls, **settings_kwargs) -> "DynamicSidecarSettings": return cls( **settings_kwargs, ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py index 2e41c86f165..0aa54065687 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py @@ -3,7 +3,7 @@ from fastapi import FastAPI -from .settings import ServiceSidecarSettings +from .settings import DynamicSidecarSettings from .storage import SharedStore from .utils import async_command, write_to_tmp_file @@ -11,7 +11,7 @@ async def write_file_and_run_command( - settings: ServiceSidecarSettings, + settings: DynamicSidecarSettings, file_content: Optional[str], command: str, command_timeout: float, @@ -30,7 +30,7 @@ async def write_file_and_run_command( async def remove_the_compose_spec( - shared_store: SharedStore, settings: ServiceSidecarSettings, command_timeout: float + shared_store: SharedStore, settings: DynamicSidecarSettings, command_timeout: float ) -> Tuple[bool, str]: stored_compose_content = shared_store.get_spec() @@ -54,7 +54,7 @@ async def remove_the_compose_spec( async def on_shutdown_handler(app: FastAPI) -> None: logging.info("Going to remove spawned containers") shared_store: SharedStore = app.state.shared_store - settings: ServiceSidecarSettings = app.state.settings + settings: DynamicSidecarSettings = app.state.settings result = await remove_the_compose_spec( shared_store=shared_store, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/storage.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/storage.py index 3ab22024512..2119f88ed50 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/storage.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/storage.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional -from .settings import ServiceSidecarSettings +from .settings import DynamicSidecarSettings from .utils import assemble_container_names, validate_compose_spec @@ -16,9 +16,9 @@ def __set_as_compose_spec_none(self): self._storage[self._K_COMPOSE_SPEC] = None self._storage[self._K_CONTAINER_NAMES] = [] - def __init__(self, settings: ServiceSidecarSettings): + def __init__(self, settings: DynamicSidecarSettings): self._storage: Dict[str, Any] = {} - self._settings: ServiceSidecarSettings = settings + self._settings: DynamicSidecarSettings = settings self._is_pulling_containsers: bool = False self.__set_as_compose_spec_none() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py index 44f0b179fce..966110ccc65 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py @@ -13,7 +13,7 @@ from async_generator import asynccontextmanager from async_timeout import timeout -from .settings import ServiceSidecarSettings +from .settings import DynamicSidecarSettings TEMPLATE_SEARCH_PATTERN = r"%%(.*?)%%" @@ -67,7 +67,7 @@ async def async_command(command, command_timeout: float) -> Tuple[bool, str]: def _assemble_container_name( - settings: ServiceSidecarSettings, + settings: DynamicSidecarSettings, service_key: str, user_given_container_name: str, index: int, @@ -192,7 +192,7 @@ def _inject_backend_networking( def validate_compose_spec( - settings: ServiceSidecarSettings, compose_file_content: str + settings: DynamicSidecarSettings, compose_file_content: str ) -> str: """ Checks the following: diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index a54e1873e94..772b4a37da5 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -12,7 +12,7 @@ from async_asgi_testclient import TestClient from fastapi import FastAPI from simcore_service_dynamic_sidecar.application import assemble_application -from simcore_service_dynamic_sidecar.settings import ServiceSidecarSettings +from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command from simcore_service_dynamic_sidecar.storage import SharedStore @@ -48,7 +48,7 @@ async def cleanup_containers(app: FastAPI) -> AsyncGenerator[None, None]: # if no compose-spec is stored skip this operation return - settings: ServiceSidecarSettings = app.state.settings + settings: DynamicSidecarSettings = app.state.settings command = ( "docker-compose -p {project} -f {file_path} " "down --remove-orphans -t {stop_and_remove_timeout}" From 3ace27792c26a6d7527408956edc83b55a4620a8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 13:39:42 +0200 Subject: [PATCH 027/102] updated name in changelog --- services/dynamic-sidecar/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/CHANGELOG.md b/services/dynamic-sidecar/CHANGELOG.md index 692f1de6841..c71ff9b100b 100644 --- a/services/dynamic-sidecar/CHANGELOG.md +++ b/services/dynamic-sidecar/CHANGELOG.md @@ -1,5 +1,5 @@ # Changelog -All notable changes to this project will be documented in this file. +All notable changes to this service will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). From fa0641f8231112c6f1451f47b0eadf52681fd83b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 13:41:45 +0200 Subject: [PATCH 028/102] fixed ptsv entrypoint --- services/dynamic-sidecar/docker/boot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/docker/boot.sh b/services/dynamic-sidecar/docker/boot.sh index c53a8500c02..cdc4b875ac1 100755 --- a/services/dynamic-sidecar/docker/boot.sh +++ b/services/dynamic-sidecar/docker/boot.sh @@ -30,7 +30,7 @@ if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ] then # NOTE: ptvsd is programmatically enabled inside of the service # this way we can have reload in place as well - exec uvicorn sidecar.app:app --reload --host 0.0.0.0 + exec uvicorn simcore_service_dynamic_sidecar.main:app --reload --host 0.0.0.0 else exec simcore_service_dynamic_sidecar_startup fi From a7ea456b6fe51a91fbe390e5d70b75b953b50bdf Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 13:50:22 +0200 Subject: [PATCH 029/102] fixed healt endpoint to fail when status is not healthy --- services/dynamic-sidecar/Dockerfile | 2 +- .../src/simcore_service_dynamic_sidecar/api/health.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/services/dynamic-sidecar/Dockerfile b/services/dynamic-sidecar/Dockerfile index b97a5877bef..d6929f4d862 100644 --- a/services/dynamic-sidecar/Dockerfile +++ b/services/dynamic-sidecar/Dockerfile @@ -116,7 +116,7 @@ HEALTHCHECK --interval=30s \ --timeout=20s \ --start-period=30s \ --retries=3 \ - CMD ["python3", "services/dynamic-sidecar/docker/healthcheck.py", "http://localhost:8000/"] + CMD ["python3", "services/dynamic-sidecar/docker/healthcheck.py", "http://localhost:8000/health"] EXPOSE 8000 diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py index f5dad519250..858dcd18e4e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py @@ -1,4 +1,4 @@ -from fastapi import APIRouter, Request +from fastapi import APIRouter, Request, Response from ..models import ApplicationHealth @@ -6,8 +6,13 @@ @health_router.get("/health", response_model=ApplicationHealth) -async def health_endpoint(request: Request) -> ApplicationHealth: - return request.app.state.application_health +async def health_endpoint(request: Request, response: Response) -> ApplicationHealth: + application_health: ApplicationHealth = request.app.state.application_health + + if application_health.is_healty is False: + response.status_code = 400 + + return application_health __all__ = ["health_router"] From 7271897c97a513f5efcecaf79add4142907508dd Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 13:57:23 +0200 Subject: [PATCH 030/102] fixed route and test --- .../src/simcore_service_dynamic_sidecar/api/health.py | 2 +- services/dynamic-sidecar/tests/unit/test_api_health.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py index 858dcd18e4e..6f804456411 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py @@ -9,7 +9,7 @@ async def health_endpoint(request: Request, response: Response) -> ApplicationHealth: application_health: ApplicationHealth = request.app.state.application_health - if application_health.is_healty is False: + if application_health.is_healthy is False: response.status_code = 400 return application_health diff --git a/services/dynamic-sidecar/tests/unit/test_api_health.py b/services/dynamic-sidecar/tests/unit/test_api_health.py index 6aa860322e2..f5a890706e1 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_health.py +++ b/services/dynamic-sidecar/tests/unit/test_api_health.py @@ -4,7 +4,9 @@ @pytest.mark.asyncio -async def test_is_healthy(test_client: TestClient): +@pytest.mark.parametrize("is_healthy,status_code", [(True, 200), (False, 400)]) +async def test_is_healthy(test_client: TestClient, is_healthy: bool, status_code: int): + test_client.application.state.application_health.is_healthy = is_healthy response = await test_client.get("/health") - assert response.status_code == 200, response - assert response.json() == ApplicationHealth().dict() + assert response.status_code == status_code, response + assert response.json() == ApplicationHealth(is_healthy=is_healthy).dict() From 6db2beb5a8be85053f34ece54d40d29b92169e6f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 14:11:28 +0200 Subject: [PATCH 031/102] mappign docs on the same route as other services --- .../src/simcore_service_dynamic_sidecar/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py index c7e34aba10d..41060b7981f 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py @@ -25,7 +25,7 @@ def assemble_application() -> FastAPI: logging.root.setLevel(dynamic_sidecar_settings.loglevel) logger.debug(dynamic_sidecar_settings.json(indent=2)) - application = FastAPI(debug=dynamic_sidecar_settings.debug) + application = FastAPI(debug=dynamic_sidecar_settings.debug, docs_url="/dev/doc") # store "settings" and "shared_store" for later usage application.state.settings = dynamic_sidecar_settings From 98fd9dab4cf3da7fb09d7261385100184d2070c1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 14:48:15 +0200 Subject: [PATCH 032/102] added api prefix for interested routes --- .../simcore_service_dynamic_sidecar/_meta.py | 1 + .../api/_routing.py | 7 +-- .../application.py | 7 ++- .../tests/unit/test_api_compose.py | 50 ++++++++++++------- .../tests/unit/test_api_container.py | 26 +++++----- .../tests/unit/test_api_containers.py | 19 ++++--- 6 files changed, 68 insertions(+), 42 deletions(-) create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py new file mode 100644 index 00000000000..bee76d27074 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py @@ -0,0 +1 @@ +api_vtag = f"v1" \ No newline at end of file diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py index 09838e1947e..5eacdc30f1b 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py @@ -7,13 +7,14 @@ from .containers import containers_router from .health import health_router from .mocked import mocked_router +from .._meta import api_vtag # setup and register all routes here form different modules main_router = APIRouter() main_router.include_router(health_router) -main_router.include_router(compose_router) -main_router.include_router(containers_router) -main_router.include_router(container_router) +main_router.include_router(compose_router, prefix=f"/{api_vtag}") +main_router.include_router(containers_router, prefix=f"/{api_vtag}") +main_router.include_router(container_router, prefix=f"/{api_vtag}") main_router.include_router(mocked_router) __all__ = ["main_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py index 41060b7981f..e6deddac273 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py @@ -8,6 +8,7 @@ from .settings import DynamicSidecarSettings from .shared_handlers import on_shutdown_handler from .storage import SharedStore +from ._meta import api_vtag logger = logging.getLogger(__name__) @@ -25,7 +26,11 @@ def assemble_application() -> FastAPI: logging.root.setLevel(dynamic_sidecar_settings.loglevel) logger.debug(dynamic_sidecar_settings.json(indent=2)) - application = FastAPI(debug=dynamic_sidecar_settings.debug, docs_url="/dev/doc") + application = FastAPI( + debug=dynamic_sidecar_settings.debug, + openapi_url=f"/api/{api_vtag}/openapi.json", + docs_url="/dev/doc", + ) # store "settings" and "shared_store" for later usage application.state.settings = dynamic_sidecar_settings diff --git a/services/dynamic-sidecar/tests/unit/test_api_compose.py b/services/dynamic-sidecar/tests/unit/test_api_compose.py index 26cb11d458e..07771a82e32 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_compose.py +++ b/services/dynamic-sidecar/tests/unit/test_api_compose.py @@ -7,6 +7,7 @@ import pytest from async_asgi_testclient import TestClient from faker import Faker +from simcore_service_dynamic_sidecar._meta import api_vtag DEFAULT_COMMAND_TIMEOUT = 10.0 @@ -25,14 +26,14 @@ def compose_spec() -> str: async def test_store_compose_spec( test_client: TestClient, compose_spec: Dict[str, Any] ): - response = await test_client.post("/compose:store", data=compose_spec) + response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text assert response.text == "" @pytest.mark.asyncio async def test_store_compose_spec_not_provided(test_client: TestClient): - response = await test_client.post("/compose:store") + response = await test_client.post(f"/{api_vtag}/compose:store") assert response.status_code == 400, response.text assert response.text == "\nProvided yaml is not valid!" @@ -40,7 +41,9 @@ async def test_store_compose_spec_not_provided(test_client: TestClient): @pytest.mark.asyncio async def test_store_compose_spec_invalid(test_client: TestClient): invalid_compose_spec = Faker().text() - response = await test_client.post("/compose:store", data=invalid_compose_spec) + response = await test_client.post( + f"/{api_vtag}/compose:store", data=invalid_compose_spec + ) assert response.status_code == 400, response.text assert response.text.endswith("\nProvided yaml is not valid!") # 28+ characters means the compos spec is also present in the error message @@ -50,7 +53,9 @@ async def test_store_compose_spec_invalid(test_client: TestClient): @pytest.mark.asyncio async def test_preload(test_client: TestClient, compose_spec: Dict[str, Any]): response = await test_client.post( - "/compose:preload", query_string=dict(command_timeout=5.0), data=compose_spec + f"/{api_vtag}/compose:preload", + query_string=dict(command_timeout=5.0), + data=compose_spec, ) assert response.status_code == 200, response.text @@ -59,7 +64,7 @@ async def test_preload(test_client: TestClient, compose_spec: Dict[str, Any]): async def test_preload_compose_spec_not_provided(test_client: TestClient): response = await test_client.post( - "/compose:preload", query_string=dict(command_timeout=5.0) + f"/{api_vtag}/compose:preload", query_string=dict(command_timeout=5.0) ) assert response.status_code == 400, response.text assert response.text == "\nProvided yaml is not valid!" @@ -68,13 +73,14 @@ async def test_preload_compose_spec_not_provided(test_client: TestClient): @pytest.mark.asyncio async def test_compuse_up(test_client: TestClient, compose_spec: Dict[str, Any]): # store spec first - response = await test_client.post("/compose:store", data=compose_spec) + response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text assert response.text == "" # pull images for spec response = await test_client.post( - "/compose", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + f"/{api_vtag}/compose", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) assert response.status_code == 200, response.text @@ -82,13 +88,14 @@ async def test_compuse_up(test_client: TestClient, compose_spec: Dict[str, Any]) @pytest.mark.asyncio async def test_pull(test_client: TestClient, compose_spec: Dict[str, Any]): # store spec first - response = await test_client.post("/compose:store", data=compose_spec) + response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text assert response.text == "" # pull images for spec response = await test_client.get( - "/compose:pull", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + f"/{api_vtag}/compose:pull", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) assert response.status_code == 200, response.text @@ -96,7 +103,8 @@ async def test_pull(test_client: TestClient, compose_spec: Dict[str, Any]): @pytest.mark.asyncio async def test_pull_missing_spec(test_client: TestClient, compose_spec: Dict[str, Any]): response = await test_client.get( - "/compose:pull", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + f"/{api_vtag}/compose:pull", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) assert response.status_code == 400, response.text assert response.text == "No started spec to pull was found" @@ -105,7 +113,8 @@ async def test_pull_missing_spec(test_client: TestClient, compose_spec: Dict[str @pytest.mark.asyncio async def test_stop_missing_spec(test_client: TestClient, compose_spec: Dict[str, Any]): response = await test_client.put( - "/compose:stop", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + f"/{api_vtag}/compose:stop", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) assert response.status_code == 400, response.text assert response.text == "No started spec to stop was found" @@ -116,18 +125,20 @@ async def test_compuse_stop_after_running( test_client: TestClient, compose_spec: Dict[str, Any] ): # store spec first - response = await test_client.post("/compose:store", data=compose_spec) + response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text assert response.text == "" # pull images for spec response = await test_client.post( - "/compose", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + f"/{api_vtag}/compose", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) assert response.status_code == 200, response.text response = await test_client.put( - "/compose:stop", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + f"/{api_vtag}/compose:stop", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) assert response.status_code == 200, response.text @@ -137,22 +148,25 @@ async def test_compuse_delete_after_stopping( test_client: TestClient, compose_spec: Dict[str, Any] ): # store spec first - response = await test_client.post("/compose:store", data=compose_spec) + response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text assert response.text == "" # pull images for spec response = await test_client.post( - "/compose", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + f"/{api_vtag}/compose", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) assert response.status_code == 200, response.text response = await test_client.put( - "/compose:stop", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + f"/{api_vtag}/compose:stop", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) assert response.status_code == 200, response.text response = await test_client.delete( - "/compose", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT) + f"/{api_vtag}/compose", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) assert response.status_code == 200, response.text diff --git a/services/dynamic-sidecar/tests/unit/test_api_container.py b/services/dynamic-sidecar/tests/unit/test_api_container.py index ea224d8468d..cea52c03378 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_container.py +++ b/services/dynamic-sidecar/tests/unit/test_api_container.py @@ -7,6 +7,7 @@ import pytest from async_asgi_testclient import TestClient from simcore_service_dynamic_sidecar.storage import SharedStore +from simcore_service_dynamic_sidecar._meta import api_vtag @pytest.fixture @@ -27,13 +28,13 @@ async def started_containers( test_client: TestClient, compose_spec: Dict[str, Any] ) -> List[str]: # store spec first - response = await test_client.post("/compose:store", data=compose_spec) + response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text assert response.text == "" # pull images for spec response = await test_client.post( - "/compose", query_string=dict(command_timeout=10.0) + f"/{api_vtag}/compose", query_string=dict(command_timeout=10.0) ) assert response.status_code == 200, response.text @@ -56,13 +57,13 @@ async def test_container_inspect_logs_remove( for container in started_containers: # get container logs response = await test_client.get( - "/container/logs", query_string=dict(container=container) + f"/{api_vtag}/container/logs", query_string=dict(container=container) ) assert response.status_code == 200, response.text # inspect container response = await test_client.get( - "/container/inspect", query_string=dict(container=container) + f"/{api_vtag}/container/inspect", query_string=dict(container=container) ) assert response.status_code == 200, response.text parsed_response = response.json() @@ -70,7 +71,7 @@ async def test_container_inspect_logs_remove( # delete container response = await test_client.delete( - "/container/remove", query_string=dict(container=container) + f"/{api_vtag}/container/remove", query_string=dict(container=container) ) assert response.status_code == 200, response.text @@ -82,7 +83,8 @@ async def test_container_logs_with_timestamps( for container in started_containers: # get container logs response = await test_client.get( - "/container/logs", query_string=dict(container=container, timestamps=True) + f"/{api_vtag}/container/logs", + query_string=dict(container=container, timestamps=True), ) assert response.status_code == 200, response.text @@ -97,21 +99,21 @@ def _expected_error_string(container: str) -> Dict[str, str]: for container in not_started_containers: # get container logs response = await test_client.get( - "/container/logs", query_string=dict(container=container) + f"/{api_vtag}/container/logs", query_string=dict(container=container) ) assert response.status_code == 400, response.text assert response.json() == _expected_error_string(container) # inspect container response = await test_client.get( - "/container/inspect", query_string=dict(container=container) + f"/{api_vtag}/container/inspect", query_string=dict(container=container) ) assert response.status_code == 400, response.text assert response.json() == _expected_error_string(container) # delete container response = await test_client.delete( - "/container/remove", query_string=dict(container=container) + f"/{api_vtag}/container/remove", query_string=dict(container=container) ) assert response.status_code == 400, response.text assert response.json() == _expected_error_string(container) @@ -129,21 +131,21 @@ def _expected_error_string() -> Dict[str, str]: for container in started_containers: # get container logs response = await test_client.get( - "/container/logs", query_string=dict(container=container) + f"/{api_vtag}/container/logs", query_string=dict(container=container) ) assert response.status_code == 400, response.text assert response.json() == _expected_error_string() # inspect container response = await test_client.get( - "/container/inspect", query_string=dict(container=container) + f"/{api_vtag}/container/inspect", query_string=dict(container=container) ) assert response.status_code == 400, response.text assert response.json() == _expected_error_string() # delete container response = await test_client.delete( - "/container/remove", query_string=dict(container=container) + f"/{api_vtag}/container/remove", query_string=dict(container=container) ) assert response.status_code == 400, response.text assert response.json() == _expected_error_string() diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 2622e5ba808..9f26bca4214 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -8,6 +8,7 @@ import pytest from async_asgi_testclient import TestClient from simcore_service_dynamic_sidecar.storage import SharedStore +from simcore_service_dynamic_sidecar._meta import api_vtag @pytest.fixture @@ -28,13 +29,13 @@ async def started_containers( test_client: TestClient, compose_spec: Dict[str, Any] ) -> List[str]: # store spec first - response = await test_client.post("/compose:store", data=compose_spec) + response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text assert response.text == "" # pull images for spec response = await test_client.post( - "/compose", query_string=dict(command_timeout=10.0) + f"/{api_vtag}/compose", query_string=dict(command_timeout=10.0) ) assert response.status_code == 200, response.text @@ -47,7 +48,7 @@ async def started_containers( @pytest.mark.asyncio async def test_containers_get(test_client: TestClient, started_containers: List[str]): - response = await test_client.get("/containers") + response = await test_client.get(f"/{api_vtag}/containers") assert response.status_code == 200, response.text assert set(json.loads(response.text)) == set(started_containers) @@ -57,7 +58,8 @@ async def test_containers_inspect( test_client: TestClient, started_containers: List[str] ): response = await test_client.get( - "/containers:inspect", query_string=dict(container_names=started_containers) + f"/{api_vtag}/containers:inspect", + query_string=dict(container_names=started_containers), ) assert response.status_code == 200, response.text assert set(json.loads(response.text).keys()) == set(started_containers) @@ -68,7 +70,8 @@ async def test_containers_inspect_docker_error( test_client: TestClient, started_containers: List[str], mock_containers_get: None ): response = await test_client.get( - "/containers:inspect", query_string=dict(container_names=started_containers) + f"/{api_vtag}/containers:inspect", + query_string=dict(container_names=started_containers), ) assert response.status_code == 400, response.text @@ -85,7 +88,7 @@ async def test_containers_docker_status( test_client: TestClient, started_containers: List[str] ): response = await test_client.get( - "/containers:docker-status", + f"/{api_vtag}/containers:docker-status", query_string=dict(container_names=started_containers), ) assert response.status_code == 200, response.text @@ -112,7 +115,7 @@ def mark_pulling(shared_store: SharedStore) -> Generator[None, None, None]: assert shared_store.is_pulling_containsers is True response = await test_client.get( - "/containers:docker-status", + f"/{api_vtag}/containers:docker-status", query_string=dict(container_names=started_containers), ) assert response.status_code == 200, response.text @@ -128,7 +131,7 @@ async def test_containers_docker_status_docker_error( test_client: TestClient, started_containers: List[str], mock_containers_get: None ): response = await test_client.get( - "/containers:docker-status", + f"/{api_vtag}/containers:docker-status", query_string=dict(container_names=started_containers), ) assert response.status_code == 400, response.text From fec4078f8cea2939b6139ba359d0aa68870cfb63 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 14:50:22 +0200 Subject: [PATCH 033/102] codestyle --- .../src/simcore_service_dynamic_sidecar/_meta.py | 2 +- .../src/simcore_service_dynamic_sidecar/api/_routing.py | 2 +- .../src/simcore_service_dynamic_sidecar/api/compose.py | 1 + .../src/simcore_service_dynamic_sidecar/application.py | 2 +- services/dynamic-sidecar/tests/unit/test_api_container.py | 2 +- services/dynamic-sidecar/tests/unit/test_api_containers.py | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py index bee76d27074..c64b6ea1619 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py @@ -1 +1 @@ -api_vtag = f"v1" \ No newline at end of file +api_vtag = "v1" diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py index 5eacdc30f1b..f991fc80b3d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py @@ -2,12 +2,12 @@ from fastapi import APIRouter +from .._meta import api_vtag from .compose import compose_router from .container import container_router from .containers import containers_router from .health import health_router from .mocked import mocked_router -from .._meta import api_vtag # setup and register all routes here form different modules main_router = APIRouter() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index e0d620c4836..379c21c913d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -33,6 +33,7 @@ async def store_docker_compose_spec_for_later_usage( return str(e) response.status_code = 204 + return None @compose_router.post("/compose:preload", response_class=PlainTextResponse) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py index e6deddac273..a73e026b569 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py @@ -2,13 +2,13 @@ from fastapi import FastAPI +from ._meta import api_vtag from .api import main_router from .models import ApplicationHealth from .remote_debug import setup as remote_debug_setup from .settings import DynamicSidecarSettings from .shared_handlers import on_shutdown_handler from .storage import SharedStore -from ._meta import api_vtag logger = logging.getLogger(__name__) diff --git a/services/dynamic-sidecar/tests/unit/test_api_container.py b/services/dynamic-sidecar/tests/unit/test_api_container.py index cea52c03378..1c2b1698dea 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_container.py +++ b/services/dynamic-sidecar/tests/unit/test_api_container.py @@ -6,8 +6,8 @@ import pytest from async_asgi_testclient import TestClient -from simcore_service_dynamic_sidecar.storage import SharedStore from simcore_service_dynamic_sidecar._meta import api_vtag +from simcore_service_dynamic_sidecar.storage import SharedStore @pytest.fixture diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 9f26bca4214..b95f7a68076 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -7,8 +7,8 @@ import pytest from async_asgi_testclient import TestClient -from simcore_service_dynamic_sidecar.storage import SharedStore from simcore_service_dynamic_sidecar._meta import api_vtag +from simcore_service_dynamic_sidecar.storage import SharedStore @pytest.fixture From 33425b5c9ddecca7d835f4a06ef07c7fa1fe2eb5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 14:57:44 +0200 Subject: [PATCH 034/102] container api routes refactored --- .../api/container.py | 30 +++++++------- .../tests/unit/test_api_container.py | 40 +++++-------------- 2 files changed, 26 insertions(+), 44 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py index 3182335af6d..c7d0fbdcdaf 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py @@ -8,12 +8,12 @@ container_router = APIRouter() -@container_router.get("/container/logs") +@container_router.get("/container/{name_or_id}/logs") async def get_container_logs( # pylint: disable=unused-argument request: Request, response: Response, - container: str, + name_or_id: str, since: int = Query( 0, title="Timstamp", @@ -33,14 +33,14 @@ async def get_container_logs( """ Returns the logs of a given container if found """ shared_store: SharedStore = request.app.state.shared_store - if container not in shared_store.get_container_names(): + if name_or_id not in shared_store.get_container_names(): response.status_code = 400 - return dict(error=f"No container '{container}' was started") + return dict(error=f"No container '{name_or_id}' was started") docker = aiodocker.Docker() try: - container_instance = await docker.containers.get(container) + container_instance = await docker.containers.get(name_or_id) args = dict(stdout=True, stderr=True) if timestamps: @@ -52,41 +52,41 @@ async def get_container_logs( return dict(error=e.message) -@container_router.get("/container/inspect") +@container_router.get("/container/{name_or_id}/inspect") async def container_inspect( - request: Request, response: Response, container: str + request: Request, response: Response, name_or_id: str ) -> Dict[str, Any]: """ Returns information about the container, like docker inspect command """ shared_store: SharedStore = request.app.state.shared_store - if container not in shared_store.get_container_names(): + if name_or_id not in shared_store.get_container_names(): response.status_code = 400 - return dict(error=f"No container '{container}' was started") + return dict(error=f"No container '{name_or_id}' was started") docker = aiodocker.Docker() try: - container_instance = await docker.containers.get(container) + container_instance = await docker.containers.get(name_or_id) return await container_instance.show() except aiodocker.exceptions.DockerError as e: response.status_code = 400 return dict(error=e.message) -@container_router.delete("/container/remove") +@container_router.delete("/container/{name_or_id}/remove") async def container_remove( - request: Request, response: Response, container: str + request: Request, response: Response, name_or_id: str ) -> Union[bool, Dict[str, Any]]: shared_store: SharedStore = request.app.state.shared_store - if container not in shared_store.get_container_names(): + if name_or_id not in shared_store.get_container_names(): response.status_code = 400 - return dict(error=f"No container '{container}' was started") + return dict(error=f"No container '{name_or_id}' was started") docker = aiodocker.Docker() try: - container_instance = await docker.containers.get(container) + container_instance = await docker.containers.get(name_or_id) await container_instance.delete() return True except aiodocker.exceptions.DockerError as e: diff --git a/services/dynamic-sidecar/tests/unit/test_api_container.py b/services/dynamic-sidecar/tests/unit/test_api_container.py index 1c2b1698dea..6703158185c 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_container.py +++ b/services/dynamic-sidecar/tests/unit/test_api_container.py @@ -56,23 +56,17 @@ async def test_container_inspect_logs_remove( ): for container in started_containers: # get container logs - response = await test_client.get( - f"/{api_vtag}/container/logs", query_string=dict(container=container) - ) + response = await test_client.get(f"/{api_vtag}/container/{container}/logs") assert response.status_code == 200, response.text # inspect container - response = await test_client.get( - f"/{api_vtag}/container/inspect", query_string=dict(container=container) - ) + response = await test_client.get(f"/{api_vtag}/container/{container}/inspect") assert response.status_code == 200, response.text parsed_response = response.json() assert parsed_response["Name"] == f"/{container}" # delete container - response = await test_client.delete( - f"/{api_vtag}/container/remove", query_string=dict(container=container) - ) + response = await test_client.delete(f"/{api_vtag}/container/{container}/remove") assert response.status_code == 200, response.text @@ -83,8 +77,8 @@ async def test_container_logs_with_timestamps( for container in started_containers: # get container logs response = await test_client.get( - f"/{api_vtag}/container/logs", - query_string=dict(container=container, timestamps=True), + f"/{api_vtag}/container/{container}/logs", + query_string=dict(timestamps=True), ) assert response.status_code == 200, response.text @@ -98,23 +92,17 @@ def _expected_error_string(container: str) -> Dict[str, str]: for container in not_started_containers: # get container logs - response = await test_client.get( - f"/{api_vtag}/container/logs", query_string=dict(container=container) - ) + response = await test_client.get(f"/{api_vtag}/container/{container}/logs") assert response.status_code == 400, response.text assert response.json() == _expected_error_string(container) # inspect container - response = await test_client.get( - f"/{api_vtag}/container/inspect", query_string=dict(container=container) - ) + response = await test_client.get(f"/{api_vtag}/container/{container}/inspect") assert response.status_code == 400, response.text assert response.json() == _expected_error_string(container) # delete container - response = await test_client.delete( - f"/{api_vtag}/container/remove", query_string=dict(container=container) - ) + response = await test_client.delete(f"/{api_vtag}/container/{container}/remove") assert response.status_code == 400, response.text assert response.json() == _expected_error_string(container) @@ -130,22 +118,16 @@ def _expected_error_string() -> Dict[str, str]: for container in started_containers: # get container logs - response = await test_client.get( - f"/{api_vtag}/container/logs", query_string=dict(container=container) - ) + response = await test_client.get(f"/{api_vtag}/container/{container}/logs") assert response.status_code == 400, response.text assert response.json() == _expected_error_string() # inspect container - response = await test_client.get( - f"/{api_vtag}/container/inspect", query_string=dict(container=container) - ) + response = await test_client.get(f"/{api_vtag}/container/{container}/inspect") assert response.status_code == 400, response.text assert response.json() == _expected_error_string() # delete container - response = await test_client.delete( - f"/{api_vtag}/container/remove", query_string=dict(container=container) - ) + response = await test_client.delete(f"/{api_vtag}/container/{container}/remove") assert response.status_code == 400, response.text assert response.json() == _expected_error_string() From 849db29c2a61d0795499d83c75d72168adba94ee Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 15:02:48 +0200 Subject: [PATCH 035/102] renamed storage to shared_store --- .../src/simcore_service_dynamic_sidecar/api/compose.py | 2 +- .../src/simcore_service_dynamic_sidecar/api/container.py | 2 +- .../src/simcore_service_dynamic_sidecar/application.py | 2 +- .../src/simcore_service_dynamic_sidecar/shared_handlers.py | 2 +- .../{storage.py => shared_store.py} | 0 services/dynamic-sidecar/tests/conftest.py | 2 +- services/dynamic-sidecar/tests/unit/test_api_container.py | 2 +- services/dynamic-sidecar/tests/unit/test_api_containers.py | 2 +- 8 files changed, 7 insertions(+), 7 deletions(-) rename services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/{storage.py => shared_store.py} (100%) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index 379c21c913d..fdaa97a8e66 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -7,7 +7,7 @@ from ..settings import DynamicSidecarSettings from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command -from ..storage import SharedStore +from ..shared_store import SharedStore from ..utils import InvalidComposeSpec logger = logging.getLogger(__name__) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py index c7d0fbdcdaf..df37150fd1f 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py @@ -3,7 +3,7 @@ import aiodocker from fastapi import APIRouter, Query, Request, Response -from ..storage import SharedStore +from ..shared_store import SharedStore container_router = APIRouter() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py index a73e026b569..d5d4de84198 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py @@ -8,7 +8,7 @@ from .remote_debug import setup as remote_debug_setup from .settings import DynamicSidecarSettings from .shared_handlers import on_shutdown_handler -from .storage import SharedStore +from .shared_store import SharedStore logger = logging.getLogger(__name__) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py index 0aa54065687..4a9f7753b4c 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py @@ -4,7 +4,7 @@ from fastapi import FastAPI from .settings import DynamicSidecarSettings -from .storage import SharedStore +from .shared_store import SharedStore from .utils import async_command, write_to_tmp_file logger = logging.getLogger(__name__) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/storage.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py similarity index 100% rename from services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/storage.py rename to services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 772b4a37da5..217951d6c93 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -14,7 +14,7 @@ from simcore_service_dynamic_sidecar.application import assemble_application from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command -from simcore_service_dynamic_sidecar.storage import SharedStore +from simcore_service_dynamic_sidecar.shared_store import SharedStore @pytest.fixture(scope="module", autouse=True) diff --git a/services/dynamic-sidecar/tests/unit/test_api_container.py b/services/dynamic-sidecar/tests/unit/test_api_container.py index 6703158185c..b1dd7996dba 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_container.py +++ b/services/dynamic-sidecar/tests/unit/test_api_container.py @@ -7,7 +7,7 @@ import pytest from async_asgi_testclient import TestClient from simcore_service_dynamic_sidecar._meta import api_vtag -from simcore_service_dynamic_sidecar.storage import SharedStore +from simcore_service_dynamic_sidecar.shared_store import SharedStore @pytest.fixture diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index b95f7a68076..f119d2cd00a 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -8,7 +8,7 @@ import pytest from async_asgi_testclient import TestClient from simcore_service_dynamic_sidecar._meta import api_vtag -from simcore_service_dynamic_sidecar.storage import SharedStore +from simcore_service_dynamic_sidecar.shared_store import SharedStore @pytest.fixture From b90e7bba214b14cbe65d7047c369a05a15728b58 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 15:49:07 +0200 Subject: [PATCH 036/102] refacted shared_store to use Pydantic models --- .../api/compose.py | 12 ++-- .../api/container.py | 6 +- .../api/containers.py | 6 +- .../shared_handlers.py | 2 +- .../shared_store.py | 57 +++++++------------ services/dynamic-sidecar/tests/conftest.py | 2 +- .../tests/unit/test_api_container.py | 2 +- .../tests/unit/test_api_containers.py | 6 +- 8 files changed, 39 insertions(+), 54 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index fdaa97a8e66..82213471d61 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -60,7 +60,7 @@ async def create_docker_compose_configuration_containers_without_starting( ) finished_without_errors, stdout = await write_file_and_run_command( settings=settings, - file_content=shared_store.get_spec(), + file_content=shared_store.compose_spec, command=command, command_timeout=command_timeout, ) @@ -84,7 +84,7 @@ async def start_or_update_docker_compose_configuration( ) finished_without_errors, stdout = await write_file_and_run_command( settings=settings, - file_content=shared_store.get_spec(), + file_content=shared_store.compose_spec, command=command, command_timeout=command_timeout, ) @@ -101,7 +101,7 @@ async def pull_docker_required_docker_images( shared_store: SharedStore = request.app.state.shared_store settings: DynamicSidecarSettings = request.app.state.settings - stored_compose_content = shared_store.get_spec() + stored_compose_content = shared_store.compose_spec if stored_compose_content is None: response.status_code = 400 return "No started spec to pull was found" @@ -113,7 +113,7 @@ async def pull_docker_required_docker_images( try: # mark as pulling images - shared_store.set_is_pulling_containsers() + shared_store.is_pulling_containsers = True finished_without_errors, stdout = await write_file_and_run_command( settings=settings, @@ -123,7 +123,7 @@ async def pull_docker_required_docker_images( ) finally: # remove mark - shared_store.unset_is_pulling_containsers() + shared_store.is_pulling_containsers = False response.status_code = 200 if finished_without_errors else 400 return stdout @@ -138,7 +138,7 @@ async def stop_containers_without_removing_them( shared_store: SharedStore = request.app.state.shared_store settings: DynamicSidecarSettings = request.app.state.settings - stored_compose_content = shared_store.get_spec() + stored_compose_content = shared_store.compose_spec if stored_compose_content is None: response.status_code = 400 return "No started spec to stop was found" diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py index df37150fd1f..249beb37844 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py @@ -33,7 +33,7 @@ async def get_container_logs( """ Returns the logs of a given container if found """ shared_store: SharedStore = request.app.state.shared_store - if name_or_id not in shared_store.get_container_names(): + if name_or_id not in shared_store.container_names: response.status_code = 400 return dict(error=f"No container '{name_or_id}' was started") @@ -59,7 +59,7 @@ async def container_inspect( """ Returns information about the container, like docker inspect command """ shared_store: SharedStore = request.app.state.shared_store - if name_or_id not in shared_store.get_container_names(): + if name_or_id not in shared_store.container_names: response.status_code = 400 return dict(error=f"No container '{name_or_id}' was started") @@ -79,7 +79,7 @@ async def container_remove( ) -> Union[bool, Dict[str, Any]]: shared_store: SharedStore = request.app.state.shared_store - if name_or_id not in shared_store.get_container_names(): + if name_or_id not in shared_store.container_names: response.status_code = 400 return dict(error=f"No container '{name_or_id}' was started") diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index e22f2adba92..8775121f96e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -9,7 +9,7 @@ @containers_router.get("/containers") async def get_spawned_container_names(request: Request) -> List[str]: """ Returns a list of containers created using docker-compose """ - return request.app.state.shared_store.get_container_names() + return request.app.state.shared_store.container_names @containers_router.get("/containers:inspect") @@ -17,7 +17,7 @@ async def containers_inspect(request: Request, response: Response) -> Dict[str, """ Returns information about the container, like docker inspect command """ docker = aiodocker.Docker() - container_names = request.app.state.shared_store.get_container_names() + container_names = request.app.state.shared_store.container_names container_names = container_names if container_names else {} results = {} @@ -45,7 +45,7 @@ def assemble_entry(status: str, error: str = "") -> Dict[str, str]: docker = aiodocker.Docker() shared_store = request.app.state.shared_store - container_names = shared_store.get_container_names() + container_names = shared_store.container_names container_names = container_names if container_names else {} # if containers are being pulled, return pulling (fake status) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py index 4a9f7753b4c..fd4e67a8699 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py @@ -33,7 +33,7 @@ async def remove_the_compose_spec( shared_store: SharedStore, settings: DynamicSidecarSettings, command_timeout: float ) -> Tuple[bool, str]: - stored_compose_content = shared_store.get_spec() + stored_compose_content = shared_store.compose_spec if stored_compose_content is None: return True, "No started spec to remove was found" diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py index 2119f88ed50..0917181981e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py @@ -1,51 +1,36 @@ -from typing import Any, Dict, List, Optional +from typing import List, Optional + +from pydantic import BaseModel, Field, PrivateAttr from .settings import DynamicSidecarSettings from .utils import assemble_container_names, validate_compose_spec -class SharedStore: - """Define custom storage abstraction for easy future extension""" - - __slots__ = ("_storage", "_settings", "_is_pulling_containsers") - - _K_COMPOSE_SPEC = "compose_spec" - _K_CONTAINER_NAMES = "container_names" +class SharedStore(BaseModel): + _settings: DynamicSidecarSettings = PrivateAttr() - def __set_as_compose_spec_none(self): - self._storage[self._K_COMPOSE_SPEC] = None - self._storage[self._K_CONTAINER_NAMES] = [] + compose_spec: Optional[str] = Field( + None, description="stores the stringified compose spec" + ) + container_names: List[str] = Field( + [], description="stores the container names from the compose_spec" + ) + is_pulling_containsers: bool = Field( + False, description="set to True while the containers are being pulled" + ) def __init__(self, settings: DynamicSidecarSettings): - self._storage: Dict[str, Any] = {} - self._settings: DynamicSidecarSettings = settings - self._is_pulling_containsers: bool = False - self.__set_as_compose_spec_none() + self._settings = settings + super().__init__() def put_spec(self, compose_file_content: Optional[str]) -> None: + """Validates the spec before storing it and updated the container_names list""" if compose_file_content is None: - self.__set_as_compose_spec_none() + self.compose_spec = None + self.container_names = [] return - self._storage[self._K_COMPOSE_SPEC] = validate_compose_spec( + self.compose_spec = validate_compose_spec( settings=self._settings, compose_file_content=compose_file_content ) - self._storage[self._K_CONTAINER_NAMES] = assemble_container_names( - self._storage[self._K_COMPOSE_SPEC] - ) - - def get_spec(self) -> Optional[str]: - return self._storage.get(self._K_COMPOSE_SPEC) - - def get_container_names(self) -> List[str]: - return self._storage[self._K_CONTAINER_NAMES] - - @property - def is_pulling_containsers(self) -> bool: - return self._is_pulling_containsers - - def set_is_pulling_containsers(self) -> None: - self._is_pulling_containsers = True - - def unset_is_pulling_containsers(self) -> None: - self._is_pulling_containsers = False + self.container_names = assemble_container_names(self.compose_spec) diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 217951d6c93..1ce9cb318b3 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -42,7 +42,7 @@ async def cleanup_containers(app: FastAPI) -> AsyncGenerator[None, None]: # run docker compose down here shared_store: SharedStore = app.state.shared_store - stored_compose_content = shared_store.get_spec() + stored_compose_content = shared_store.compose_spec if stored_compose_content is None: # if no compose-spec is stored skip this operation diff --git a/services/dynamic-sidecar/tests/unit/test_api_container.py b/services/dynamic-sidecar/tests/unit/test_api_container.py index b1dd7996dba..579b6f21fb0 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_container.py +++ b/services/dynamic-sidecar/tests/unit/test_api_container.py @@ -39,7 +39,7 @@ async def started_containers( assert response.status_code == 200, response.text shared_store: SharedStore = test_client.application.state.shared_store - container_names = shared_store.get_container_names() + container_names = shared_store.container_names assert len(container_names) == 2 return container_names diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index f119d2cd00a..7010f2373c2 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -40,7 +40,7 @@ async def started_containers( assert response.status_code == 200, response.text shared_store: SharedStore = test_client.application.state.shared_store - container_names = shared_store.get_container_names() + container_names = shared_store.container_names assert len(container_names) == 2 return container_names @@ -104,10 +104,10 @@ async def test_containers_docker_status_pulling_containers( @contextmanager def mark_pulling(shared_store: SharedStore) -> Generator[None, None, None]: try: - shared_store.set_is_pulling_containsers() + shared_store.is_pulling_containsers = True yield finally: - shared_store.unset_is_pulling_containsers() + shared_store.is_pulling_containsers = False shared_store: SharedStore = test_client.application.state.shared_store From 2c172e4a182cb173f573d8598c1ed6f5d8452d76 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 16:51:27 +0200 Subject: [PATCH 037/102] missed a rename --- services/dynamic-sidecar/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 1ce9cb318b3..217951d6c93 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -42,7 +42,7 @@ async def cleanup_containers(app: FastAPI) -> AsyncGenerator[None, None]: # run docker compose down here shared_store: SharedStore = app.state.shared_store - stored_compose_content = shared_store.compose_spec + stored_compose_content = shared_store.get_spec() if stored_compose_content is None: # if no compose-spec is stored skip this operation From 4d987762e26015c26a738a5b0a56d017168ce133 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 16:51:45 +0200 Subject: [PATCH 038/102] added makefile entry to generate openapi spec --- services/dynamic-sidecar/.env-devel | 12 ++++++++++++ services/dynamic-sidecar/Makefile | 15 +++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 services/dynamic-sidecar/.env-devel diff --git a/services/dynamic-sidecar/.env-devel b/services/dynamic-sidecar/.env-devel new file mode 100644 index 00000000000..a7c9f775252 --- /dev/null +++ b/services/dynamic-sidecar/.env-devel @@ -0,0 +1,12 @@ +# Environment used to configure storage services +# +# - To expose in cmd: export $(grep -v '^#' .env-devel | xargs -0) +# + +# environs in Dockerfile ---------------- +SC_BOOT_MODE=local-development + + +# service specific required vars +DYNAMIC_SIDECAR_compose_namespace=dev-namespace +DYNAMIC_SIDECAR_docker_compose_down_timeout=15 \ No newline at end of file diff --git a/services/dynamic-sidecar/Makefile b/services/dynamic-sidecar/Makefile index 3c42d266e68..bacf8223dcb 100644 --- a/services/dynamic-sidecar/Makefile +++ b/services/dynamic-sidecar/Makefile @@ -6,6 +6,21 @@ APP_NAME := $(notdir $(CURDIR)) .DEFAULT_GOAL := help +.env: .env-devel ## creates .env file from defaults in .env-devel + $(if $(wildcard $@), \ + @echo "WARNING ##### $< is newer than $@ ####"; diff -uN $@ $<; false;,\ + @echo "WARNING ##### $@ does not exist, cloning $< as $@ ############"; cp $< $@) + + +.PHONY: openapi.json +openapi.json: .env ## Creates OAS document openapi.json + # generating openapi specs (OAS) file + export $(shell grep -v '^#' $< | xargs -0) && python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(app.openapi(), indent=2) )" > $@ + # validates OAS file: $@ + @cd $(CURDIR); \ + $(SCRIPTS_DIR)/openapi-generator-cli.bash validate --input-spec /local/$@ + + .PHONY: codestyle codestyle: ## enforces codestyle and runs pylint and mypy isort setup.py src/simcore_service_dynamic_sidecar tests From 769fe4e8be2dc64756e7d4a16041684da27ab25b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 14 Apr 2021 17:00:11 +0200 Subject: [PATCH 039/102] adding correct mypy file --- ci/github/unit-testing/dynamic-sidecar.bash | 2 +- services/dynamic-sidecar/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/github/unit-testing/dynamic-sidecar.bash b/ci/github/unit-testing/dynamic-sidecar.bash index 21817ec5847..9336795d157 100755 --- a/ci/github/unit-testing/dynamic-sidecar.bash +++ b/ci/github/unit-testing/dynamic-sidecar.bash @@ -20,7 +20,7 @@ codestyle(){ echo "pylint" pylint --rcfile=../../.pylintrc src/simcore_service_dynamic_sidecar tests/ echo "mypy" - mypy src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports + mypy mypy --config-file mypy.ini src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports popd } diff --git a/services/dynamic-sidecar/Makefile b/services/dynamic-sidecar/Makefile index bacf8223dcb..5f1a99c5d4d 100644 --- a/services/dynamic-sidecar/Makefile +++ b/services/dynamic-sidecar/Makefile @@ -26,7 +26,7 @@ codestyle: ## enforces codestyle and runs pylint and mypy isort setup.py src/simcore_service_dynamic_sidecar tests black src/simcore_service_dynamic_sidecar tests/ pylint --rcfile=../../.pylintrc src/simcore_service_dynamic_sidecar tests/ - mypy src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports + mypy --config-file ../../mypy.ini src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports .PHONY: run-github-action-locally From e19e1b09ea7f3051374590a49562b28085165e42 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 15 Apr 2021 08:47:43 +0200 Subject: [PATCH 040/102] mypy suggestions --- services/dynamic-sidecar/Makefile | 2 + .../api/__init__.py | 2 + .../api/container.py | 6 ++- .../api/containers.py | 16 ++++--- .../application.py | 2 +- .../simcore_service_dynamic_sidecar/main.py | 2 +- .../remote_debug.py | 2 +- .../settings.py | 12 +++--- .../simcore_service_dynamic_sidecar/utils.py | 8 ++-- services/dynamic-sidecar/tests/conftest.py | 42 +++---------------- .../tests/unit/test_api_compose.py | 28 ++++++++----- .../tests/unit/test_api_container.py | 8 ++-- .../tests/unit/test_api_containers.py | 14 ++++--- .../tests/unit/test_api_health.py | 4 +- .../tests/unit/test_api_mocked.py | 2 +- 15 files changed, 70 insertions(+), 80 deletions(-) diff --git a/services/dynamic-sidecar/Makefile b/services/dynamic-sidecar/Makefile index 5f1a99c5d4d..45624b3930d 100644 --- a/services/dynamic-sidecar/Makefile +++ b/services/dynamic-sidecar/Makefile @@ -32,4 +32,6 @@ codestyle: ## enforces codestyle and runs pylint and mypy .PHONY: run-github-action-locally run-github-action-locally: ## runs "unit-test-dynamic-sidecar" defined int github workflow locally # Note: ⚡ act is required https://github.com/nektos/act to emulate the github actons locally + #TODO: run as docker in docker https://www.quickdevnotes.com/run-github-actions-locally-docker-nektos-act/ + # change the script to simply start the commmand from the docker image @act -C ../../. -P ubuntu-20.04=catthehacker/ubuntu:act-20.04 -j unit-test-dynamic-sidecar diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/__init__.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/__init__.py index 94ff56968e4..97f65e482d1 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/__init__.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/__init__.py @@ -1 +1,3 @@ from ._routing import main_router + +__all__ = ["main_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py index 249beb37844..7fa7006ab74 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py @@ -46,7 +46,8 @@ async def get_container_logs( if timestamps: args["timestamps"] = True - return await container_instance.log(**args) + container_logs: str = await container_instance.log(**args) + return container_logs except aiodocker.exceptions.DockerError as e: response.status_code = 400 return dict(error=e.message) @@ -67,7 +68,8 @@ async def container_inspect( try: container_instance = await docker.containers.get(name_or_id) - return await container_instance.show() + inspect_result: Dict[str, Any] = await container_instance.show() + return inspect_result except aiodocker.exceptions.DockerError as e: response.status_code = 400 return dict(error=e.message) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 8775121f96e..e072190638d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -3,13 +3,16 @@ import aiodocker from fastapi import APIRouter, Request, Response +from ..shared_store import SharedStore + containers_router = APIRouter() @containers_router.get("/containers") async def get_spawned_container_names(request: Request) -> List[str]: """ Returns a list of containers created using docker-compose """ - return request.app.state.shared_store.container_names + shared_store: SharedStore = request.app.state.shared_store + return shared_store.container_names @containers_router.get("/containers:inspect") @@ -17,8 +20,10 @@ async def containers_inspect(request: Request, response: Response) -> Dict[str, """ Returns information about the container, like docker inspect command """ docker = aiodocker.Docker() - container_names = request.app.state.shared_store.container_names - container_names = container_names if container_names else {} + shared_store: SharedStore = request.app.state.shared_store + container_names = ( + shared_store.container_names if shared_store.container_names else {} + ) results = {} @@ -45,8 +50,9 @@ def assemble_entry(status: str, error: str = "") -> Dict[str, str]: docker = aiodocker.Docker() shared_store = request.app.state.shared_store - container_names = shared_store.container_names - container_names = container_names if container_names else {} + container_names = ( + shared_store.container_names if shared_store.container_names else {} + ) # if containers are being pulled, return pulling (fake status) if shared_store.is_pulling_containsers: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py index d5d4de84198..5d7116dcff7 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py @@ -34,7 +34,7 @@ def assemble_application() -> FastAPI: # store "settings" and "shared_store" for later usage application.state.settings = dynamic_sidecar_settings - application.state.shared_store = SharedStore(settings=dynamic_sidecar_settings) + application.state.shared_store = SharedStore(settings=dynamic_sidecar_settings) # type: ignore # used to keep track of the health of the application # also will be used in the /health endpoint application.state.application_health = ApplicationHealth() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py index 6e7c19fb647..e7a2bdabe30 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py @@ -12,7 +12,7 @@ app: FastAPI = assemble_application() -def main(): +def main() -> None: settings: DynamicSidecarSettings = app.state.settings uvicorn.run( diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/remote_debug.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/remote_debug.py index 75e49be73a9..8e4f19e00d5 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/remote_debug.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/remote_debug.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) -def setup(app: FastAPI): +def setup(app: FastAPI) -> None: settings = app.state.settings def on_startup() -> None: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py index 4a252432d4c..e3e9ecf02f5 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py @@ -1,5 +1,5 @@ import logging -from typing import Optional +from typing import Any, Optional from models_library.basic_types import BootModeEnum, PortInt from pydantic import BaseSettings, Field, PositiveInt, validator @@ -7,7 +7,7 @@ class DynamicSidecarSettings(BaseSettings): @classmethod - def create(cls, **settings_kwargs) -> "DynamicSidecarSettings": + def create(cls, **settings_kwargs: Any) -> "DynamicSidecarSettings": return cls( **settings_kwargs, ) @@ -23,7 +23,7 @@ def create(cls, **settings_kwargs) -> "DynamicSidecarSettings": @validator("log_level_name") @classmethod - def match_logging_level(cls, v) -> str: + def match_logging_level(cls, v: str) -> str: try: getattr(logging, v.upper()) except AttributeError as err: @@ -74,13 +74,13 @@ def match_logging_level(cls, v) -> str: ) @property - def is_development_mode(self): + def is_development_mode(self) -> bool: """If in development mode this will be True""" - return self.boot_mode == BootModeEnum.DEVELOPMENT + return self.boot_mode is BootModeEnum.DEVELOPMENT @property def loglevel(self) -> int: - return getattr(logging, self.log_level_name) + return int(getattr(logging, self.log_level_name)) class Config: case_sensitive = False diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py index 966110ccc65..17410e82b55 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py @@ -6,7 +6,7 @@ import tempfile import traceback from pathlib import Path -from typing import Any, Dict, Generator, List, Tuple +from typing import Any, AsyncGenerator, Dict, Generator, List, Tuple import aiofiles import yaml @@ -25,10 +25,10 @@ class InvalidComposeSpec(Exception): @asynccontextmanager -async def write_to_tmp_file(file_contents): +async def write_to_tmp_file(file_contents: str) -> AsyncGenerator[Path, None]: """Disposes of file on exit""" # pylint: disable=protected-access,stop-iteration-return - file_path = Path("/") / f"tmp/{next(tempfile._get_candidate_names())}" + file_path = Path("/") / f"tmp/{next(tempfile._get_candidate_names())}" # type: ignore async with aiofiles.open(file_path, mode="w") as tmp_file: await tmp_file.write(file_contents) try: @@ -37,7 +37,7 @@ async def write_to_tmp_file(file_contents): await aiofiles.os.remove(file_path) -async def async_command(command, command_timeout: float) -> Tuple[bool, str]: +async def async_command(command: str, command_timeout: float) -> Tuple[bool, str]: """Returns if the command exited correctly and the stdout of the command """ proc = await asyncio.create_subprocess_shell( command, diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 217951d6c93..bdfd16c7073 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -2,15 +2,14 @@ # pylint: disable=redefined-outer-name import os -import subprocess -import sys -from typing import AsyncGenerator +from typing import Any, AsyncGenerator from unittest import mock import aiodocker import pytest from async_asgi_testclient import TestClient from fastapi import FastAPI +from pytest_mock.plugin import MockerFixture from simcore_service_dynamic_sidecar.application import assemble_application from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command @@ -42,7 +41,7 @@ async def cleanup_containers(app: FastAPI) -> AsyncGenerator[None, None]: # run docker compose down here shared_store: SharedStore = app.state.shared_store - stored_compose_content = shared_store.get_spec() + stored_compose_content = shared_store.compose_spec if stored_compose_content is None: # if no compose-spec is stored skip this operation @@ -61,40 +60,9 @@ async def cleanup_containers(app: FastAPI) -> AsyncGenerator[None, None]: ) -@pytest.fixture(autouse=True) -async def monkey_patch_asyncio_subprocess(mocker) -> None: - # TODO: The below bug is not allowing me to fully test, - # mocking and waiting for an update - # https://bugs.python.org/issue35621 - # this issue was patched in 3.8, no need - if sys.version_info.major == 3 and sys.version_info.minor >= 8: - raise RuntimeError( - "Issue no longer present in this version of python, " - "please remote this mock on python >= 3.8" - ) - - async def create_subprocess_exec(*command, **extra_params): - class MockResponse: - def __init__(self, command, **kwargs): - self.proc = subprocess.Popen(command, **extra_params) - - async def communicate(self): - return self.proc.communicate() - - @property - def returncode(self): - return self.proc.returncode - - mock_response = MockResponse(command, **extra_params) - - return mock_response - - mocker.patch("asyncio.create_subprocess_exec", side_effect=create_subprocess_exec) - - @pytest.fixture -def mock_containers_get(mocker) -> None: - async def mock_get(*args, **kwargs): +def mock_containers_get(mocker: MockerFixture) -> None: + async def mock_get(*args: str, **kwargs: Any) -> None: raise aiodocker.exceptions.DockerError( status="mock", data=dict(message="aiodocker_mocked_error") ) diff --git a/services/dynamic-sidecar/tests/unit/test_api_compose.py b/services/dynamic-sidecar/tests/unit/test_api_compose.py index 07771a82e32..e95bd40afdb 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_compose.py +++ b/services/dynamic-sidecar/tests/unit/test_api_compose.py @@ -25,21 +25,21 @@ def compose_spec() -> str: @pytest.mark.asyncio async def test_store_compose_spec( test_client: TestClient, compose_spec: Dict[str, Any] -): +) -> None: response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text assert response.text == "" @pytest.mark.asyncio -async def test_store_compose_spec_not_provided(test_client: TestClient): +async def test_store_compose_spec_not_provided(test_client: TestClient) -> None: response = await test_client.post(f"/{api_vtag}/compose:store") assert response.status_code == 400, response.text assert response.text == "\nProvided yaml is not valid!" @pytest.mark.asyncio -async def test_store_compose_spec_invalid(test_client: TestClient): +async def test_store_compose_spec_invalid(test_client: TestClient) -> None: invalid_compose_spec = Faker().text() response = await test_client.post( f"/{api_vtag}/compose:store", data=invalid_compose_spec @@ -51,7 +51,7 @@ async def test_store_compose_spec_invalid(test_client: TestClient): @pytest.mark.asyncio -async def test_preload(test_client: TestClient, compose_spec: Dict[str, Any]): +async def test_preload(test_client: TestClient, compose_spec: Dict[str, Any]) -> None: response = await test_client.post( f"/{api_vtag}/compose:preload", query_string=dict(command_timeout=5.0), @@ -61,7 +61,7 @@ async def test_preload(test_client: TestClient, compose_spec: Dict[str, Any]): @pytest.mark.asyncio -async def test_preload_compose_spec_not_provided(test_client: TestClient): +async def test_preload_compose_spec_not_provided(test_client: TestClient) -> None: response = await test_client.post( f"/{api_vtag}/compose:preload", query_string=dict(command_timeout=5.0) @@ -71,7 +71,9 @@ async def test_preload_compose_spec_not_provided(test_client: TestClient): @pytest.mark.asyncio -async def test_compuse_up(test_client: TestClient, compose_spec: Dict[str, Any]): +async def test_compuse_up( + test_client: TestClient, compose_spec: Dict[str, Any] +) -> None: # store spec first response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text @@ -86,7 +88,7 @@ async def test_compuse_up(test_client: TestClient, compose_spec: Dict[str, Any]) @pytest.mark.asyncio -async def test_pull(test_client: TestClient, compose_spec: Dict[str, Any]): +async def test_pull(test_client: TestClient, compose_spec: Dict[str, Any]) -> None: # store spec first response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text @@ -101,7 +103,9 @@ async def test_pull(test_client: TestClient, compose_spec: Dict[str, Any]): @pytest.mark.asyncio -async def test_pull_missing_spec(test_client: TestClient, compose_spec: Dict[str, Any]): +async def test_pull_missing_spec( + test_client: TestClient, compose_spec: Dict[str, Any] +) -> None: response = await test_client.get( f"/{api_vtag}/compose:pull", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), @@ -111,7 +115,9 @@ async def test_pull_missing_spec(test_client: TestClient, compose_spec: Dict[str @pytest.mark.asyncio -async def test_stop_missing_spec(test_client: TestClient, compose_spec: Dict[str, Any]): +async def test_stop_missing_spec( + test_client: TestClient, compose_spec: Dict[str, Any] +) -> None: response = await test_client.put( f"/{api_vtag}/compose:stop", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), @@ -123,7 +129,7 @@ async def test_stop_missing_spec(test_client: TestClient, compose_spec: Dict[str @pytest.mark.asyncio async def test_compuse_stop_after_running( test_client: TestClient, compose_spec: Dict[str, Any] -): +) -> None: # store spec first response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text @@ -146,7 +152,7 @@ async def test_compuse_stop_after_running( @pytest.mark.asyncio async def test_compuse_delete_after_stopping( test_client: TestClient, compose_spec: Dict[str, Any] -): +) -> None: # store spec first response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) assert response.status_code == 204, response.text diff --git a/services/dynamic-sidecar/tests/unit/test_api_container.py b/services/dynamic-sidecar/tests/unit/test_api_container.py index 579b6f21fb0..1f7e50b2239 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_container.py +++ b/services/dynamic-sidecar/tests/unit/test_api_container.py @@ -53,7 +53,7 @@ def not_started_containers() -> List[str]: @pytest.mark.asyncio async def test_container_inspect_logs_remove( test_client: TestClient, started_containers: List[str] -): +) -> None: for container in started_containers: # get container logs response = await test_client.get(f"/{api_vtag}/container/{container}/logs") @@ -73,7 +73,7 @@ async def test_container_inspect_logs_remove( @pytest.mark.asyncio async def test_container_logs_with_timestamps( test_client: TestClient, started_containers: List[str] -): +) -> None: for container in started_containers: # get container logs response = await test_client.get( @@ -86,7 +86,7 @@ async def test_container_logs_with_timestamps( @pytest.mark.asyncio async def test_container_missing_container( test_client: TestClient, not_started_containers: List[str] -): +) -> None: def _expected_error_string(container: str) -> Dict[str, str]: return dict(error=f"No container '{container}' was started") @@ -112,7 +112,7 @@ async def test_container_docker_error( test_client: TestClient, started_containers: List[str], mock_containers_get: None, -): +) -> None: def _expected_error_string() -> Dict[str, str]: return dict(error="aiodocker_mocked_error") diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 7010f2373c2..7d533e9a205 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -47,7 +47,9 @@ async def started_containers( @pytest.mark.asyncio -async def test_containers_get(test_client: TestClient, started_containers: List[str]): +async def test_containers_get( + test_client: TestClient, started_containers: List[str] +) -> None: response = await test_client.get(f"/{api_vtag}/containers") assert response.status_code == 200, response.text assert set(json.loads(response.text)) == set(started_containers) @@ -56,7 +58,7 @@ async def test_containers_get(test_client: TestClient, started_containers: List[ @pytest.mark.asyncio async def test_containers_inspect( test_client: TestClient, started_containers: List[str] -): +) -> None: response = await test_client.get( f"/{api_vtag}/containers:inspect", query_string=dict(container_names=started_containers), @@ -68,7 +70,7 @@ async def test_containers_inspect( @pytest.mark.asyncio async def test_containers_inspect_docker_error( test_client: TestClient, started_containers: List[str], mock_containers_get: None -): +) -> None: response = await test_client.get( f"/{api_vtag}/containers:inspect", query_string=dict(container_names=started_containers), @@ -86,7 +88,7 @@ def assert_keys_exist(result: Dict[str, Any]) -> bool: @pytest.mark.asyncio async def test_containers_docker_status( test_client: TestClient, started_containers: List[str] -): +) -> None: response = await test_client.get( f"/{api_vtag}/containers:docker-status", query_string=dict(container_names=started_containers), @@ -100,7 +102,7 @@ async def test_containers_docker_status( @pytest.mark.asyncio async def test_containers_docker_status_pulling_containers( test_client: TestClient, started_containers: List[str] -): +) -> None: @contextmanager def mark_pulling(shared_store: SharedStore) -> Generator[None, None, None]: try: @@ -129,7 +131,7 @@ def mark_pulling(shared_store: SharedStore) -> Generator[None, None, None]: @pytest.mark.asyncio async def test_containers_docker_status_docker_error( test_client: TestClient, started_containers: List[str], mock_containers_get: None -): +) -> None: response = await test_client.get( f"/{api_vtag}/containers:docker-status", query_string=dict(container_names=started_containers), diff --git a/services/dynamic-sidecar/tests/unit/test_api_health.py b/services/dynamic-sidecar/tests/unit/test_api_health.py index f5a890706e1..d7cd0654567 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_health.py +++ b/services/dynamic-sidecar/tests/unit/test_api_health.py @@ -5,7 +5,9 @@ @pytest.mark.asyncio @pytest.mark.parametrize("is_healthy,status_code", [(True, 200), (False, 400)]) -async def test_is_healthy(test_client: TestClient, is_healthy: bool, status_code: int): +async def test_is_healthy( + test_client: TestClient, is_healthy: bool, status_code: int +) -> None: test_client.application.state.application_health.is_healthy = is_healthy response = await test_client.get("/health") assert response.status_code == status_code, response diff --git a/services/dynamic-sidecar/tests/unit/test_api_mocked.py b/services/dynamic-sidecar/tests/unit/test_api_mocked.py index e52ad9b73fe..43b6b601fd5 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_mocked.py +++ b/services/dynamic-sidecar/tests/unit/test_api_mocked.py @@ -29,6 +29,6 @@ def assert_200_empty(response: Response) -> bool: ("/state", "POST"), ], ) -async def test_mocked_modules(test_client: TestClient, route: str, method: str): +async def test_mocked_modules(test_client: TestClient, route: str, method: str) -> None: response = await test_client.open(route, method=method) assert assert_200_empty(response) is True From 793cf31188d1eb1c3a23474f5b78a256baf13447 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 15 Apr 2021 08:51:03 +0200 Subject: [PATCH 041/102] added openapi.json for the service --- services/dynamic-sidecar/openapi.json | 578 ++++++++++++++++++++++++++ 1 file changed, 578 insertions(+) create mode 100644 services/dynamic-sidecar/openapi.json diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json new file mode 100644 index 00000000000..e3db377f288 --- /dev/null +++ b/services/dynamic-sidecar/openapi.json @@ -0,0 +1,578 @@ +{ + "openapi": "3.0.2", + "info": { + "title": "FastAPI", + "version": "0.1.0" + }, + "paths": { + "/health": { + "get": { + "summary": "Health Endpoint", + "operationId": "health_endpoint_health_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApplicationHealth" + } + } + } + } + } + } + }, + "/v1/compose:store": { + "post": { + "summary": "Store Docker Compose Spec For Later Usage", + "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", + "operationId": "store_docker_compose_spec_for_later_usage_v1_compose_store_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "204": { + "description": "No Content" + } + } + } + }, + "/v1/compose:preload": { + "post": { + "summary": "Create Docker Compose Configuration Containers Without Starting", + "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", + "operationId": "create_docker_compose_configuration_containers_without_starting_v1_compose_preload_post", + "parameters": [ + { + "required": true, + "schema": { + "title": "Command Timeout", + "type": "number" + }, + "name": "command_timeout", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/compose": { + "post": { + "summary": "Start Or Update Docker Compose Configuration", + "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", + "operationId": "start_or_update_docker_compose_configuration_v1_compose_post", + "parameters": [ + { + "required": true, + "schema": { + "title": "Command Timeout", + "type": "number" + }, + "name": "command_timeout", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "summary": "Remove Docker Compose Configuration", + "description": "Removes the previously started service\nand returns the docker-compose output", + "operationId": "remove_docker_compose_configuration_v1_compose_delete", + "parameters": [ + { + "required": true, + "schema": { + "title": "Command Timeout", + "type": "number" + }, + "name": "command_timeout", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/compose:pull": { + "get": { + "summary": "Pull Docker Required Docker Images", + "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", + "operationId": "pull_docker_required_docker_images_v1_compose_pull_get", + "parameters": [ + { + "required": true, + "schema": { + "title": "Command Timeout", + "type": "number" + }, + "name": "command_timeout", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/compose:stop": { + "put": { + "summary": "Stop Containers Without Removing Them", + "description": "Stops the previously started service\nand returns the docker-compose output", + "operationId": "stop_containers_without_removing_them_v1_compose_stop_put", + "parameters": [ + { + "required": true, + "schema": { + "title": "Command Timeout", + "type": "number" + }, + "name": "command_timeout", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/containers": { + "get": { + "summary": "Get Spawned Container Names", + "description": "Returns a list of containers created using docker-compose ", + "operationId": "get_spawned_container_names_v1_containers_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/v1/containers:inspect": { + "get": { + "summary": "Containers Inspect", + "description": "Returns information about the container, like docker inspect command ", + "operationId": "containers_inspect_v1_containers_inspect_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/v1/containers:docker-status": { + "get": { + "summary": "Containers Docker Status", + "description": "Returns the status of the containers ", + "operationId": "containers_docker_status_v1_containers_docker_status_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/v1/container/{name_or_id}/logs": { + "get": { + "summary": "Get Container Logs", + "description": "Returns the logs of a given container if found ", + "operationId": "get_container_logs_v1_container__name_or_id__logs_get", + "parameters": [ + { + "required": true, + "schema": { + "title": "Name Or Id", + "type": "string" + }, + "name": "name_or_id", + "in": "path" + }, + { + "description": "Only return logs since this time, as a UNIX timestamp", + "required": false, + "schema": { + "title": "Timstamp", + "type": "integer", + "description": "Only return logs since this time, as a UNIX timestamp", + "default": 0 + }, + "name": "since", + "in": "query" + }, + { + "description": "Only return logs before this time, as a UNIX timestamp", + "required": false, + "schema": { + "title": "Timstamp", + "type": "integer", + "description": "Only return logs before this time, as a UNIX timestamp", + "default": 0 + }, + "name": "until", + "in": "query" + }, + { + "description": "Enabling this parameter will include timestamps in logs", + "required": false, + "schema": { + "title": "Display timestamps", + "type": "boolean", + "description": "Enabling this parameter will include timestamps in logs", + "default": false + }, + "name": "timestamps", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/container/{name_or_id}/inspect": { + "get": { + "summary": "Container Inspect", + "description": "Returns information about the container, like docker inspect command ", + "operationId": "container_inspect_v1_container__name_or_id__inspect_get", + "parameters": [ + { + "required": true, + "schema": { + "title": "Name Or Id", + "type": "string" + }, + "name": "name_or_id", + "in": "path" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/container/{name_or_id}/remove": { + "delete": { + "summary": "Container Remove", + "operationId": "container_remove_v1_container__name_or_id__remove_delete", + "parameters": [ + { + "required": true, + "schema": { + "title": "Name Or Id", + "type": "string" + }, + "name": "name_or_id", + "in": "path" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/push": { + "post": { + "summary": "Post Push", + "operationId": "post_push_push_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/retrieve": { + "get": { + "summary": "Get Retrive", + "operationId": "get_retrive_retrieve_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + }, + "post": { + "summary": "Post Retrive", + "operationId": "post_retrive_retrieve_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/state": { + "get": { + "summary": "Get State", + "operationId": "get_state_state_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + }, + "post": { + "summary": "Post State", + "operationId": "post_state_state_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + } + }, + "components": { + "schemas": { + "ApplicationHealth": { + "title": "ApplicationHealth", + "type": "object", + "properties": { + "is_healthy": { + "title": "Is Healthy", + "type": "boolean", + "description": "returns True if the service sis running correctly", + "default": true + } + } + }, + "HTTPValidationError": { + "title": "HTTPValidationError", + "type": "object", + "properties": { + "detail": { + "title": "Detail", + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + } + } + } + }, + "ValidationError": { + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "type": "object", + "properties": { + "loc": { + "title": "Location", + "type": "array", + "items": { + "type": "string" + } + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + } + } + } + } +} From 8f3b8cb1d1f2d1051adf4b04598e7b0c16d90f02 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 15 Apr 2021 08:52:05 +0200 Subject: [PATCH 042/102] codestyle also updates openapi.json --- services/dynamic-sidecar/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/services/dynamic-sidecar/Makefile b/services/dynamic-sidecar/Makefile index 45624b3930d..e583f54edce 100644 --- a/services/dynamic-sidecar/Makefile +++ b/services/dynamic-sidecar/Makefile @@ -23,6 +23,7 @@ openapi.json: .env ## Creates OAS document openapi.json .PHONY: codestyle codestyle: ## enforces codestyle and runs pylint and mypy + make openapi.json isort setup.py src/simcore_service_dynamic_sidecar tests black src/simcore_service_dynamic_sidecar tests/ pylint --rcfile=../../.pylintrc src/simcore_service_dynamic_sidecar tests/ From bed77cb53b831c7601338a3735b136514c4a7214 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 15 Apr 2021 11:21:19 +0200 Subject: [PATCH 043/102] added common scripts to enforce codestyles in development and CI --- ci/github/unit-testing/dynamic-sidecar.bash | 11 +----- scripts/codestyle.bash | 40 +++++++++++++++++++++ scripts/common.Makefile | 5 +++ services/dynamic-sidecar/Makefile | 8 ----- 4 files changed, 46 insertions(+), 18 deletions(-) create mode 100755 scripts/codestyle.bash diff --git a/ci/github/unit-testing/dynamic-sidecar.bash b/ci/github/unit-testing/dynamic-sidecar.bash index 9336795d157..0e8112e14a7 100755 --- a/ci/github/unit-testing/dynamic-sidecar.bash +++ b/ci/github/unit-testing/dynamic-sidecar.bash @@ -12,16 +12,7 @@ install() { } codestyle(){ - pushd services/dynamic-sidecar - echo "isort" - isort --check setup.py src/simcore_service_dynamic_sidecar tests - echo "black" - black --check src/simcore_service_dynamic_sidecar tests/ - echo "pylint" - pylint --rcfile=../../.pylintrc src/simcore_service_dynamic_sidecar tests/ - echo "mypy" - mypy mypy --config-file mypy.ini src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports - popd + scripts/codestyle.bash ci simcore_service_dynamic_sidecar services/dynamic-sidecar } test() { diff --git a/scripts/codestyle.bash b/scripts/codestyle.bash new file mode 100755 index 00000000000..647577eff59 --- /dev/null +++ b/scripts/codestyle.bash @@ -0,0 +1,40 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit +set -o nounset +set -o pipefail +IFS=$'\n\t' + +SRC_DIRECTORY_NAME=${2} +BASE_PATH_DIR=${3-MISSING_DIR} + + +# used for development (fails on pylint and mypy) +development(){ + echo "enforcing codestyle to soruce_directory=$SRC_DIRECTORY_NAME" + echo "isort" + isort setup.py src/$SRC_DIRECTORY_NAME tests + echo "black" + black src/$SRC_DIRECTORY_NAME tests/ + echo "pylint" + pylint --rcfile=../../.pylintrc src/$SRC_DIRECTORY_NAME tests/ + echo "mypy" + mypy --ignore-missing-imports --config-file ../../mypy.ini src/$SRC_DIRECTORY_NAME tests/ +} + + +# invoked by ci as test (also fails on isort and black) +ci(){ + echo "checking codestyle in service=$BASE_PATH_DIR with soruce_directory=$SRC_DIRECTORY_NAME" + echo "isort" + isort --check setup.py $BASE_PATH_DIR/src/$SRC_DIRECTORY_NAME $BASE_PATH_DIR/tests + echo "black" + black --check $BASE_PATH_DIR/src/$SRC_DIRECTORY_NAME $BASE_PATH_DIR/tests + echo "pylint" + pylint --rcfile=.pylintrc $BASE_PATH_DIR/src/$SRC_DIRECTORY_NAME $BASE_PATH_DIR/tests + echo "mypy" + mypy --config-file mypy.ini --ignore-missing-imports $BASE_PATH_DIR/src/$SRC_DIRECTORY_NAME $BASE_PATH_DIR/tests +} + +# Allows to call a function based on arguments passed to the script +$* diff --git a/scripts/common.Makefile b/scripts/common.Makefile index 851c688fc38..2ff3d35b4f9 100644 --- a/scripts/common.Makefile +++ b/scripts/common.Makefile @@ -131,6 +131,11 @@ code-analysis: $(REPO_BASE_DIR)/.codeclimate.yml ## runs code-climate analysis @-rm $(CURDIR)/.codeclimate.yml +.PHONY: codestyle +codestyle: ## enforces codestyle and runs pylint and mypy + @../../scripts/codestyle.bash development $(shell basename "${SRC_DIR}") + + .PHONY: version-patch version-minor version-major version-patch: ## commits version with bug fixes not affecting the cookiecuter config $(_bumpversion) diff --git a/services/dynamic-sidecar/Makefile b/services/dynamic-sidecar/Makefile index e583f54edce..88f7a4471a1 100644 --- a/services/dynamic-sidecar/Makefile +++ b/services/dynamic-sidecar/Makefile @@ -21,14 +21,6 @@ openapi.json: .env ## Creates OAS document openapi.json $(SCRIPTS_DIR)/openapi-generator-cli.bash validate --input-spec /local/$@ -.PHONY: codestyle -codestyle: ## enforces codestyle and runs pylint and mypy - make openapi.json - isort setup.py src/simcore_service_dynamic_sidecar tests - black src/simcore_service_dynamic_sidecar tests/ - pylint --rcfile=../../.pylintrc src/simcore_service_dynamic_sidecar tests/ - mypy --config-file ../../mypy.ini src/simcore_service_dynamic_sidecar tests/ --ignore-missing-imports - .PHONY: run-github-action-locally run-github-action-locally: ## runs "unit-test-dynamic-sidecar" defined int github workflow locally From 954ced18e82cbab5cc35ac9687d051e50c6b2f9d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 15 Apr 2021 16:33:28 +0200 Subject: [PATCH 044/102] imported statuses from fastapi/starlette definitions --- .../api/compose.py | 31 +++++++++++++------ .../api/container.py | 13 ++++---- .../api/containers.py | 5 +-- .../api/health.py | 3 +- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index 82213471d61..f929498ed4a 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, Request, Response from fastapi.responses import PlainTextResponse +from fastapi.status import HTTP_200_OK, HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST from ..settings import DynamicSidecarSettings from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command @@ -29,10 +30,10 @@ async def store_docker_compose_spec_for_later_usage( shared_store.put_spec(body_as_text) except InvalidComposeSpec as e: logger.warning("Error detected %s", traceback.format_exc()) - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return str(e) - response.status_code = 204 + response.status_code = HTTP_204_NO_CONTENT return None @@ -50,7 +51,7 @@ async def create_docker_compose_configuration_containers_without_starting( shared_store.put_spec(body_as_text) except InvalidComposeSpec as e: logger.warning("Error detected %s", traceback.format_exc()) - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return str(e) # --no-build might be a security risk building is disabled @@ -65,7 +66,9 @@ async def create_docker_compose_configuration_containers_without_starting( command_timeout=command_timeout, ) - response.status_code = 200 if finished_without_errors else 400 + response.status_code = ( + HTTP_200_OK if finished_without_errors else HTTP_400_BAD_REQUEST + ) return stdout @@ -89,7 +92,9 @@ async def start_or_update_docker_compose_configuration( command_timeout=command_timeout, ) - response.status_code = 200 if finished_without_errors else 400 + response.status_code = ( + HTTP_200_OK if finished_without_errors else HTTP_400_BAD_REQUEST + ) return stdout @@ -103,7 +108,7 @@ async def pull_docker_required_docker_images( stored_compose_content = shared_store.compose_spec if stored_compose_content is None: - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return "No started spec to pull was found" command = ( @@ -125,7 +130,9 @@ async def pull_docker_required_docker_images( # remove mark shared_store.is_pulling_containsers = False - response.status_code = 200 if finished_without_errors else 400 + response.status_code = ( + HTTP_200_OK if finished_without_errors else HTTP_400_BAD_REQUEST + ) return stdout @@ -140,7 +147,7 @@ async def stop_containers_without_removing_them( stored_compose_content = shared_store.compose_spec if stored_compose_content is None: - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return "No started spec to stop was found" command = ( @@ -154,7 +161,9 @@ async def stop_containers_without_removing_them( command_timeout=command_timeout, ) - response.status_code = 200 if finished_without_errors else 400 + response.status_code = ( + HTTP_200_OK if finished_without_errors else HTTP_400_BAD_REQUEST + ) return stdout @@ -170,7 +179,9 @@ async def remove_docker_compose_configuration( command_timeout=command_timeout, ) - response.status_code = 200 if finished_without_errors else 400 + response.status_code = ( + HTTP_200_OK if finished_without_errors else HTTP_400_BAD_REQUEST + ) return stdout diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py index 7fa7006ab74..2384ac1910b 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py @@ -2,6 +2,7 @@ import aiodocker from fastapi import APIRouter, Query, Request, Response +from fastapi.status import HTTP_400_BAD_REQUEST from ..shared_store import SharedStore @@ -34,7 +35,7 @@ async def get_container_logs( shared_store: SharedStore = request.app.state.shared_store if name_or_id not in shared_store.container_names: - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return dict(error=f"No container '{name_or_id}' was started") docker = aiodocker.Docker() @@ -49,7 +50,7 @@ async def get_container_logs( container_logs: str = await container_instance.log(**args) return container_logs except aiodocker.exceptions.DockerError as e: - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return dict(error=e.message) @@ -61,7 +62,7 @@ async def container_inspect( shared_store: SharedStore = request.app.state.shared_store if name_or_id not in shared_store.container_names: - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return dict(error=f"No container '{name_or_id}' was started") docker = aiodocker.Docker() @@ -71,7 +72,7 @@ async def container_inspect( inspect_result: Dict[str, Any] = await container_instance.show() return inspect_result except aiodocker.exceptions.DockerError as e: - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return dict(error=e.message) @@ -82,7 +83,7 @@ async def container_remove( shared_store: SharedStore = request.app.state.shared_store if name_or_id not in shared_store.container_names: - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return dict(error=f"No container '{name_or_id}' was started") docker = aiodocker.Docker() @@ -92,7 +93,7 @@ async def container_remove( await container_instance.delete() return True except aiodocker.exceptions.DockerError as e: - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return dict(error=e.message) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index e072190638d..df5ac169a70 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -2,6 +2,7 @@ import aiodocker from fastapi import APIRouter, Request, Response +from fastapi.status import HTTP_400_BAD_REQUEST from ..shared_store import SharedStore @@ -32,7 +33,7 @@ async def containers_inspect(request: Request, response: Response) -> Dict[str, container_instance = await docker.containers.get(container) results[container] = await container_instance.show() except aiodocker.exceptions.DockerError as e: - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return dict(error=e.message) return results @@ -73,7 +74,7 @@ def assemble_entry(status: str, error: str = "") -> Dict[str, str]: "Error": container_state.get("Error", ""), } except aiodocker.exceptions.DockerError as e: - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return dict(error=e.message) return results diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py index 6f804456411..0175bedf399 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py @@ -1,4 +1,5 @@ from fastapi import APIRouter, Request, Response +from fastapi.status import HTTP_400_BAD_REQUEST from ..models import ApplicationHealth @@ -10,7 +11,7 @@ async def health_endpoint(request: Request, response: Response) -> ApplicationHe application_health: ApplicationHealth = request.app.state.application_health if application_health.is_healthy is False: - response.status_code = 400 + response.status_code = HTTP_400_BAD_REQUEST return application_health From bf8fbf1fdabb0f13e4aabfc0c1d4a86cce84569f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 15 Apr 2021 16:42:06 +0200 Subject: [PATCH 045/102] updated docstring --- .../src/simcore_service_dynamic_sidecar/utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py index 17410e82b55..070a47ac83f 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py @@ -195,10 +195,12 @@ def validate_compose_spec( settings: DynamicSidecarSettings, compose_file_content: str ) -> str: """ - Checks the following: - - proper yaml format - - no "container_name" service property allowed, because it can - spawn 2 cotainers with the same name + Validates what looks like a docker compose spec and injects + additional data to mainly make sure: + - no collisions occur between container names + - containers are located on the same docker network + - properly target environment variables formwarded via + settings on the service """ try: @@ -259,7 +261,6 @@ def validate_compose_spec( ] spec_services[container_name_service_key] = service_data - # TODO: replace names in depends_on keys # transform back to string and return validated_compose_file_content = yaml.safe_dump(parsed_compose_spec) From e7614c2b367a52a09e34d3166cc71534fc387a75 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 15 Apr 2021 16:48:14 +0200 Subject: [PATCH 046/102] fixed imports, now importing from starlette --- .../src/simcore_service_dynamic_sidecar/api/compose.py | 2 +- .../src/simcore_service_dynamic_sidecar/api/container.py | 2 +- .../src/simcore_service_dynamic_sidecar/api/containers.py | 2 +- .../src/simcore_service_dynamic_sidecar/api/health.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index f929498ed4a..5b14480a63c 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -4,7 +4,7 @@ from fastapi import APIRouter, Request, Response from fastapi.responses import PlainTextResponse -from fastapi.status import HTTP_200_OK, HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST +from starlette.status import HTTP_200_OK, HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST from ..settings import DynamicSidecarSettings from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py index 2384ac1910b..258a27f4228 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py @@ -2,7 +2,7 @@ import aiodocker from fastapi import APIRouter, Query, Request, Response -from fastapi.status import HTTP_400_BAD_REQUEST +from starlette.status import HTTP_400_BAD_REQUEST from ..shared_store import SharedStore diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index df5ac169a70..3dc9b315755 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -2,7 +2,7 @@ import aiodocker from fastapi import APIRouter, Request, Response -from fastapi.status import HTTP_400_BAD_REQUEST +from starlette.status import HTTP_400_BAD_REQUEST from ..shared_store import SharedStore diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py index 0175bedf399..429727cc011 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py @@ -1,5 +1,5 @@ from fastapi import APIRouter, Request, Response -from fastapi.status import HTTP_400_BAD_REQUEST +from starlette.status import HTTP_400_BAD_REQUEST from ..models import ApplicationHealth From c6160404311dd06f0c70eef0e7346fc39f5ae54a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 15 Apr 2021 16:48:57 +0200 Subject: [PATCH 047/102] moved mark.asyncio --- .../src/simcore_service_dynamic_sidecar/utils.py | 4 ++-- .../dynamic-sidecar/tests/unit/test_api_compose.py | 13 ++----------- .../tests/unit/test_api_container.py | 6 ++---- .../tests/unit/test_api_containers.py | 8 ++------ .../dynamic-sidecar/tests/unit/test_api_health.py | 3 ++- .../dynamic-sidecar/tests/unit/test_api_mocked.py | 3 ++- 6 files changed, 12 insertions(+), 25 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py index 070a47ac83f..512d32ece61 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py @@ -195,11 +195,11 @@ def validate_compose_spec( settings: DynamicSidecarSettings, compose_file_content: str ) -> str: """ - Validates what looks like a docker compose spec and injects + Validates what looks like a docker compose spec and injects additional data to mainly make sure: - no collisions occur between container names - containers are located on the same docker network - - properly target environment variables formwarded via + - properly target environment variables formwarded via settings on the service """ diff --git a/services/dynamic-sidecar/tests/unit/test_api_compose.py b/services/dynamic-sidecar/tests/unit/test_api_compose.py index e95bd40afdb..eae3dc8440f 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_compose.py +++ b/services/dynamic-sidecar/tests/unit/test_api_compose.py @@ -11,6 +11,8 @@ DEFAULT_COMMAND_TIMEOUT = 10.0 +pytestmark = pytest.mark.asyncio + @pytest.fixture def compose_spec() -> str: @@ -22,7 +24,6 @@ def compose_spec() -> str: ) -@pytest.mark.asyncio async def test_store_compose_spec( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: @@ -31,14 +32,12 @@ async def test_store_compose_spec( assert response.text == "" -@pytest.mark.asyncio async def test_store_compose_spec_not_provided(test_client: TestClient) -> None: response = await test_client.post(f"/{api_vtag}/compose:store") assert response.status_code == 400, response.text assert response.text == "\nProvided yaml is not valid!" -@pytest.mark.asyncio async def test_store_compose_spec_invalid(test_client: TestClient) -> None: invalid_compose_spec = Faker().text() response = await test_client.post( @@ -50,7 +49,6 @@ async def test_store_compose_spec_invalid(test_client: TestClient) -> None: assert len(response.text) > 28 -@pytest.mark.asyncio async def test_preload(test_client: TestClient, compose_spec: Dict[str, Any]) -> None: response = await test_client.post( f"/{api_vtag}/compose:preload", @@ -60,7 +58,6 @@ async def test_preload(test_client: TestClient, compose_spec: Dict[str, Any]) -> assert response.status_code == 200, response.text -@pytest.mark.asyncio async def test_preload_compose_spec_not_provided(test_client: TestClient) -> None: response = await test_client.post( @@ -70,7 +67,6 @@ async def test_preload_compose_spec_not_provided(test_client: TestClient) -> Non assert response.text == "\nProvided yaml is not valid!" -@pytest.mark.asyncio async def test_compuse_up( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: @@ -87,7 +83,6 @@ async def test_compuse_up( assert response.status_code == 200, response.text -@pytest.mark.asyncio async def test_pull(test_client: TestClient, compose_spec: Dict[str, Any]) -> None: # store spec first response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) @@ -102,7 +97,6 @@ async def test_pull(test_client: TestClient, compose_spec: Dict[str, Any]) -> No assert response.status_code == 200, response.text -@pytest.mark.asyncio async def test_pull_missing_spec( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: @@ -114,7 +108,6 @@ async def test_pull_missing_spec( assert response.text == "No started spec to pull was found" -@pytest.mark.asyncio async def test_stop_missing_spec( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: @@ -126,7 +119,6 @@ async def test_stop_missing_spec( assert response.text == "No started spec to stop was found" -@pytest.mark.asyncio async def test_compuse_stop_after_running( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: @@ -149,7 +141,6 @@ async def test_compuse_stop_after_running( assert response.status_code == 200, response.text -@pytest.mark.asyncio async def test_compuse_delete_after_stopping( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: diff --git a/services/dynamic-sidecar/tests/unit/test_api_container.py b/services/dynamic-sidecar/tests/unit/test_api_container.py index 1f7e50b2239..485868c6132 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_container.py +++ b/services/dynamic-sidecar/tests/unit/test_api_container.py @@ -9,6 +9,8 @@ from simcore_service_dynamic_sidecar._meta import api_vtag from simcore_service_dynamic_sidecar.shared_store import SharedStore +pytestmark = pytest.mark.asyncio + @pytest.fixture def compose_spec() -> str: @@ -50,7 +52,6 @@ def not_started_containers() -> List[str]: return [f"missing-container-{i}" for i in range(5)] -@pytest.mark.asyncio async def test_container_inspect_logs_remove( test_client: TestClient, started_containers: List[str] ) -> None: @@ -70,7 +71,6 @@ async def test_container_inspect_logs_remove( assert response.status_code == 200, response.text -@pytest.mark.asyncio async def test_container_logs_with_timestamps( test_client: TestClient, started_containers: List[str] ) -> None: @@ -83,7 +83,6 @@ async def test_container_logs_with_timestamps( assert response.status_code == 200, response.text -@pytest.mark.asyncio async def test_container_missing_container( test_client: TestClient, not_started_containers: List[str] ) -> None: @@ -107,7 +106,6 @@ def _expected_error_string(container: str) -> Dict[str, str]: assert response.json() == _expected_error_string(container) -@pytest.mark.asyncio async def test_container_docker_error( test_client: TestClient, started_containers: List[str], diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 7d533e9a205..32e5d916f8c 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -10,6 +10,8 @@ from simcore_service_dynamic_sidecar._meta import api_vtag from simcore_service_dynamic_sidecar.shared_store import SharedStore +pytestmark = pytest.mark.asyncio + @pytest.fixture def compose_spec() -> str: @@ -46,7 +48,6 @@ async def started_containers( return container_names -@pytest.mark.asyncio async def test_containers_get( test_client: TestClient, started_containers: List[str] ) -> None: @@ -55,7 +56,6 @@ async def test_containers_get( assert set(json.loads(response.text)) == set(started_containers) -@pytest.mark.asyncio async def test_containers_inspect( test_client: TestClient, started_containers: List[str] ) -> None: @@ -67,7 +67,6 @@ async def test_containers_inspect( assert set(json.loads(response.text).keys()) == set(started_containers) -@pytest.mark.asyncio async def test_containers_inspect_docker_error( test_client: TestClient, started_containers: List[str], mock_containers_get: None ) -> None: @@ -85,7 +84,6 @@ def assert_keys_exist(result: Dict[str, Any]) -> bool: return True -@pytest.mark.asyncio async def test_containers_docker_status( test_client: TestClient, started_containers: List[str] ) -> None: @@ -99,7 +97,6 @@ async def test_containers_docker_status( assert assert_keys_exist(decoded_response) is True -@pytest.mark.asyncio async def test_containers_docker_status_pulling_containers( test_client: TestClient, started_containers: List[str] ) -> None: @@ -128,7 +125,6 @@ def mark_pulling(shared_store: SharedStore) -> Generator[None, None, None]: assert entry["Status"] == "pulling" -@pytest.mark.asyncio async def test_containers_docker_status_docker_error( test_client: TestClient, started_containers: List[str], mock_containers_get: None ) -> None: diff --git a/services/dynamic-sidecar/tests/unit/test_api_health.py b/services/dynamic-sidecar/tests/unit/test_api_health.py index d7cd0654567..0f41070cca8 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_health.py +++ b/services/dynamic-sidecar/tests/unit/test_api_health.py @@ -2,8 +2,9 @@ from async_asgi_testclient import TestClient from simcore_service_dynamic_sidecar.models import ApplicationHealth +pytestmark = pytest.mark.asyncio + -@pytest.mark.asyncio @pytest.mark.parametrize("is_healthy,status_code", [(True, 200), (False, 400)]) async def test_is_healthy( test_client: TestClient, is_healthy: bool, status_code: int diff --git a/services/dynamic-sidecar/tests/unit/test_api_mocked.py b/services/dynamic-sidecar/tests/unit/test_api_mocked.py index 43b6b601fd5..f7fcd50af5a 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_mocked.py +++ b/services/dynamic-sidecar/tests/unit/test_api_mocked.py @@ -8,6 +8,8 @@ from async_asgi_testclient import TestClient from async_asgi_testclient.response import Response +pytestmark = pytest.mark.asyncio + def assert_200_empty(response: Response) -> bool: assert response.status_code == 200, response.text @@ -15,7 +17,6 @@ def assert_200_empty(response: Response) -> bool: return True -@pytest.mark.asyncio @pytest.mark.parametrize( "route,method", [ From 52c1e3939dece564a28f820e3eeb977cae5aaf32 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 16 Apr 2021 08:19:51 +0200 Subject: [PATCH 048/102] moved act into bash script --- scripts/act.bash | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 scripts/act.bash diff --git a/scripts/act.bash b/scripts/act.bash new file mode 100755 index 00000000000..e61fd25e6a2 --- /dev/null +++ b/scripts/act.bash @@ -0,0 +1,36 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit +set -o nounset +set -o pipefail +IFS=$'\n\t' + +# points to root soruce directory of this project, usually ../../ +ROOT_PROJECT_DIR=$1 +# name of the job, usually defined in the .github/workflows/ci-testing-deploy.yaml +JOB_TO_RUN=$2 + + +DOCKER_IMAGE_NAME=dind-act-runner +ACT_RUNNER=ubuntu-20.04=catthehacker/ubuntu:act-20.04 +ACT_VERSION_TAG=v0.2.20 # from https://github.com/nektos/act/releases + + +docker build -t $DOCKER_IMAGE_NAME -< Date: Fri, 16 Apr 2021 08:21:26 +0200 Subject: [PATCH 049/102] added act entry to common.Makefile --- scripts/common.Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/common.Makefile b/scripts/common.Makefile index 2ff3d35b4f9..48c8a721ffa 100644 --- a/scripts/common.Makefile +++ b/scripts/common.Makefile @@ -133,7 +133,12 @@ code-analysis: $(REPO_BASE_DIR)/.codeclimate.yml ## runs code-climate analysis .PHONY: codestyle codestyle: ## enforces codestyle and runs pylint and mypy - @../../scripts/codestyle.bash development $(shell basename "${SRC_DIR}") + @$(SCRIPTS_DIR)/codestyle.bash development $(shell basename "${SRC_DIR}") + +.PHONY: github-workflow-job +github-workflow-job: ## runs a github workflow job using act locally, run using "make github-workflow-job job=JOB_NAME" + # running job "${job}" + $(SCRIPTS_DIR)/act.bash ../.. ${job} .PHONY: version-patch version-minor version-major From 1567aa9108e0394b23d5a7bbf358bc7f56af7a67 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 16 Apr 2021 08:21:38 +0200 Subject: [PATCH 050/102] added entry for development of dunamic-sidecar --- services/dynamic-sidecar/Makefile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/dynamic-sidecar/Makefile b/services/dynamic-sidecar/Makefile index 88f7a4471a1..234af0650ea 100644 --- a/services/dynamic-sidecar/Makefile +++ b/services/dynamic-sidecar/Makefile @@ -24,7 +24,4 @@ openapi.json: .env ## Creates OAS document openapi.json .PHONY: run-github-action-locally run-github-action-locally: ## runs "unit-test-dynamic-sidecar" defined int github workflow locally - # Note: ⚡ act is required https://github.com/nektos/act to emulate the github actons locally - #TODO: run as docker in docker https://www.quickdevnotes.com/run-github-actions-locally-docker-nektos-act/ - # change the script to simply start the commmand from the docker image - @act -C ../../. -P ubuntu-20.04=catthehacker/ubuntu:act-20.04 -j unit-test-dynamic-sidecar + @make github-workflow-job job=unit-test-dynamic-sidecar From 60780deee54ea04032f4d02b17b1abbd2bb62940 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 20 Apr 2021 14:01:55 +0200 Subject: [PATCH 051/102] added test to check the spec has was updated --- .../dynamic-sidecar/tests/unit/test_oas_spec.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 services/dynamic-sidecar/tests/unit/test_oas_spec.py diff --git a/services/dynamic-sidecar/tests/unit/test_oas_spec.py b/services/dynamic-sidecar/tests/unit/test_oas_spec.py new file mode 100644 index 00000000000..1c64b65799f --- /dev/null +++ b/services/dynamic-sidecar/tests/unit/test_oas_spec.py @@ -0,0 +1,13 @@ +import json +from pathlib import Path + +from fastapi import FastAPI + + +def test_openapi_spec(app: FastAPI, tests_dir: Path) -> None: + spec_from_app = app.openapi() + open_api_json_file = tests_dir / ".." / "openapi.json" + stored_openapi_json_file = json.loads(open_api_json_file.read_text()) + assert ( + spec_from_app == stored_openapi_json_file + ), "make sure to run `make openapi.json`" From ce7c169b0788451fda98f4a5457c5f81cbcfc0bc Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 20 Apr 2021 14:02:08 +0200 Subject: [PATCH 052/102] added missing fixture --- services/dynamic-sidecar/tests/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index bdfd16c7073..ee8181cc70c 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -2,6 +2,8 @@ # pylint: disable=redefined-outer-name import os +import sys +from pathlib import Path from typing import Any, AsyncGenerator from unittest import mock @@ -68,3 +70,8 @@ async def mock_get(*args: str, **kwargs: Any) -> None: ) mocker.patch("aiodocker.containers.DockerContainers.get", side_effect=mock_get) + + +@pytest.fixture +def tests_dir() -> Path: + return Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent From 8a988a64b227f08978afad2f759b68cd429f3ff5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 20 Apr 2021 14:18:21 +0200 Subject: [PATCH 053/102] updated readme --- scripts/common.Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/common.Makefile b/scripts/common.Makefile index 48c8a721ffa..428786c2eb2 100644 --- a/scripts/common.Makefile +++ b/scripts/common.Makefile @@ -132,7 +132,7 @@ code-analysis: $(REPO_BASE_DIR)/.codeclimate.yml ## runs code-climate analysis .PHONY: codestyle -codestyle: ## enforces codestyle and runs pylint and mypy +codestyle: ## enforces codestyle (isort & black) finally runs pylint & mypy @$(SCRIPTS_DIR)/codestyle.bash development $(shell basename "${SRC_DIR}") .PHONY: github-workflow-job From 5860b8f4c4677f4d0075da47cb17cf93321f2b36 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 22 Apr 2021 08:16:33 +0200 Subject: [PATCH 054/102] removed unused APIs and renamed existing --- services/dynamic-sidecar/openapi.json | 189 +++++++----------- .../api/_routing.py | 2 - .../api/compose.py | 76 +------ .../api/container.py | 100 --------- .../api/containers.py | 97 ++++++++- .../api/mocked.py | 12 +- .../tests/unit/test_api_compose.py | 61 +----- .../tests/unit/test_api_container.py | 131 ------------ .../tests/unit/test_api_containers.py | 90 +++++++++ 9 files changed, 274 insertions(+), 484 deletions(-) delete mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py delete mode 100644 services/dynamic-sidecar/tests/unit/test_api_container.py diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index e3db377f288..d1dcf05a8e8 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -25,9 +25,12 @@ }, "/v1/compose:store": { "post": { - "summary": "Store Docker Compose Spec For Later Usage", + "tags": [ + "docker-compose" + ], + "summary": "Validates Docker Compose Spec And Stores It", "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", - "operationId": "store_docker_compose_spec_for_later_usage_v1_compose_store_post", + "operationId": "validates_docker_compose_spec_and_stores_it_v1_compose_store_post", "responses": { "200": { "description": "Successful Response", @@ -45,51 +48,14 @@ } } }, - "/v1/compose:preload": { - "post": { - "summary": "Create Docker Compose Configuration Containers Without Starting", - "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", - "operationId": "create_docker_compose_configuration_containers_without_starting_v1_compose_preload_post", - "parameters": [ - { - "required": true, - "schema": { - "title": "Command Timeout", - "type": "number" - }, - "name": "command_timeout", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, "/v1/compose": { "post": { - "summary": "Start Or Update Docker Compose Configuration", + "tags": [ + "docker-compose" + ], + "summary": "Runs Docker Compose Up", "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", - "operationId": "start_or_update_docker_compose_configuration_v1_compose_post", + "operationId": "runs_docker_compose_up_v1_compose_post", "parameters": [ { "required": true, @@ -125,9 +91,12 @@ } }, "delete": { - "summary": "Remove Docker Compose Configuration", + "tags": [ + "docker-compose" + ], + "summary": "Runs Docker Compose Down", "description": "Removes the previously started service\nand returns the docker-compose output", - "operationId": "remove_docker_compose_configuration_v1_compose_delete", + "operationId": "runs_docker_compose_down_v1_compose_delete", "parameters": [ { "required": true, @@ -165,49 +134,12 @@ }, "/v1/compose:pull": { "get": { - "summary": "Pull Docker Required Docker Images", - "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", - "operationId": "pull_docker_required_docker_images_v1_compose_pull_get", - "parameters": [ - { - "required": true, - "schema": { - "title": "Command Timeout", - "type": "number" - }, - "name": "command_timeout", - "in": "query" - } + "tags": [ + "docker-compose" ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/compose:stop": { - "put": { - "summary": "Stop Containers Without Removing Them", - "description": "Stops the previously started service\nand returns the docker-compose output", - "operationId": "stop_containers_without_removing_them_v1_compose_stop_put", + "summary": "Runs Docker Compose Pull", + "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", + "operationId": "runs_docker_compose_pull_v1_compose_pull_get", "parameters": [ { "required": true, @@ -245,6 +177,9 @@ }, "/v1/containers": { "get": { + "tags": [ + "containers" + ], "summary": "Get Spawned Container Names", "description": "Returns a list of containers created using docker-compose ", "operationId": "get_spawned_container_names_v1_containers_get", @@ -262,6 +197,9 @@ }, "/v1/containers:inspect": { "get": { + "tags": [ + "containers" + ], "summary": "Containers Inspect", "description": "Returns information about the container, like docker inspect command ", "operationId": "containers_inspect_v1_containers_inspect_get", @@ -279,6 +217,9 @@ }, "/v1/containers:docker-status": { "get": { + "tags": [ + "containers" + ], "summary": "Containers Docker Status", "description": "Returns the status of the containers ", "operationId": "containers_docker_status_v1_containers_docker_status_get", @@ -294,19 +235,22 @@ } } }, - "/v1/container/{name_or_id}/logs": { + "/v1/containers/{id}/logs": { "get": { + "tags": [ + "containers" + ], "summary": "Get Container Logs", "description": "Returns the logs of a given container if found ", - "operationId": "get_container_logs_v1_container__name_or_id__logs_get", + "operationId": "get_container_logs_v1_containers__id__logs_get", "parameters": [ { "required": true, "schema": { - "title": "Name Or Id", + "title": "Id", "type": "string" }, - "name": "name_or_id", + "name": "id", "in": "path" }, { @@ -368,19 +312,22 @@ } } }, - "/v1/container/{name_or_id}/inspect": { + "/v1/containers/{id}/inspect": { "get": { - "summary": "Container Inspect", + "tags": [ + "containers" + ], + "summary": "Inspect Container", "description": "Returns information about the container, like docker inspect command ", - "operationId": "container_inspect_v1_container__name_or_id__inspect_get", + "operationId": "inspect_container_v1_containers__id__inspect_get", "parameters": [ { "required": true, "schema": { - "title": "Name Or Id", + "title": "Id", "type": "string" }, - "name": "name_or_id", + "name": "id", "in": "path" } ], @@ -406,18 +353,21 @@ } } }, - "/v1/container/{name_or_id}/remove": { + "/v1/containers/{id}/remove": { "delete": { - "summary": "Container Remove", - "operationId": "container_remove_v1_container__name_or_id__remove_delete", + "tags": [ + "containers" + ], + "summary": "Remove Container", + "operationId": "remove_container_v1_containers__id__remove_delete", "parameters": [ { "required": true, "schema": { - "title": "Name Or Id", + "title": "Id", "type": "string" }, - "name": "name_or_id", + "name": "id", "in": "path" } ], @@ -445,8 +395,11 @@ }, "/push": { "post": { - "summary": "Post Push", - "operationId": "post_push_push_post", + "tags": [ + "Mocked frontend calls" + ], + "summary": "Ignored Push Post", + "operationId": "ignored_push_post_push_post", "responses": { "200": { "description": "Successful Response", @@ -461,8 +414,11 @@ }, "/retrieve": { "get": { - "summary": "Get Retrive", - "operationId": "get_retrive_retrieve_get", + "tags": [ + "Mocked frontend calls" + ], + "summary": "Ignored Port Data Load", + "operationId": "ignored_port_data_load_retrieve_get", "responses": { "200": { "description": "Successful Response", @@ -475,8 +431,11 @@ } }, "post": { - "summary": "Post Retrive", - "operationId": "post_retrive_retrieve_post", + "tags": [ + "Mocked frontend calls" + ], + "summary": "Ignored Port Data Save", + "operationId": "ignored_port_data_save_retrieve_post", "responses": { "200": { "description": "Successful Response", @@ -491,8 +450,11 @@ }, "/state": { "get": { - "summary": "Get State", - "operationId": "get_state_state_get", + "tags": [ + "Mocked frontend calls" + ], + "summary": "Ignored Load Service State State", + "operationId": "ignored_load_service_state_state_state_get", "responses": { "200": { "description": "Successful Response", @@ -505,8 +467,11 @@ } }, "post": { - "summary": "Post State", - "operationId": "post_state_state_post", + "tags": [ + "Mocked frontend calls" + ], + "summary": "Ignored Save Service State State", + "operationId": "ignored_save_service_state_state_state_post", "responses": { "200": { "description": "Successful Response", diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py index f991fc80b3d..ebdb7478c68 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py @@ -4,7 +4,6 @@ from .._meta import api_vtag from .compose import compose_router -from .container import container_router from .containers import containers_router from .health import health_router from .mocked import mocked_router @@ -14,7 +13,6 @@ main_router.include_router(health_router) main_router.include_router(compose_router, prefix=f"/{api_vtag}") main_router.include_router(containers_router, prefix=f"/{api_vtag}") -main_router.include_router(container_router, prefix=f"/{api_vtag}") main_router.include_router(mocked_router) __all__ = ["main_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index 5b14480a63c..c3212454c42 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -12,13 +12,13 @@ from ..utils import InvalidComposeSpec logger = logging.getLogger(__name__) -compose_router = APIRouter() +compose_router = APIRouter(tags=["docker-compose"]) @compose_router.post( "/compose:store", response_class=PlainTextResponse, responses={204: {"model": None}} ) -async def store_docker_compose_spec_for_later_usage( +async def validates_docker_compose_spec_and_stores_it( request: Request, response: Response ) -> Optional[str]: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ @@ -37,43 +37,8 @@ async def store_docker_compose_spec_for_later_usage( return None -@compose_router.post("/compose:preload", response_class=PlainTextResponse) -async def create_docker_compose_configuration_containers_without_starting( - request: Request, response: Response, command_timeout: float -) -> str: - """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - body_as_text = (await request.body()).decode("utf-8") - - settings: DynamicSidecarSettings = request.app.state.settings - shared_store: SharedStore = request.app.state.shared_store - - try: - shared_store.put_spec(body_as_text) - except InvalidComposeSpec as e: - logger.warning("Error detected %s", traceback.format_exc()) - response.status_code = HTTP_400_BAD_REQUEST - return str(e) - - # --no-build might be a security risk building is disabled - command = ( - "docker-compose --project-name {project} --file {file_path} " - "up --no-build --no-start" - ) - finished_without_errors, stdout = await write_file_and_run_command( - settings=settings, - file_content=shared_store.compose_spec, - command=command, - command_timeout=command_timeout, - ) - - response.status_code = ( - HTTP_200_OK if finished_without_errors else HTTP_400_BAD_REQUEST - ) - return stdout - - @compose_router.post("/compose", response_class=PlainTextResponse) -async def start_or_update_docker_compose_configuration( +async def runs_docker_compose_up( request: Request, response: Response, command_timeout: float ) -> str: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ @@ -99,7 +64,7 @@ async def start_or_update_docker_compose_configuration( @compose_router.get("/compose:pull", response_class=PlainTextResponse) -async def pull_docker_required_docker_images( +async def runs_docker_compose_pull( request: Request, response: Response, command_timeout: float ) -> str: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ @@ -136,39 +101,8 @@ async def pull_docker_required_docker_images( return stdout -@compose_router.put("/compose:stop", response_class=PlainTextResponse) -async def stop_containers_without_removing_them( - request: Request, response: Response, command_timeout: float -) -> str: - """Stops the previously started service - and returns the docker-compose output""" - shared_store: SharedStore = request.app.state.shared_store - settings: DynamicSidecarSettings = request.app.state.settings - - stored_compose_content = shared_store.compose_spec - if stored_compose_content is None: - response.status_code = HTTP_400_BAD_REQUEST - return "No started spec to stop was found" - - command = ( - "docker-compose --project-name {project} --file {file_path} " - "stop --timeout {stop_and_remove_timeout}" - ) - finished_without_errors, stdout = await write_file_and_run_command( - settings=settings, - file_content=stored_compose_content, - command=command, - command_timeout=command_timeout, - ) - - response.status_code = ( - HTTP_200_OK if finished_without_errors else HTTP_400_BAD_REQUEST - ) - return stdout - - @compose_router.delete("/compose", response_class=PlainTextResponse) -async def remove_docker_compose_configuration( +async def runs_docker_compose_down( request: Request, response: Response, command_timeout: float ) -> str: """Removes the previously started service diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py deleted file mode 100644 index 258a27f4228..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/container.py +++ /dev/null @@ -1,100 +0,0 @@ -from typing import Any, Dict, Union - -import aiodocker -from fastapi import APIRouter, Query, Request, Response -from starlette.status import HTTP_400_BAD_REQUEST - -from ..shared_store import SharedStore - -container_router = APIRouter() - - -@container_router.get("/container/{name_or_id}/logs") -async def get_container_logs( - # pylint: disable=unused-argument - request: Request, - response: Response, - name_or_id: str, - since: int = Query( - 0, - title="Timstamp", - description="Only return logs since this time, as a UNIX timestamp", - ), - until: int = Query( - 0, - title="Timstamp", - description="Only return logs before this time, as a UNIX timestamp", - ), - timestamps: bool = Query( - False, - title="Display timestamps", - description="Enabling this parameter will include timestamps in logs", - ), -) -> Union[str, Dict[str, Any]]: - """ Returns the logs of a given container if found """ - shared_store: SharedStore = request.app.state.shared_store - - if name_or_id not in shared_store.container_names: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=f"No container '{name_or_id}' was started") - - docker = aiodocker.Docker() - - try: - container_instance = await docker.containers.get(name_or_id) - - args = dict(stdout=True, stderr=True) - if timestamps: - args["timestamps"] = True - - container_logs: str = await container_instance.log(**args) - return container_logs - except aiodocker.exceptions.DockerError as e: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=e.message) - - -@container_router.get("/container/{name_or_id}/inspect") -async def container_inspect( - request: Request, response: Response, name_or_id: str -) -> Dict[str, Any]: - """ Returns information about the container, like docker inspect command """ - shared_store: SharedStore = request.app.state.shared_store - - if name_or_id not in shared_store.container_names: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=f"No container '{name_or_id}' was started") - - docker = aiodocker.Docker() - - try: - container_instance = await docker.containers.get(name_or_id) - inspect_result: Dict[str, Any] = await container_instance.show() - return inspect_result - except aiodocker.exceptions.DockerError as e: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=e.message) - - -@container_router.delete("/container/{name_or_id}/remove") -async def container_remove( - request: Request, response: Response, name_or_id: str -) -> Union[bool, Dict[str, Any]]: - shared_store: SharedStore = request.app.state.shared_store - - if name_or_id not in shared_store.container_names: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=f"No container '{name_or_id}' was started") - - docker = aiodocker.Docker() - - try: - container_instance = await docker.containers.get(name_or_id) - await container_instance.delete() - return True - except aiodocker.exceptions.DockerError as e: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=e.message) - - -__all__ = ["container_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 3dc9b315755..aa785ea114b 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -1,12 +1,13 @@ -from typing import Any, Dict, List +# pylint: disable=redefined-builtin +from typing import Any, Dict, List, Union import aiodocker -from fastapi import APIRouter, Request, Response +from fastapi import APIRouter, Query, Request, Response from starlette.status import HTTP_400_BAD_REQUEST from ..shared_store import SharedStore -containers_router = APIRouter() +containers_router = APIRouter(tags=["containers"]) @containers_router.get("/containers") @@ -80,4 +81,94 @@ def assemble_entry(status: str, error: str = "") -> Dict[str, str]: return results +@containers_router.get("/containers/{id}/logs") +async def get_container_logs( + # pylint: disable=unused-argument + request: Request, + response: Response, + id: str, + since: int = Query( + 0, + title="Timstamp", + description="Only return logs since this time, as a UNIX timestamp", + ), + until: int = Query( + 0, + title="Timstamp", + description="Only return logs before this time, as a UNIX timestamp", + ), + timestamps: bool = Query( + False, + title="Display timestamps", + description="Enabling this parameter will include timestamps in logs", + ), +) -> Union[str, Dict[str, Any]]: + """ Returns the logs of a given container if found """ + # TODO: remove from here and dump directly into the logs of this service + # do this in PR#1887 + shared_store: SharedStore = request.app.state.shared_store + + if id not in shared_store.container_names: + response.status_code = HTTP_400_BAD_REQUEST + return dict(error=f"No container '{id}' was started") + + docker = aiodocker.Docker() + + try: + container_instance = await docker.containers.get(id) + + args = dict(stdout=True, stderr=True) + if timestamps: + args["timestamps"] = True + + container_logs: str = await container_instance.log(**args) + return container_logs + except aiodocker.exceptions.DockerError as e: + response.status_code = HTTP_400_BAD_REQUEST + return dict(error=e.message) + + +@containers_router.get("/containers/{id}/inspect") +async def inspect_container( + request: Request, response: Response, id: str +) -> Dict[str, Any]: + """ Returns information about the container, like docker inspect command """ + shared_store: SharedStore = request.app.state.shared_store + + if id not in shared_store.container_names: + response.status_code = HTTP_400_BAD_REQUEST + return dict(error=f"No container '{id}' was started") + + docker = aiodocker.Docker() + + try: + container_instance = await docker.containers.get(id) + inspect_result: Dict[str, Any] = await container_instance.show() + return inspect_result + except aiodocker.exceptions.DockerError as e: + response.status_code = HTTP_400_BAD_REQUEST + return dict(error=e.message) + + +@containers_router.delete("/containers/{id}/remove") +async def remove_container( + request: Request, response: Response, id: str +) -> Union[bool, Dict[str, Any]]: + shared_store: SharedStore = request.app.state.shared_store + + if id not in shared_store.container_names: + response.status_code = HTTP_400_BAD_REQUEST + return dict(error=f"No container '{id}' was started") + + docker = aiodocker.Docker() + + try: + container_instance = await docker.containers.get(id) + await container_instance.delete() + return True + except aiodocker.exceptions.DockerError as e: + response.status_code = HTTP_400_BAD_REQUEST + return dict(error=e.message) + + __all__ = ["containers_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py index b09957c5802..6c96d172519 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/mocked.py @@ -10,35 +10,35 @@ logger = logging.getLogger(__name__) -mocked_router = APIRouter() +mocked_router = APIRouter(tags=["Mocked frontend calls"]) @mocked_router.post("/push") -async def post_push() -> str: +async def ignored_push_post() -> str: logger.warning("ignoring call POST /push from frontend") return "" @mocked_router.get("/retrieve") -async def get_retrive() -> str: +async def ignored_port_data_load() -> str: logger.warning("ignoring call GET /retrive from frontend") return "" @mocked_router.post("/retrieve") -async def post_retrive() -> str: +async def ignored_port_data_save() -> str: logger.warning("ignoring call POST /retrive from frontend") return "" @mocked_router.get("/state") -async def get_state() -> str: +async def ignored_load_service_state_state() -> str: logger.warning("ignoring call GET /state from frontend") return "" @mocked_router.post("/state") -async def post_state() -> str: +async def ignored_save_service_state_state() -> str: logger.warning("ignoring call POST /state from frontend") return "" diff --git a/services/dynamic-sidecar/tests/unit/test_api_compose.py b/services/dynamic-sidecar/tests/unit/test_api_compose.py index eae3dc8440f..6be4fcfc2c2 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_compose.py +++ b/services/dynamic-sidecar/tests/unit/test_api_compose.py @@ -49,25 +49,7 @@ async def test_store_compose_spec_invalid(test_client: TestClient) -> None: assert len(response.text) > 28 -async def test_preload(test_client: TestClient, compose_spec: Dict[str, Any]) -> None: - response = await test_client.post( - f"/{api_vtag}/compose:preload", - query_string=dict(command_timeout=5.0), - data=compose_spec, - ) - assert response.status_code == 200, response.text - - -async def test_preload_compose_spec_not_provided(test_client: TestClient) -> None: - - response = await test_client.post( - f"/{api_vtag}/compose:preload", query_string=dict(command_timeout=5.0) - ) - assert response.status_code == 400, response.text - assert response.text == "\nProvided yaml is not valid!" - - -async def test_compuse_up( +async def test_compose_up( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: # store spec first @@ -108,40 +90,7 @@ async def test_pull_missing_spec( assert response.text == "No started spec to pull was found" -async def test_stop_missing_spec( - test_client: TestClient, compose_spec: Dict[str, Any] -) -> None: - response = await test_client.put( - f"/{api_vtag}/compose:stop", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - ) - assert response.status_code == 400, response.text - assert response.text == "No started spec to stop was found" - - -async def test_compuse_stop_after_running( - test_client: TestClient, compose_spec: Dict[str, Any] -) -> None: - # store spec first - response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) - assert response.status_code == 204, response.text - assert response.text == "" - - # pull images for spec - response = await test_client.post( - f"/{api_vtag}/compose", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - ) - assert response.status_code == 200, response.text - - response = await test_client.put( - f"/{api_vtag}/compose:stop", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - ) - assert response.status_code == 200, response.text - - -async def test_compuse_delete_after_stopping( +async def test_compose_delete_after_stopping( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: # store spec first @@ -156,12 +105,6 @@ async def test_compuse_delete_after_stopping( ) assert response.status_code == 200, response.text - response = await test_client.put( - f"/{api_vtag}/compose:stop", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - ) - assert response.status_code == 200, response.text - response = await test_client.delete( f"/{api_vtag}/compose", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), diff --git a/services/dynamic-sidecar/tests/unit/test_api_container.py b/services/dynamic-sidecar/tests/unit/test_api_container.py deleted file mode 100644 index 485868c6132..00000000000 --- a/services/dynamic-sidecar/tests/unit/test_api_container.py +++ /dev/null @@ -1,131 +0,0 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument - -import json -from typing import Any, Dict, List - -import pytest -from async_asgi_testclient import TestClient -from simcore_service_dynamic_sidecar._meta import api_vtag -from simcore_service_dynamic_sidecar.shared_store import SharedStore - -pytestmark = pytest.mark.asyncio - - -@pytest.fixture -def compose_spec() -> str: - return json.dumps( - { - "version": "3", - "services": { - "first-box": {"image": "busybox"}, - "second-box": {"image": "busybox"}, - }, - } - ) - - -@pytest.fixture -async def started_containers( - test_client: TestClient, compose_spec: Dict[str, Any] -) -> List[str]: - # store spec first - response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) - assert response.status_code == 204, response.text - assert response.text == "" - - # pull images for spec - response = await test_client.post( - f"/{api_vtag}/compose", query_string=dict(command_timeout=10.0) - ) - assert response.status_code == 200, response.text - - shared_store: SharedStore = test_client.application.state.shared_store - container_names = shared_store.container_names - assert len(container_names) == 2 - - return container_names - - -@pytest.fixture -def not_started_containers() -> List[str]: - return [f"missing-container-{i}" for i in range(5)] - - -async def test_container_inspect_logs_remove( - test_client: TestClient, started_containers: List[str] -) -> None: - for container in started_containers: - # get container logs - response = await test_client.get(f"/{api_vtag}/container/{container}/logs") - assert response.status_code == 200, response.text - - # inspect container - response = await test_client.get(f"/{api_vtag}/container/{container}/inspect") - assert response.status_code == 200, response.text - parsed_response = response.json() - assert parsed_response["Name"] == f"/{container}" - - # delete container - response = await test_client.delete(f"/{api_vtag}/container/{container}/remove") - assert response.status_code == 200, response.text - - -async def test_container_logs_with_timestamps( - test_client: TestClient, started_containers: List[str] -) -> None: - for container in started_containers: - # get container logs - response = await test_client.get( - f"/{api_vtag}/container/{container}/logs", - query_string=dict(timestamps=True), - ) - assert response.status_code == 200, response.text - - -async def test_container_missing_container( - test_client: TestClient, not_started_containers: List[str] -) -> None: - def _expected_error_string(container: str) -> Dict[str, str]: - return dict(error=f"No container '{container}' was started") - - for container in not_started_containers: - # get container logs - response = await test_client.get(f"/{api_vtag}/container/{container}/logs") - assert response.status_code == 400, response.text - assert response.json() == _expected_error_string(container) - - # inspect container - response = await test_client.get(f"/{api_vtag}/container/{container}/inspect") - assert response.status_code == 400, response.text - assert response.json() == _expected_error_string(container) - - # delete container - response = await test_client.delete(f"/{api_vtag}/container/{container}/remove") - assert response.status_code == 400, response.text - assert response.json() == _expected_error_string(container) - - -async def test_container_docker_error( - test_client: TestClient, - started_containers: List[str], - mock_containers_get: None, -) -> None: - def _expected_error_string() -> Dict[str, str]: - return dict(error="aiodocker_mocked_error") - - for container in started_containers: - # get container logs - response = await test_client.get(f"/{api_vtag}/container/{container}/logs") - assert response.status_code == 400, response.text - assert response.json() == _expected_error_string() - - # inspect container - response = await test_client.get(f"/{api_vtag}/container/{container}/inspect") - assert response.status_code == 400, response.text - assert response.json() == _expected_error_string() - - # delete container - response = await test_client.delete(f"/{api_vtag}/container/{container}/remove") - assert response.status_code == 400, response.text - assert response.json() == _expected_error_string() diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 32e5d916f8c..7f661f40b12 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -48,6 +48,11 @@ async def started_containers( return container_names +@pytest.fixture +def not_started_containers() -> List[str]: + return [f"missing-container-{i}" for i in range(5)] + + async def test_containers_get( test_client: TestClient, started_containers: List[str] ) -> None: @@ -133,3 +138,88 @@ async def test_containers_docker_status_docker_error( query_string=dict(container_names=started_containers), ) assert response.status_code == 400, response.text + + +async def test_container_inspect_logs_remove( + test_client: TestClient, started_containers: List[str] +) -> None: + for container in started_containers: + # get container logs + response = await test_client.get(f"/{api_vtag}/containers/{container}/logs") + assert response.status_code == 200, response.text + + # inspect container + response = await test_client.get(f"/{api_vtag}/containers/{container}/inspect") + assert response.status_code == 200, response.text + parsed_response = response.json() + assert parsed_response["Name"] == f"/{container}" + + # delete container + response = await test_client.delete( + f"/{api_vtag}/containers/{container}/remove" + ) + assert response.status_code == 200, response.text + + +async def test_container_logs_with_timestamps( + test_client: TestClient, started_containers: List[str] +) -> None: + for container in started_containers: + # get container logs + response = await test_client.get( + f"/{api_vtag}/containers/{container}/logs", + query_string=dict(timestamps=True), + ) + assert response.status_code == 200, response.text + + +async def test_container_missing_container( + test_client: TestClient, not_started_containers: List[str] +) -> None: + def _expected_error_string(container: str) -> Dict[str, str]: + return dict(error=f"No container '{container}' was started") + + for container in not_started_containers: + # get container logs + response = await test_client.get(f"/{api_vtag}/containers/{container}/logs") + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string(container) + + # inspect container + response = await test_client.get(f"/{api_vtag}/containers/{container}/inspect") + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string(container) + + # delete container + response = await test_client.delete( + f"/{api_vtag}/containers/{container}/remove" + ) + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string(container) + + +async def test_container_docker_error( + test_client: TestClient, + started_containers: List[str], + mock_containers_get: None, +) -> None: + def _expected_error_string() -> Dict[str, str]: + return dict(error="aiodocker_mocked_error") + + for container in started_containers: + # get container logs + response = await test_client.get(f"/{api_vtag}/containers/{container}/logs") + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string() + + # inspect container + response = await test_client.get(f"/{api_vtag}/containers/{container}/inspect") + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string() + + # delete container + response = await test_client.delete( + f"/{api_vtag}/containers/{container}/remove" + ) + assert response.status_code == 400, response.text + assert response.json() == _expected_error_string() From 4d6fb94659401ecad65cd93d344ddb65dc742f92 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 22 Apr 2021 08:37:20 +0200 Subject: [PATCH 055/102] added docker-compose config validation --- services/dynamic-sidecar/openapi.json | 22 ++ .../api/compose.py | 20 +- .../shared_handlers.py | 5 +- .../shared_store.py | 23 +- .../simcore_service_dynamic_sidecar/utils.py | 219 +--------------- .../validation.py | 238 ++++++++++++++++++ 6 files changed, 283 insertions(+), 244 deletions(-) create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index d1dcf05a8e8..7d02f603003 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -31,6 +31,18 @@ "summary": "Validates Docker Compose Spec And Stores It", "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", "operationId": "validates_docker_compose_spec_and_stores_it_v1_compose_store_post", + "parameters": [ + { + "required": false, + "schema": { + "title": "Command Timeout", + "type": "number", + "default": 5.0 + }, + "name": "command_timeout", + "in": "query" + } + ], "responses": { "200": { "description": "Successful Response", @@ -44,6 +56,16 @@ }, "204": { "description": "No Content" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } } } } diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index c3212454c42..5619e9bdb45 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -9,7 +9,8 @@ from ..settings import DynamicSidecarSettings from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command from ..shared_store import SharedStore -from ..utils import InvalidComposeSpec +from ..utils import assemble_container_names +from ..validation import InvalidComposeSpec, validate_compose_spec logger = logging.getLogger(__name__) compose_router = APIRouter(tags=["docker-compose"]) @@ -19,15 +20,28 @@ "/compose:store", response_class=PlainTextResponse, responses={204: {"model": None}} ) async def validates_docker_compose_spec_and_stores_it( - request: Request, response: Response + request: Request, response: Response, command_timeout: float = 5.0 ) -> Optional[str]: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ body_as_text = (await request.body()).decode("utf-8") + settings: DynamicSidecarSettings = request.app.state.settings shared_store: SharedStore = request.app.state.shared_store + if body_as_text is None: + shared_store.compose_spec = None + shared_store.container_names = [] + return None + try: - shared_store.put_spec(body_as_text) + shared_store.compose_spec = await validate_compose_spec( + settings=settings, + compose_file_content=body_as_text, + command_timeout=command_timeout, + ) + shared_store.container_names = assemble_container_names( + shared_store.compose_spec + ) except InvalidComposeSpec as e: logger.warning("Error detected %s", traceback.format_exc()) response.status_code = HTTP_400_BAD_REQUEST diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py index fd4e67a8699..8a31a91c69e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py @@ -47,7 +47,10 @@ async def remove_the_compose_spec( command=command, command_timeout=command_timeout, ) - shared_store.put_spec(None) # removing compose-file spec + # removing compose-file spec + shared_store.compose_spec = None + shared_store.container_names = [] + return result diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py index 0917181981e..c6707b3fc4f 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py @@ -1,14 +1,9 @@ from typing import List, Optional -from pydantic import BaseModel, Field, PrivateAttr - -from .settings import DynamicSidecarSettings -from .utils import assemble_container_names, validate_compose_spec +from pydantic import BaseModel, Field class SharedStore(BaseModel): - _settings: DynamicSidecarSettings = PrivateAttr() - compose_spec: Optional[str] = Field( None, description="stores the stringified compose spec" ) @@ -18,19 +13,3 @@ class SharedStore(BaseModel): is_pulling_containsers: bool = Field( False, description="set to True while the containers are being pulled" ) - - def __init__(self, settings: DynamicSidecarSettings): - self._settings = settings - super().__init__() - - def put_spec(self, compose_file_content: Optional[str]) -> None: - """Validates the spec before storing it and updated the container_names list""" - if compose_file_content is None: - self.compose_spec = None - self.container_names = [] - return - - self.compose_spec = validate_compose_spec( - settings=self._settings, compose_file_content=compose_file_content - ) - self.container_names = assemble_container_names(self.compose_spec) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py index 512d32ece61..00dd1ea8447 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py @@ -1,29 +1,20 @@ import asyncio -import json import logging -import os -import re import tempfile import traceback from pathlib import Path -from typing import Any, AsyncGenerator, Dict, Generator, List, Tuple +from typing import AsyncGenerator, List, Tuple import aiofiles import yaml from async_generator import asynccontextmanager from async_timeout import timeout -from .settings import DynamicSidecarSettings - TEMPLATE_SEARCH_PATTERN = r"%%(.*?)%%" logger = logging.getLogger(__name__) -class InvalidComposeSpec(Exception): - """Exception used to signal incorrect docker-compose configuration file""" - - @asynccontextmanager async def write_to_tmp_file(file_contents: str) -> AsyncGenerator[Path, None]: """Disposes of file on exit""" @@ -66,214 +57,6 @@ async def async_command(command: str, command_timeout: float) -> Tuple[bool, str return finished_without_errors, decoded_stdout -def _assemble_container_name( - settings: DynamicSidecarSettings, - service_key: str, - user_given_container_name: str, - index: int, -) -> str: - strings_to_use = [ - settings.compose_namespace, - str(index), - user_given_container_name, - service_key, - ] - - container_name = "-".join([x for x in strings_to_use if len(x) > 0])[ - : settings.max_combined_container_name_length - ] - return container_name.replace("_", "-") - - -def _get_forwarded_env_vars(container_key: str) -> List[str]: - """retruns env vars targeted to each container in the compose spec""" - results = [ - # some services expect it, using it as empty - "SIMCORE_NODE_BASEPATH=", - ] - for key in os.environ.keys(): - if key.startswith("FORWARD_ENV_"): - new_entry_key = key.replace("FORWARD_ENV_", "") - - # parsing `VAR={"destination_container": "destination_container", "env_var": "PAYLOAD"}` - new_entry_payload = json.loads(os.environ[key]) - if new_entry_payload["destination_container"] != container_key: - continue - - new_entry_value = new_entry_payload["env_var"] - new_entry = f"{new_entry_key}={new_entry_value}" - results.append(new_entry) - return results - - -def _extract_templated_entries(text: str) -> List[str]: - return re.findall(TEMPLATE_SEARCH_PATTERN, text) - - -def _apply_templating_directives( - stringified_compose_spec: str, - services: Dict[str, Any], - spec_services_to_container_name: Dict[str, str], -) -> str: - """ - Some custom rules are supported for replacing `container_name` - with the following syntax `%%container_name.SERVICE_KEY_NAME%%`, - where `SERVICE_KEY_NAME` targets a container in the compose spec - - If the directive cannot be applied it will just be left untouched - """ - matches = set(_extract_templated_entries(stringified_compose_spec)) - for match in matches: - parts = match.split(".") - - if len(parts) != 2: - continue # templating will be skipped - - target_property = parts[0] - services_key = parts[1] - if target_property != "container_name": - continue # also ignore if the container_name is not the directive to replace - - remapped_service_key = spec_services_to_container_name[services_key] - replace_with = services.get(remapped_service_key, {}).get( - "container_name", None - ) - if replace_with is None: - continue # also skip here if nothing was found - - match_pattern = f"%%{match}%%" - stringified_compose_spec = stringified_compose_spec.replace( - match_pattern, replace_with - ) - - return stringified_compose_spec - - -def _merge_env_vars( - compose_spec_env_vars: List[str], settings_env_vars: List[str] -) -> List[str]: - def _gen_parts_env_vars( - env_vars: List[str], - ) -> Generator[Tuple[str, str], None, None]: - for env_var in env_vars: - key, value = env_var.split("=") - yield key, value - - # pylint: disable=unnecessary-comprehension - dict_spec_env_vars = {k: v for k, v in _gen_parts_env_vars(compose_spec_env_vars)} - dict_settings_env_vars = {k: v for k, v in _gen_parts_env_vars(settings_env_vars)} - - # overwrite spec vars with vars from settings - for key, value in dict_settings_env_vars.items(): - dict_spec_env_vars[key] = value - - # returns a single list of vars - return [f"{k}={v}" for k, v in dict_spec_env_vars.items()] - - -def _inject_backend_networking( - parsed_compose_spec: Dict[str, Any], network_name: str = "__backend__" -) -> None: - """ - Put all containers in the compose spec in the same network. - The `network_name` must only be unique inside the user defined spec; - docker-compose will add some prefix to it. - """ - - networks = parsed_compose_spec.get("networks", {}) - networks[network_name] = None - - for service_content in parsed_compose_spec["services"].values(): - service_networks = service_content.get("networks", []) - service_networks.append(network_name) - service_content["networks"] = service_networks - - parsed_compose_spec["networks"] = networks - - -def validate_compose_spec( - settings: DynamicSidecarSettings, compose_file_content: str -) -> str: - """ - Validates what looks like a docker compose spec and injects - additional data to mainly make sure: - - no collisions occur between container names - - containers are located on the same docker network - - properly target environment variables formwarded via - settings on the service - """ - - try: - parsed_compose_spec = yaml.safe_load(compose_file_content) - except yaml.YAMLError as e: - raise InvalidComposeSpec( - f"{str(e)}\n{compose_file_content}\nProvided yaml is not valid!" - ) from e - - if parsed_compose_spec is None or not isinstance(parsed_compose_spec, dict): - raise InvalidComposeSpec(f"{compose_file_content}\nProvided yaml is not valid!") - - if not {"version", "services"}.issubset(set(parsed_compose_spec.keys())): - raise InvalidComposeSpec(f"{compose_file_content}\nProvided yaml is not valid!") - - version = parsed_compose_spec["version"] - if version.startswith("1"): - raise InvalidComposeSpec(f"Provided spec version '{version}' is not supported") - - spec_services_to_container_name: Dict[str, str] = {} - - spec_services = parsed_compose_spec["services"] - for index, service in enumerate(spec_services): - service_content = spec_services[service] - - # assemble and inject the container name - user_given_container_name = service_content.get("container_name", "") - container_name = _assemble_container_name( - settings, service, user_given_container_name, index - ) - service_content["container_name"] = container_name - spec_services_to_container_name[service] = container_name - - # inject forwarded environment variables - environment_entries = service_content.get("environment", []) - service_settings_env_vars = _get_forwarded_env_vars(service) - service_content["environment"] = _merge_env_vars( - compose_spec_env_vars=environment_entries, - settings_env_vars=service_settings_env_vars, - ) - - # if more then one container is defined, add an "backend" network - if len(spec_services) > 1: - _inject_backend_networking(parsed_compose_spec) - - # replace service_key with the container_name int the dict - for service_key in list(spec_services.keys()): - container_name_service_key = spec_services_to_container_name[service_key] - service_data = spec_services.pop(service_key) - - depends_on = service_data.get("depends_on", None) - if depends_on is not None: - service_data["depends_on"] = [ - # replaces with the container name - # if not found it leaves the old value - spec_services_to_container_name.get(x, x) - for x in depends_on - ] - - spec_services[container_name_service_key] = service_data - - # transform back to string and return - validated_compose_file_content = yaml.safe_dump(parsed_compose_spec) - - compose_spec = _apply_templating_directives( - stringified_compose_spec=validated_compose_file_content, - services=spec_services, - spec_services_to_container_name=spec_services_to_container_name, - ) - - return compose_spec - - def assemble_container_names(validated_compose_content: str) -> List[str]: """returns the list of container names from a validated compose_spec""" parsed_compose_spec = yaml.safe_load(validated_compose_content) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py new file mode 100644 index 00000000000..cd3ea2b1306 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py @@ -0,0 +1,238 @@ +import json +import logging +import os +import re +from typing import Any, Dict, Generator, List, Tuple + +import yaml + +from .settings import DynamicSidecarSettings +from .shared_handlers import write_file_and_run_command + +TEMPLATE_SEARCH_PATTERN = r"%%(.*?)%%" + +logger = logging.getLogger(__name__) + + +class InvalidComposeSpec(Exception): + """Exception used to signal incorrect docker-compose configuration file""" + + +def _assemble_container_name( + settings: DynamicSidecarSettings, + service_key: str, + user_given_container_name: str, + index: int, +) -> str: + strings_to_use = [ + settings.compose_namespace, + str(index), + user_given_container_name, + service_key, + ] + + container_name = "-".join([x for x in strings_to_use if len(x) > 0])[ + : settings.max_combined_container_name_length + ] + return container_name.replace("_", "-") + + +def _get_forwarded_env_vars(container_key: str) -> List[str]: + """retruns env vars targeted to each container in the compose spec""" + results = [ + # some services expect it, using it as empty + "SIMCORE_NODE_BASEPATH=", + ] + for key in os.environ.keys(): + if key.startswith("FORWARD_ENV_"): + new_entry_key = key.replace("FORWARD_ENV_", "") + + # parsing `VAR={"destination_container": "destination_container", "env_var": "PAYLOAD"}` + new_entry_payload = json.loads(os.environ[key]) + if new_entry_payload["destination_container"] != container_key: + continue + + new_entry_value = new_entry_payload["env_var"] + new_entry = f"{new_entry_key}={new_entry_value}" + results.append(new_entry) + return results + + +def _extract_templated_entries(text: str) -> List[str]: + return re.findall(TEMPLATE_SEARCH_PATTERN, text) + + +def _apply_templating_directives( + stringified_compose_spec: str, + services: Dict[str, Any], + spec_services_to_container_name: Dict[str, str], +) -> str: + """ + Some custom rules are supported for replacing `container_name` + with the following syntax `%%container_name.SERVICE_KEY_NAME%%`, + where `SERVICE_KEY_NAME` targets a container in the compose spec + + If the directive cannot be applied it will just be left untouched + """ + matches = set(_extract_templated_entries(stringified_compose_spec)) + for match in matches: + parts = match.split(".") + + if len(parts) != 2: + continue # templating will be skipped + + target_property = parts[0] + services_key = parts[1] + if target_property != "container_name": + continue # also ignore if the container_name is not the directive to replace + + remapped_service_key = spec_services_to_container_name[services_key] + replace_with = services.get(remapped_service_key, {}).get( + "container_name", None + ) + if replace_with is None: + continue # also skip here if nothing was found + + match_pattern = f"%%{match}%%" + stringified_compose_spec = stringified_compose_spec.replace( + match_pattern, replace_with + ) + + return stringified_compose_spec + + +def _merge_env_vars( + compose_spec_env_vars: List[str], settings_env_vars: List[str] +) -> List[str]: + def _gen_parts_env_vars( + env_vars: List[str], + ) -> Generator[Tuple[str, str], None, None]: + for env_var in env_vars: + key, value = env_var.split("=") + yield key, value + + # pylint: disable=unnecessary-comprehension + dict_spec_env_vars = {k: v for k, v in _gen_parts_env_vars(compose_spec_env_vars)} + dict_settings_env_vars = {k: v for k, v in _gen_parts_env_vars(settings_env_vars)} + + # overwrite spec vars with vars from settings + for key, value in dict_settings_env_vars.items(): + dict_spec_env_vars[key] = value + + # returns a single list of vars + return [f"{k}={v}" for k, v in dict_spec_env_vars.items()] + + +def _inject_backend_networking( + parsed_compose_spec: Dict[str, Any], network_name: str = "__backend__" +) -> None: + """ + Put all containers in the compose spec in the same network. + The `network_name` must only be unique inside the user defined spec; + docker-compose will add some prefix to it. + """ + + networks = parsed_compose_spec.get("networks", {}) + networks[network_name] = None + + for service_content in parsed_compose_spec["services"].values(): + service_networks = service_content.get("networks", []) + service_networks.append(network_name) + service_content["networks"] = service_networks + + parsed_compose_spec["networks"] = networks + + +async def validate_compose_spec( + settings: DynamicSidecarSettings, compose_file_content: str, command_timeout: float +) -> str: + """ + Validates what looks like a docker compose spec and injects + additional data to mainly make sure: + - no collisions occur between container names + - containers are located on the same docker network + - properly target environment variables formwarded via + settings on the service + """ + + try: + parsed_compose_spec = yaml.safe_load(compose_file_content) + except yaml.YAMLError as e: + raise InvalidComposeSpec( + f"{str(e)}\n{compose_file_content}\nProvided yaml is not valid!" + ) from e + + if parsed_compose_spec is None or not isinstance(parsed_compose_spec, dict): + raise InvalidComposeSpec(f"{compose_file_content}\nProvided yaml is not valid!") + + if not {"version", "services"}.issubset(set(parsed_compose_spec.keys())): + raise InvalidComposeSpec(f"{compose_file_content}\nProvided yaml is not valid!") + + version = parsed_compose_spec["version"] + if version.startswith("1"): + raise InvalidComposeSpec(f"Provided spec version '{version}' is not supported") + + spec_services_to_container_name: Dict[str, str] = {} + + spec_services = parsed_compose_spec["services"] + for index, service in enumerate(spec_services): + service_content = spec_services[service] + + # assemble and inject the container name + user_given_container_name = service_content.get("container_name", "") + container_name = _assemble_container_name( + settings, service, user_given_container_name, index + ) + service_content["container_name"] = container_name + spec_services_to_container_name[service] = container_name + + # inject forwarded environment variables + environment_entries = service_content.get("environment", []) + service_settings_env_vars = _get_forwarded_env_vars(service) + service_content["environment"] = _merge_env_vars( + compose_spec_env_vars=environment_entries, + settings_env_vars=service_settings_env_vars, + ) + + # if more then one container is defined, add an "backend" network + if len(spec_services) > 1: + _inject_backend_networking(parsed_compose_spec) + + # replace service_key with the container_name int the dict + for service_key in list(spec_services.keys()): + container_name_service_key = spec_services_to_container_name[service_key] + service_data = spec_services.pop(service_key) + + depends_on = service_data.get("depends_on", None) + if depends_on is not None: + service_data["depends_on"] = [ + # replaces with the container name + # if not found it leaves the old value + spec_services_to_container_name.get(x, x) + for x in depends_on + ] + + spec_services[container_name_service_key] = service_data + + # transform back to string and return + validated_compose_file_content = yaml.safe_dump(parsed_compose_spec) + + compose_spec = _apply_templating_directives( + stringified_compose_spec=validated_compose_file_content, + services=spec_services, + spec_services_to_container_name=spec_services_to_container_name, + ) + + # validate against docker-compose config + + command = "docker-compose --file {file_path} config" + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=compose_spec, + command=command, + command_timeout=command_timeout, + ) + if not finished_without_errors: + raise InvalidComposeSpec(f"docker-compose config validation failed\n{stdout}") + + return compose_spec From 8e9b4cfb5c39745298687f3a8e3f595320f2b972 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 22 Apr 2021 08:49:54 +0200 Subject: [PATCH 056/102] moved settings and shared state to dependencies --- .../api/compose.py | 35 +++++++++++-------- .../api/containers.py | 20 +++++------ .../dependencies.py | 12 +++++++ 3 files changed, 42 insertions(+), 25 deletions(-) create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index 5619e9bdb45..84301ad07f6 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -2,10 +2,11 @@ import traceback from typing import Optional -from fastapi import APIRouter, Request, Response +from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import PlainTextResponse from starlette.status import HTTP_200_OK, HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST +from ..dependencies import get_settings, get_shared_store from ..settings import DynamicSidecarSettings from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command from ..shared_store import SharedStore @@ -20,14 +21,15 @@ "/compose:store", response_class=PlainTextResponse, responses={204: {"model": None}} ) async def validates_docker_compose_spec_and_stores_it( - request: Request, response: Response, command_timeout: float = 5.0 + request: Request, + response: Response, + command_timeout: float = 5.0, + settings: DynamicSidecarSettings = Depends(get_settings), + shared_store: SharedStore = Depends(get_shared_store), ) -> Optional[str]: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ body_as_text = (await request.body()).decode("utf-8") - settings: DynamicSidecarSettings = request.app.state.settings - shared_store: SharedStore = request.app.state.shared_store - if body_as_text is None: shared_store.compose_spec = None shared_store.container_names = [] @@ -53,11 +55,12 @@ async def validates_docker_compose_spec_and_stores_it( @compose_router.post("/compose", response_class=PlainTextResponse) async def runs_docker_compose_up( - request: Request, response: Response, command_timeout: float + response: Response, + command_timeout: float, + settings: DynamicSidecarSettings = Depends(get_settings), + shared_store: SharedStore = Depends(get_shared_store), ) -> str: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - settings: DynamicSidecarSettings = request.app.state.settings - shared_store: SharedStore = request.app.state.shared_store # --no-build might be a security risk building is disabled command = ( @@ -79,11 +82,12 @@ async def runs_docker_compose_up( @compose_router.get("/compose:pull", response_class=PlainTextResponse) async def runs_docker_compose_pull( - request: Request, response: Response, command_timeout: float + response: Response, + command_timeout: float, + settings: DynamicSidecarSettings = Depends(get_settings), + shared_store: SharedStore = Depends(get_shared_store), ) -> str: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - shared_store: SharedStore = request.app.state.shared_store - settings: DynamicSidecarSettings = request.app.state.settings stored_compose_content = shared_store.compose_spec if stored_compose_content is None: @@ -117,13 +121,16 @@ async def runs_docker_compose_pull( @compose_router.delete("/compose", response_class=PlainTextResponse) async def runs_docker_compose_down( - request: Request, response: Response, command_timeout: float + response: Response, + command_timeout: float, + settings: DynamicSidecarSettings = Depends(get_settings), + shared_store: SharedStore = Depends(get_shared_store), ) -> str: """Removes the previously started service and returns the docker-compose output""" finished_without_errors, stdout = await remove_the_compose_spec( - shared_store=request.app.state.shared_store, - settings=request.app.state.settings, + shared_store=shared_store, + settings=settings, command_timeout=command_timeout, ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index aa785ea114b..3f0473579d6 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -2,9 +2,10 @@ from typing import Any, Dict, List, Union import aiodocker -from fastapi import APIRouter, Query, Request, Response +from fastapi import APIRouter, Depends, Query, Request, Response from starlette.status import HTTP_400_BAD_REQUEST +from ..dependencies import get_shared_store from ..shared_store import SharedStore containers_router = APIRouter(tags=["containers"]) @@ -18,11 +19,12 @@ async def get_spawned_container_names(request: Request) -> List[str]: @containers_router.get("/containers:inspect") -async def containers_inspect(request: Request, response: Response) -> Dict[str, Any]: +async def containers_inspect( + response: Response, shared_store: SharedStore = Depends(get_shared_store) +) -> Dict[str, Any]: """ Returns information about the container, like docker inspect command """ docker = aiodocker.Docker() - shared_store: SharedStore = request.app.state.shared_store container_names = ( shared_store.container_names if shared_store.container_names else {} ) @@ -42,7 +44,7 @@ async def containers_inspect(request: Request, response: Response) -> Dict[str, @containers_router.get("/containers:docker-status") async def containers_docker_status( - request: Request, response: Response + response: Response, shared_store: SharedStore = Depends(get_shared_store) ) -> Dict[str, Any]: """ Returns the status of the containers """ @@ -51,7 +53,6 @@ def assemble_entry(status: str, error: str = "") -> Dict[str, str]: docker = aiodocker.Docker() - shared_store = request.app.state.shared_store container_names = ( shared_store.container_names if shared_store.container_names else {} ) @@ -84,7 +85,6 @@ def assemble_entry(status: str, error: str = "") -> Dict[str, str]: @containers_router.get("/containers/{id}/logs") async def get_container_logs( # pylint: disable=unused-argument - request: Request, response: Response, id: str, since: int = Query( @@ -102,11 +102,11 @@ async def get_container_logs( title="Display timestamps", description="Enabling this parameter will include timestamps in logs", ), + shared_store: SharedStore = Depends(get_shared_store), ) -> Union[str, Dict[str, Any]]: """ Returns the logs of a given container if found """ # TODO: remove from here and dump directly into the logs of this service # do this in PR#1887 - shared_store: SharedStore = request.app.state.shared_store if id not in shared_store.container_names: response.status_code = HTTP_400_BAD_REQUEST @@ -130,10 +130,9 @@ async def get_container_logs( @containers_router.get("/containers/{id}/inspect") async def inspect_container( - request: Request, response: Response, id: str + response: Response, id: str, shared_store: SharedStore = Depends(get_shared_store) ) -> Dict[str, Any]: """ Returns information about the container, like docker inspect command """ - shared_store: SharedStore = request.app.state.shared_store if id not in shared_store.container_names: response.status_code = HTTP_400_BAD_REQUEST @@ -152,9 +151,8 @@ async def inspect_container( @containers_router.delete("/containers/{id}/remove") async def remove_container( - request: Request, response: Response, id: str + response: Response, id: str, shared_store: SharedStore = Depends(get_shared_store) ) -> Union[bool, Dict[str, Any]]: - shared_store: SharedStore = request.app.state.shared_store if id not in shared_store.container_names: response.status_code = HTTP_400_BAD_REQUEST diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py new file mode 100644 index 00000000000..e764a2f8536 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py @@ -0,0 +1,12 @@ +from fastapi import Request + +from .settings import DynamicSidecarSettings +from .shared_store import SharedStore + + +def get_settings(request: Request) -> DynamicSidecarSettings: + return request.app.state.settings # type: ignore + + +def get_shared_store(request: Request) -> SharedStore: + return request.app.state.shared_store # type: ignore From 35d249037efe94f3009f48066ab6edfd8d1b7080 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 22 Apr 2021 09:20:11 +0200 Subject: [PATCH 057/102] minor improvments --- .../simcore_service_dynamic_sidecar/validation.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py index cd3ea2b1306..a71ce777b13 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py @@ -33,8 +33,9 @@ def _assemble_container_name( container_name = "-".join([x for x in strings_to_use if len(x) > 0])[ : settings.max_combined_container_name_length - ] - return container_name.replace("_", "-") + ].replace("_", "-") + + return container_name def _get_forwarded_env_vars(container_key: str) -> List[str]: @@ -153,6 +154,8 @@ async def validate_compose_spec( - containers are located on the same docker network - properly target environment variables formwarded via settings on the service + + Finally runs docker-compose config to properly validate the result """ try: @@ -233,6 +236,10 @@ async def validate_compose_spec( command_timeout=command_timeout, ) if not finished_without_errors: - raise InvalidComposeSpec(f"docker-compose config validation failed\n{stdout}") + message = ( + f"'docker-compose config' failed for:\n{compose_spec}\nSTDOUT\n{stdout}" + ) + logger.warning(message) + raise InvalidComposeSpec(f"filed to run {command}") return compose_spec From e7030ff801d5c8abd9d34d5e2bd713d1bfd22bf5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 22 Apr 2021 09:39:27 +0200 Subject: [PATCH 058/102] updated requirements --- services/dynamic-sidecar/requirements/_base.txt | 12 ++++-------- services/dynamic-sidecar/requirements/_test.txt | 8 +++----- services/dynamic-sidecar/requirements/_tools.txt | 10 +++++----- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index 3a19f47282e..8ba0ccccfa7 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -49,13 +49,13 @@ distro==1.5.0 # via docker-compose dnspython==2.1.0 # via email-validator -docker-compose==1.27.4 +docker-compose==1.29.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in -docker[ssh]==4.4.4 +docker[ssh]==5.0.0 # via docker-compose dockerpty==0.4.1 # via docker-compose @@ -73,14 +73,11 @@ idna-ssl==1.1.0 # via aiohttp idna==2.10 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # email-validator # idna-ssl # requests # yarl -importlib-metadata==3.10.1 +importlib-metadata==4.0.1 # via # jsonschema # sqlalchemy @@ -121,12 +118,11 @@ requests==2.25.1 six==1.15.0 # via # bcrypt - # docker # dockerpty # jsonschema # pynacl # websocket-client -sqlalchemy[postgresql_psycopg2binary]==1.4.7 +sqlalchemy[postgresql_psycopg2binary]==1.4.11 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt diff --git a/services/dynamic-sidecar/requirements/_test.txt b/services/dynamic-sidecar/requirements/_test.txt index 0b841a507fe..3efa909d812 100644 --- a/services/dynamic-sidecar/requirements/_test.txt +++ b/services/dynamic-sidecar/requirements/_test.txt @@ -17,10 +17,8 @@ coverage==5.5 faker==8.1.0 # via -r requirements/_test.in idna==2.10 - # via - # -c requirements/../../../requirements/constraints.txt - # requests -importlib-metadata==3.10.1 + # via requests +importlib-metadata==4.0.1 # via # pluggy # pytest @@ -36,7 +34,7 @@ py==1.10.0 # via pytest pyparsing==2.4.7 # via packaging -pytest-asyncio==0.14.0 +pytest-asyncio==0.15.1 # via -r requirements/_test.in pytest-cov==2.11.1 # via -r requirements/_test.in diff --git a/services/dynamic-sidecar/requirements/_tools.txt b/services/dynamic-sidecar/requirements/_tools.txt index af67058507f..9a446d986ef 100644 --- a/services/dynamic-sidecar/requirements/_tools.txt +++ b/services/dynamic-sidecar/requirements/_tools.txt @@ -31,9 +31,9 @@ distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv -identify==2.2.3 +identify==2.2.4 # via pre-commit -importlib-metadata==3.10.1 +importlib-metadata==4.0.1 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -65,9 +65,9 @@ pathspec==0.8.1 # via black pep517==0.10.0 # via pip-tools -pip-tools==6.0.1 +pip-tools==6.1.0 # via -r requirements/../../../requirements/devenv.txt -pre-commit==2.12.0 +pre-commit==2.12.1 # via -r requirements/../../../requirements/devenv.txt pylint==2.7.4 # via -r requirements/_tools.in @@ -102,7 +102,7 @@ typing-extensions==3.7.4.3 # black # importlib-metadata # mypy -virtualenv==20.4.3 +virtualenv==20.4.4 # via pre-commit wrapt==1.12.1 # via astroid From 8d4838b9e46497ea9f8d5268ef094a9ba01fba0d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 22 Apr 2021 10:31:38 +0200 Subject: [PATCH 059/102] added missing service to SERVICES_LIST --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index a0563b02a4c..b6b5f45611c 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ SERVICES_LIST := \ catalog \ director \ director-v2 \ + dynamic-sidecar \ migration \ sidecar \ storage \ From 46fb13137db438016bb272b828634a2991194f2a Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Thu, 22 Apr 2021 15:23:03 +0200 Subject: [PATCH 060/102] Some suggestions (#3) from sanderegg * naming * consistency * shellcheck --- .github/workflows/ci-testing-deploy.yml | 2 +- scripts/act.bash | 11 +++--- scripts/codestyle.bash | 24 ++++++------- services/dynamic-sidecar/docker/entrypoint.sh | 34 +++++++++---------- 4 files changed, 33 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index dccc4ab42fa..56fc563747c 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -344,7 +344,7 @@ jobs: ${{ runner.os }}- - name: install run: ./ci/github/unit-testing/dynamic-sidecar.bash install - - name: codestyle-ci + - name: codestyle run: ./ci/github/unit-testing/dynamic-sidecar.bash codestyle - name: test run: ./ci/github/unit-testing/dynamic-sidecar.bash test diff --git a/scripts/act.bash b/scripts/act.bash index e61fd25e6a2..8a9bdadeb21 100755 --- a/scripts/act.bash +++ b/scripts/act.bash @@ -10,13 +10,11 @@ ROOT_PROJECT_DIR=$1 # name of the job, usually defined in the .github/workflows/ci-testing-deploy.yaml JOB_TO_RUN=$2 - DOCKER_IMAGE_NAME=dind-act-runner ACT_RUNNER=ubuntu-20.04=catthehacker/ubuntu:act-20.04 ACT_VERSION_TAG=v0.2.20 # from https://github.com/nektos/act/releases - -docker build -t $DOCKER_IMAGE_NAME -< /dev/null 2>&1 -then - echo "$INFO detected docker socket is mounted, adding user to group..." - GROUPID=$(stat --format=%g $DOCKER_MOUNT) - GROUPNAME=scdocker +if stat $DOCKER_MOUNT >/dev/null 2>&1; then + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker - if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 - then - echo "$WARNING docker group with $GROUPID already exists, getting group name..." - # if group already exists in container, then reuse name - GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) - echo "$WARNING docker group with $GROUPID has name $GROUPNAME" - fi - adduser "$SC_USER_NAME" "$GROUPNAME" + if ! addgroup --gid "$GROUPID" $GROUPNAME >/dev/null 2>&1; then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." + # if group already exists in container, then reuse name + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" + fi + adduser "$SC_USER_NAME" "$GROUPNAME" fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" From 25a952caea2c5f509a82997867d54bdc6987cddc Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 22 Apr 2021 17:27:02 +0200 Subject: [PATCH 061/102] proposed refactoring on dynamic sidecar (#4) * removes responses and raise exceptions for errors * Minor typo * Should include standard entrypoint * refactor health and added dependencies * minor * fixes --- services/dynamic-sidecar/setup.py | 2 + .../api/compose.py | 11 +- .../api/containers.py | 154 +++++++++++------- .../api/health.py | 26 ++- .../dependencies.py | 15 +- .../shared_store.py | 2 +- .../tests/unit/test_api_containers.py | 6 +- 7 files changed, 138 insertions(+), 78 deletions(-) diff --git a/services/dynamic-sidecar/setup.py b/services/dynamic-sidecar/setup.py index b04ec058b98..486343f80c2 100644 --- a/services/dynamic-sidecar/setup.py +++ b/services/dynamic-sidecar/setup.py @@ -36,6 +36,8 @@ def read_reqs(reqs_path: Path): setup_requires=["setuptools_scm"], entry_points={ "console_scripts": [ + "simcore-service-dynamic-sidecar=simcore_service_dynamic_sidecar.main:main", + # alternative entry-points "simcore_service_dynamic_sidecar_startup = simcore_service_dynamic_sidecar.main:main", ], }, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index 84301ad07f6..0c65ac1da92 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -27,6 +27,8 @@ async def validates_docker_compose_spec_and_stores_it( settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), ) -> Optional[str]: + # FIXME: why is this not an argument into CREATE /containers ? + """ Expects the docker-compose spec as raw-body utf-8 encoded text """ body_as_text = (await request.body()).decode("utf-8") @@ -61,6 +63,7 @@ async def runs_docker_compose_up( shared_store: SharedStore = Depends(get_shared_store), ) -> str: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ + # FIXME: why is this not in /containers:up ? # --no-build might be a security risk building is disabled command = ( @@ -88,7 +91,7 @@ async def runs_docker_compose_pull( shared_store: SharedStore = Depends(get_shared_store), ) -> str: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - + # FIXME: why is this not in /containers:pull ? stored_compose_content = shared_store.compose_spec if stored_compose_content is None: response.status_code = HTTP_400_BAD_REQUEST @@ -101,7 +104,7 @@ async def runs_docker_compose_pull( try: # mark as pulling images - shared_store.is_pulling_containsers = True + shared_store.is_pulling_containers = True finished_without_errors, stdout = await write_file_and_run_command( settings=settings, @@ -111,7 +114,7 @@ async def runs_docker_compose_pull( ) finally: # remove mark - shared_store.is_pulling_containsers = False + shared_store.is_pulling_containers = False response.status_code = ( HTTP_200_OK if finished_without_errors else HTTP_400_BAD_REQUEST @@ -126,6 +129,8 @@ async def runs_docker_compose_down( settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), ) -> str: + # FIXME: why is this not in /containers:down ? + """Removes the previously started service and returns the docker-compose output""" finished_without_errors, stdout = await remove_the_compose_spec( diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 3f0473579d6..e096cbbbaf3 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -2,8 +2,7 @@ from typing import Any, Dict, List, Union import aiodocker -from fastapi import APIRouter, Depends, Query, Request, Response -from starlette.status import HTTP_400_BAD_REQUEST +from fastapi import APIRouter, Depends, HTTPException, Query, status from ..dependencies import get_shared_store from ..shared_store import SharedStore @@ -12,55 +11,57 @@ @containers_router.get("/containers") -async def get_spawned_container_names(request: Request) -> List[str]: +async def get_spawned_container_names( + shared_store: SharedStore = Depends(get_shared_store), +) -> List[str]: """ Returns a list of containers created using docker-compose """ - shared_store: SharedStore = request.app.state.shared_store return shared_store.container_names -@containers_router.get("/containers:inspect") +@containers_router.get( + "/containers:inspect", + responses={status.HTTP_400_BAD_REQUEST: {"description": "Erros in container"}}, +) async def containers_inspect( - response: Response, shared_store: SharedStore = Depends(get_shared_store) + shared_store: SharedStore = Depends(get_shared_store), ) -> Dict[str, Any]: """ Returns information about the container, like docker inspect command """ docker = aiodocker.Docker() - container_names = ( - shared_store.container_names if shared_store.container_names else {} - ) - results = {} - for container in container_names: + for container in shared_store.container_names: try: container_instance = await docker.containers.get(container) results[container] = await container_instance.show() - except aiodocker.exceptions.DockerError as e: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=e.message) + except aiodocker.exceptions.DockerError as err: + # FIXME: This is NOT a client error https://httpstatuses.com/400. More or a server problem? 5XX?? + + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + detail=err.message, + ) from err return results -@containers_router.get("/containers:docker-status") +@containers_router.get( + "/containers/status", + responses={status.HTTP_400_BAD_REQUEST: {"description": "Errors in container"}}, +) async def containers_docker_status( - response: Response, shared_store: SharedStore = Depends(get_shared_store) + shared_store: SharedStore = Depends(get_shared_store), ) -> Dict[str, Any]: """ Returns the status of the containers """ - def assemble_entry(status: str, error: str = "") -> Dict[str, str]: - return {"Status": status, "Error": error} - docker = aiodocker.Docker() - container_names = ( - shared_store.container_names if shared_store.container_names else {} - ) + container_names = shared_store.container_names # if containers are being pulled, return pulling (fake status) - if shared_store.is_pulling_containsers: + if shared_store.is_pulling_containers: # pulling is a fake state use to share more information with the frontend - return {x: assemble_entry(status="pulling") for x in container_names} + return {x: {"Status": "pulling", "Error": ""} for x in container_names} results = {} @@ -75,26 +76,32 @@ def assemble_entry(status: str, error: str = "") -> Dict[str, str]: "Status": container_state.get("Status", "pending"), "Error": container_state.get("Error", ""), } - except aiodocker.exceptions.DockerError as e: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=e.message) + except aiodocker.exceptions.DockerError as err: + # FIXME: This is NOT a client error https://httpstatuses.com/400. More or a server problem? 5XX?? + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + detail=err.message, + ) from err return results -@containers_router.get("/containers/{id}/logs") +@containers_router.get( + "/containers/{id}/logs", + responses={ + status.HTTP_400_BAD_REQUEST: {"description": "Container does not exists"} + }, +) async def get_container_logs( - # pylint: disable=unused-argument - response: Response, id: str, since: int = Query( 0, - title="Timstamp", + title="Timestamp", description="Only return logs since this time, as a UNIX timestamp", ), until: int = Query( 0, - title="Timstamp", + title="Timestamp", description="Only return logs before this time, as a UNIX timestamp", ), timestamps: bool = Query( @@ -107,13 +114,13 @@ async def get_container_logs( """ Returns the logs of a given container if found """ # TODO: remove from here and dump directly into the logs of this service # do this in PR#1887 - if id not in shared_store.container_names: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=f"No container '{id}' was started") + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + detail=f"No container '{id}' was started", + ) docker = aiodocker.Docker() - try: container_instance = await docker.containers.get(id) @@ -121,22 +128,39 @@ async def get_container_logs( if timestamps: args["timestamps"] = True + if since or until: + raise HTTPException( + status.HTTP_501_NOT_IMPLEMENTED, + detail="since and until options are still not implemented", + ) + container_logs: str = await container_instance.log(**args) return container_logs - except aiodocker.exceptions.DockerError as e: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=e.message) + except aiodocker.exceptions.DockerError as err: + # FIXME: This is NOT a client error https://httpstatuses.com/400. More or a server problem? 5XX?? + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + detail=err.message, + ) from err -@containers_router.get("/containers/{id}/inspect") + +@containers_router.get( + "/containers/{id}:inspect", + responses={ + status.HTTP_400_BAD_REQUEST: {"description": "Container does not exist"} + }, +) async def inspect_container( - response: Response, id: str, shared_store: SharedStore = Depends(get_shared_store) + id: str, shared_store: SharedStore = Depends(get_shared_store) ) -> Dict[str, Any]: """ Returns information about the container, like docker inspect command """ if id not in shared_store.container_names: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=f"No container '{id}' was started") + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + detail=f"No container '{id}' was started", + ) docker = aiodocker.Docker() @@ -144,29 +168,43 @@ async def inspect_container( container_instance = await docker.containers.get(id) inspect_result: Dict[str, Any] = await container_instance.show() return inspect_result - except aiodocker.exceptions.DockerError as e: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=e.message) - - -@containers_router.delete("/containers/{id}/remove") + except aiodocker.exceptions.DockerError as err: + # FIXME: This is NOT a client error https://httpstatuses.com/400. More or a server problem? 5XX?? + + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + detail=f"No container '{id}' was started", + ) from err + + +@containers_router.delete( + "/containers/{id}", + status_code=status.HTTP_204_NO_CONTENT, + responses={ + status.HTTP_400_BAD_REQUEST: {"description": "Container does not exist"} + }, +) async def remove_container( - response: Response, id: str, shared_store: SharedStore = Depends(get_shared_store) -) -> Union[bool, Dict[str, Any]]: - + id: str, shared_store: SharedStore = Depends(get_shared_store) +): if id not in shared_store.container_names: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=f"No container '{id}' was started") + raise HTTPException( + status.HTTP_400_BAD_REQUEST, detail=f"No container '{id}' was started" + ) docker = aiodocker.Docker() try: container_instance = await docker.containers.get(id) await container_instance.delete() - return True - except aiodocker.exceptions.DockerError as e: - response.status_code = HTTP_400_BAD_REQUEST - return dict(error=e.message) + + except aiodocker.exceptions.DockerError as err: + # FIXME: This is NOT a client error https://httpstatuses.com/400. More or a server problem? 5XX?? + + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + detail=err.message, + ) from err __all__ = ["containers_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py index 429727cc011..28e2b9741f6 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py @@ -1,17 +1,27 @@ -from fastapi import APIRouter, Request, Response -from starlette.status import HTTP_400_BAD_REQUEST +from fastapi import APIRouter, Depends, HTTPException, status +from ..dependencies import State, get_app_state from ..models import ApplicationHealth health_router = APIRouter() -@health_router.get("/health", response_model=ApplicationHealth) -async def health_endpoint(request: Request, response: Response) -> ApplicationHealth: - application_health: ApplicationHealth = request.app.state.application_health - - if application_health.is_healthy is False: - response.status_code = HTTP_400_BAD_REQUEST +@health_router.get( + "/health", + response_model=ApplicationHealth, + responses={ + status.HTTP_503_SERVICE_UNAVAILABLE: {"description": "Service is unhealthy"} + }, +) +async def health_endpoint( + app_state: State = Depends(get_app_state), +): + application_health: ApplicationHealth = app_state.application_health + + if not application_health.is_healthy: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, detail="Marked as unhealthy" + ) return application_health diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py index e764a2f8536..3bf519ff6cf 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py @@ -1,12 +1,17 @@ -from fastapi import Request +from fastapi import Depends, Request +from fastapi.datastructures import State from .settings import DynamicSidecarSettings from .shared_store import SharedStore -def get_settings(request: Request) -> DynamicSidecarSettings: - return request.app.state.settings # type: ignore +def get_app_state(request: Request) -> State: + return request.app.state # type: ignore -def get_shared_store(request: Request) -> SharedStore: - return request.app.state.shared_store # type: ignore +def get_settings(app_state: State = Depends(get_app_state)) -> DynamicSidecarSettings: + return app_state.settings # type: ignore + + +def get_shared_store(app_state: State = Depends(get_app_state)) -> SharedStore: + return app_state.shared_store # type: ignore diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py index c6707b3fc4f..ebf5af7c4ff 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py @@ -10,6 +10,6 @@ class SharedStore(BaseModel): container_names: List[str] = Field( [], description="stores the container names from the compose_spec" ) - is_pulling_containsers: bool = Field( + is_pulling_containers: bool = Field( False, description="set to True while the containers are being pulled" ) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 7f661f40b12..e60fe12e07b 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -108,15 +108,15 @@ async def test_containers_docker_status_pulling_containers( @contextmanager def mark_pulling(shared_store: SharedStore) -> Generator[None, None, None]: try: - shared_store.is_pulling_containsers = True + shared_store.is_pulling_containers = True yield finally: - shared_store.is_pulling_containsers = False + shared_store.is_pulling_containers = False shared_store: SharedStore = test_client.application.state.shared_store with mark_pulling(shared_store): - assert shared_store.is_pulling_containsers is True + assert shared_store.is_pulling_containers is True response = await test_client.get( f"/{api_vtag}/containers:docker-status", From 9bf1ef93e192cb22eca9707325a0317c0016d25d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 07:50:50 +0200 Subject: [PATCH 062/102] fixed codestyle --- .../src/simcore_service_dynamic_sidecar/api/containers.py | 2 +- .../src/simcore_service_dynamic_sidecar/api/health.py | 8 +++----- .../src/simcore_service_dynamic_sidecar/dependencies.py | 7 +++++++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index e096cbbbaf3..d5d19ccedf1 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -186,7 +186,7 @@ async def inspect_container( ) async def remove_container( id: str, shared_store: SharedStore = Depends(get_shared_store) -): +) -> None: if id not in shared_store.container_names: raise HTTPException( status.HTTP_400_BAD_REQUEST, detail=f"No container '{id}' was started" diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py index 28e2b9741f6..de0c6254de6 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, Depends, HTTPException, status -from ..dependencies import State, get_app_state +from ..dependencies import get_application_health from ..models import ApplicationHealth health_router = APIRouter() @@ -14,10 +14,8 @@ }, ) async def health_endpoint( - app_state: State = Depends(get_app_state), -): - application_health: ApplicationHealth = app_state.application_health - + application_health: ApplicationHealth = Depends(get_application_health), +) -> ApplicationHealth: if not application_health.is_healthy: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, detail="Marked as unhealthy" diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py index 3bf519ff6cf..d5b35a545fb 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py @@ -1,6 +1,7 @@ from fastapi import Depends, Request from fastapi.datastructures import State +from .models import ApplicationHealth from .settings import DynamicSidecarSettings from .shared_store import SharedStore @@ -9,6 +10,12 @@ def get_app_state(request: Request) -> State: return request.app.state # type: ignore +def get_application_health( + app_state: State = Depends(get_app_state), +) -> ApplicationHealth: + return app_state.application_health # type: ignore + + def get_settings(app_state: State = Depends(get_app_state)) -> DynamicSidecarSettings: return app_state.settings # type: ignore From 0cc50a4ebe5ca8a7526208336993e177e4f0f687 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 08:31:05 +0200 Subject: [PATCH 063/102] fixed typos --- scripts/codestyle.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/codestyle.bash b/scripts/codestyle.bash index 6493e3e5cd4..bc24fcf4dda 100755 --- a/scripts/codestyle.bash +++ b/scripts/codestyle.bash @@ -10,7 +10,7 @@ BASE_PATH_DIR=${3-MISSING_DIR} # used for development (fails on pylint and mypy) development() { - echo "enforcing codestyle to soruce_directory=$SRC_DIRECTORY_NAME" + echo "enforcing codestyle to source_directory=$SRC_DIRECTORY_NAME" echo "isort" isort setup.py src/"$SRC_DIRECTORY_NAME" tests echo "black" @@ -23,7 +23,7 @@ development() { # invoked by ci as test (also fails on isort and black) ci() { - echo "checking codestyle in service=$BASE_PATH_DIR with soruce_directory=$SRC_DIRECTORY_NAME" + echo "checking codestyle in service=$BASE_PATH_DIR with source_directory=$SRC_DIRECTORY_NAME" echo "isort" isort --check setup.py "$BASE_PATH_DIR"/src/"$SRC_DIRECTORY_NAME" "$BASE_PATH_DIR"/tests echo "black" From 1442c8d5b4f97590c6f98e4bfd1db10c4f6ce3f1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 08:31:44 +0200 Subject: [PATCH 064/102] fixed compose API entrypoints and tests --- .../api/compose.py | 71 +++++-------- .../tests/unit/test_api_compose.py | 100 ++++++++++-------- 2 files changed, 85 insertions(+), 86 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index 0c65ac1da92..0c538d45f53 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -1,10 +1,9 @@ import logging import traceback -from typing import Optional -from fastapi import APIRouter, Depends, Request, Response +from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import PlainTextResponse -from starlette.status import HTTP_200_OK, HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST +from starlette.status import HTTP_200_OK, HTTP_500_INTERNAL_SERVER_ERROR from ..dependencies import get_settings, get_shared_store from ..settings import DynamicSidecarSettings @@ -17,25 +16,18 @@ compose_router = APIRouter(tags=["docker-compose"]) -@compose_router.post( - "/compose:store", response_class=PlainTextResponse, responses={204: {"model": None}} -) -async def validates_docker_compose_spec_and_stores_it( +@compose_router.post("/containers:up", response_class=PlainTextResponse) +async def runs_docker_compose_up( request: Request, response: Response, - command_timeout: float = 5.0, + command_timeout: float, settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), -) -> Optional[str]: - # FIXME: why is this not an argument into CREATE /containers ? - +) -> str: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - body_as_text = (await request.body()).decode("utf-8") - if body_as_text is None: - shared_store.compose_spec = None - shared_store.container_names = [] - return None + # stores the compose spec after validation + body_as_text = (await request.body()).decode("utf-8") try: shared_store.compose_spec = await validate_compose_spec( @@ -48,23 +40,9 @@ async def validates_docker_compose_spec_and_stores_it( ) except InvalidComposeSpec as e: logger.warning("Error detected %s", traceback.format_exc()) - response.status_code = HTTP_400_BAD_REQUEST - return str(e) - - response.status_code = HTTP_204_NO_CONTENT - return None - - -@compose_router.post("/compose", response_class=PlainTextResponse) -async def runs_docker_compose_up( - response: Response, - command_timeout: float, - settings: DynamicSidecarSettings = Depends(get_settings), - shared_store: SharedStore = Depends(get_shared_store), -) -> str: - """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - # FIXME: why is this not in /containers:up ? + raise HTTPException(HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e + # create the compose spec # --no-build might be a security risk building is disabled command = ( "docker-compose --project-name {project} --file {file_path} " @@ -78,24 +56,25 @@ async def runs_docker_compose_up( ) response.status_code = ( - HTTP_200_OK if finished_without_errors else HTTP_400_BAD_REQUEST + HTTP_200_OK if finished_without_errors else HTTP_500_INTERNAL_SERVER_ERROR ) return stdout -@compose_router.get("/compose:pull", response_class=PlainTextResponse) +@compose_router.get("/containers:pull", response_class=PlainTextResponse) async def runs_docker_compose_pull( response: Response, command_timeout: float, settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), ) -> str: - """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - # FIXME: why is this not in /containers:pull ? + stored_compose_content = shared_store.compose_spec if stored_compose_content is None: - response.status_code = HTTP_400_BAD_REQUEST - return "No started spec to pull was found" + raise HTTPException( + HTTP_500_INTERNAL_SERVER_ERROR, + detail="No spec for docker-compose pull was found", + ) command = ( "docker-compose --project-name {project} --file {file_path} " @@ -117,22 +96,28 @@ async def runs_docker_compose_pull( shared_store.is_pulling_containers = False response.status_code = ( - HTTP_200_OK if finished_without_errors else HTTP_400_BAD_REQUEST + HTTP_200_OK if finished_without_errors else HTTP_500_INTERNAL_SERVER_ERROR ) return stdout -@compose_router.delete("/compose", response_class=PlainTextResponse) +@compose_router.post("/containers:down", response_class=PlainTextResponse) async def runs_docker_compose_down( response: Response, command_timeout: float, settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), ) -> str: - # FIXME: why is this not in /containers:down ? - """Removes the previously started service and returns the docker-compose output""" + + stored_compose_content = shared_store.compose_spec + if stored_compose_content is None: + raise HTTPException( + HTTP_500_INTERNAL_SERVER_ERROR, + detail="No spec for docker-compose down was found", + ) + finished_without_errors, stdout = await remove_the_compose_spec( shared_store=shared_store, settings=settings, @@ -140,7 +125,7 @@ async def runs_docker_compose_down( ) response.status_code = ( - HTTP_200_OK if finished_without_errors else HTTP_400_BAD_REQUEST + HTTP_200_OK if finished_without_errors else HTTP_500_INTERNAL_SERVER_ERROR ) return stdout diff --git a/services/dynamic-sidecar/tests/unit/test_api_compose.py b/services/dynamic-sidecar/tests/unit/test_api_compose.py index 6be4fcfc2c2..4f647ef86ce 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_compose.py +++ b/services/dynamic-sidecar/tests/unit/test_api_compose.py @@ -8,6 +8,7 @@ from async_asgi_testclient import TestClient from faker import Faker from simcore_service_dynamic_sidecar._meta import api_vtag +from starlette.status import HTTP_200_OK, HTTP_500_INTERNAL_SERVER_ERROR DEFAULT_COMMAND_TIMEOUT = 10.0 @@ -24,89 +25,102 @@ def compose_spec() -> str: ) -async def test_store_compose_spec( +async def test_compose_up( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: - response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) - assert response.status_code == 204, response.text - assert response.text == "" + + response = await test_client.post( + f"/{api_vtag}/containers:up", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), + data=compose_spec, + ) + assert response.status_code == HTTP_200_OK, response.text -async def test_store_compose_spec_not_provided(test_client: TestClient) -> None: - response = await test_client.post(f"/{api_vtag}/compose:store") - assert response.status_code == 400, response.text - assert response.text == "\nProvided yaml is not valid!" +async def test_compose_up_spec_not_provided(test_client: TestClient) -> None: + response = await test_client.post( + f"/{api_vtag}/containers:up", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), + ) + assert response.status_code == HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert json.loads(response.text) == {"detail": "\nProvided yaml is not valid!"} -async def test_store_compose_spec_invalid(test_client: TestClient) -> None: +async def test_compose_up_spec_invalid(test_client: TestClient) -> None: invalid_compose_spec = Faker().text() response = await test_client.post( - f"/{api_vtag}/compose:store", data=invalid_compose_spec + f"/{api_vtag}/containers:up", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), + data=invalid_compose_spec, ) - assert response.status_code == 400, response.text - assert response.text.endswith("\nProvided yaml is not valid!") + assert response.status_code == HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert "Provided yaml is not valid!" in response.text # 28+ characters means the compos spec is also present in the error message assert len(response.text) > 28 -async def test_compose_up( +async def test_containers_pull( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: # store spec first - response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) - assert response.status_code == 204, response.text - assert response.text == "" - - # pull images for spec response = await test_client.post( - f"/{api_vtag}/compose", + f"/{api_vtag}/containers:up", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), + data=compose_spec, ) - assert response.status_code == 200, response.text - - -async def test_pull(test_client: TestClient, compose_spec: Dict[str, Any]) -> None: - # store spec first - response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) - assert response.status_code == 204, response.text - assert response.text == "" + assert response.status_code == HTTP_200_OK, response.text + assert response.text != "" # pull images for spec response = await test_client.get( - f"/{api_vtag}/compose:pull", + f"/{api_vtag}/containers:pull", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) - assert response.status_code == 200, response.text + assert response.status_code == HTTP_200_OK, response.text + assert response.text != "" -async def test_pull_missing_spec( +async def test_containers_pull_missing_spec( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: response = await test_client.get( - f"/{api_vtag}/compose:pull", + f"/{api_vtag}/containers:pull", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) - assert response.status_code == 400, response.text - assert response.text == "No started spec to pull was found" + assert response.status_code == HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert json.loads(response.text) == { + "detail": "No spec for docker-compose pull was found" + } -async def test_compose_delete_after_stopping( +async def test_containers_down_after_stopping( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: # store spec first - response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) - assert response.status_code == 204, response.text - assert response.text == "" + response = await test_client.post( + f"/{api_vtag}/containers:up", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), + data=compose_spec, + ) + assert response.status_code == HTTP_200_OK, response.text + assert response.text != "" - # pull images for spec response = await test_client.post( - f"/{api_vtag}/compose", + f"/{api_vtag}/containers:down", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) - assert response.status_code == 200, response.text + assert response.status_code == HTTP_200_OK, response.text + assert response.text != "" + - response = await test_client.delete( - f"/{api_vtag}/compose", +async def test_containers_down_missing_spec( + test_client: TestClient, compose_spec: Dict[str, Any] +) -> None: + response = await test_client.post( + f"/{api_vtag}/containers:down", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) - assert response.status_code == 200, response.text + assert response.status_code == HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert json.loads(response.text) == { + "detail": "No spec for docker-compose down was found" + } From f04a61ef2bc6f133855b682ae07661d6da0f0814 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 14:06:09 +0200 Subject: [PATCH 065/102] refactored api structure and endpoints --- .../api/compose.py | 119 +++++---- .../api/containers.py | 233 +++++++++--------- .../api/health.py | 2 +- 3 files changed, 175 insertions(+), 179 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py index 0c538d45f53..b8e810609d0 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py @@ -1,9 +1,18 @@ import logging import traceback - -from fastapi import APIRouter, Depends, HTTPException, Request, Response +from typing import Any, Dict, Optional, Union + +from fastapi import ( + APIRouter, + BackgroundTasks, + Depends, + HTTPException, + Query, + Request, + Response, + status, +) from fastapi.responses import PlainTextResponse -from starlette.status import HTTP_200_OK, HTTP_500_INTERNAL_SERVER_ERROR from ..dependencies import get_settings, get_shared_store from ..settings import DynamicSidecarSettings @@ -16,33 +25,11 @@ compose_router = APIRouter(tags=["docker-compose"]) -@compose_router.post("/containers:up", response_class=PlainTextResponse) -async def runs_docker_compose_up( - request: Request, - response: Response, +async def task_docker_compose_up( command_timeout: float, - settings: DynamicSidecarSettings = Depends(get_settings), - shared_store: SharedStore = Depends(get_shared_store), -) -> str: - """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - - # stores the compose spec after validation - body_as_text = (await request.body()).decode("utf-8") - - try: - shared_store.compose_spec = await validate_compose_spec( - settings=settings, - compose_file_content=body_as_text, - command_timeout=command_timeout, - ) - shared_store.container_names = assemble_container_names( - shared_store.compose_spec - ) - except InvalidComposeSpec as e: - logger.warning("Error detected %s", traceback.format_exc()) - raise HTTPException(HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e - - # create the compose spec + settings: DynamicSidecarSettings, + shared_store: SharedStore, +) -> None: # --no-build might be a security risk building is disabled command = ( "docker-compose --project-name {project} --file {file_path} " @@ -54,51 +41,53 @@ async def runs_docker_compose_up( command=command, command_timeout=command_timeout, ) + message = f"Finished {command} with output\n{stdout}" - response.status_code = ( - HTTP_200_OK if finished_without_errors else HTTP_500_INTERNAL_SERVER_ERROR - ) - return stdout + if finished_without_errors: + logger.info(message) + else: + logger.error(message) + return None -@compose_router.get("/containers:pull", response_class=PlainTextResponse) -async def runs_docker_compose_pull( - response: Response, - command_timeout: float, + +@compose_router.post("/containers", status_code=status.HTTP_201_CREATED) +async def runs_docker_compose_up( + request: Request, + background_tasks: BackgroundTasks, + command_timeout: float = Query( + ..., + description=( + "docker-compose up also pulls images, this value " + "needs to be big enough to account for that" + ), + ), settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), -) -> str: - - stored_compose_content = shared_store.compose_spec - if stored_compose_content is None: - raise HTTPException( - HTTP_500_INTERNAL_SERVER_ERROR, - detail="No spec for docker-compose pull was found", - ) +) -> Optional[Dict[str, Any]]: + """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - command = ( - "docker-compose --project-name {project} --file {file_path} " - "pull --include-deps" - ) + # stores the compose spec after validation + body_as_text = (await request.body()).decode("utf-8") try: - # mark as pulling images - shared_store.is_pulling_containers = True - - finished_without_errors, stdout = await write_file_and_run_command( + shared_store.compose_spec = await validate_compose_spec( settings=settings, - file_content=stored_compose_content, - command=command, + compose_file_content=body_as_text, command_timeout=command_timeout, ) - finally: - # remove mark - shared_store.is_pulling_containers = False + shared_store.container_names = assemble_container_names( + shared_store.compose_spec + ) + except InvalidComposeSpec as e: + logger.warning("Error detected %s", traceback.format_exc()) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e - response.status_code = ( - HTTP_200_OK if finished_without_errors else HTTP_500_INTERNAL_SERVER_ERROR + # run docker-compose in a background queue and return early + background_tasks.add_task( + task_docker_compose_up, command_timeout, settings, shared_store ) - return stdout + return None @compose_router.post("/containers:down", response_class=PlainTextResponse) @@ -107,14 +96,14 @@ async def runs_docker_compose_down( command_timeout: float, settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), -) -> str: +) -> Union[str, Dict[str, Any]]: """Removes the previously started service and returns the docker-compose output""" stored_compose_content = shared_store.compose_spec if stored_compose_content is None: raise HTTPException( - HTTP_500_INTERNAL_SERVER_ERROR, + status.HTTP_500_INTERNAL_SERVER_ERROR, detail="No spec for docker-compose down was found", ) @@ -125,7 +114,9 @@ async def runs_docker_compose_down( ) response.status_code = ( - HTTP_200_OK if finished_without_errors else HTTP_500_INTERNAL_SERVER_ERROR + status.HTTP_200_OK + if finished_without_errors + else status.HTTP_500_INTERNAL_SERVER_ERROR ) return stdout diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index d5d19ccedf1..c85c64f328c 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -1,95 +1,100 @@ +import logging +import traceback + # pylint: disable=redefined-builtin -from typing import Any, Dict, List, Union +from typing import Any, Dict, Optional, Union import aiodocker from fastapi import APIRouter, Depends, HTTPException, Query, status from ..dependencies import get_shared_store from ..shared_store import SharedStore +from ..utils import docker_client -containers_router = APIRouter(tags=["containers"]) - +logger = logging.getLogger(__name__) -@containers_router.get("/containers") -async def get_spawned_container_names( - shared_store: SharedStore = Depends(get_shared_store), -) -> List[str]: - """ Returns a list of containers created using docker-compose """ - return shared_store.container_names +containers_router = APIRouter(tags=["containers"]) @containers_router.get( "/containers:inspect", - responses={status.HTTP_400_BAD_REQUEST: {"description": "Erros in container"}}, + responses={ + status.HTTP_500_INTERNAL_SERVER_ERROR: {"description": "Erros in container"} + }, ) async def containers_inspect( shared_store: SharedStore = Depends(get_shared_store), ) -> Dict[str, Any]: """ Returns information about the container, like docker inspect command """ - docker = aiodocker.Docker() + with docker_client() as docker: + results = {} - results = {} + for container in shared_store.container_names: + try: + container_instance = await docker.containers.get(container) + results[container] = await container_instance.show() + except aiodocker.exceptions.DockerError as err: + logger.warning( + "An unexpected Docker error occurred:\n%s", + str(traceback.format_exc()), + ) + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, detail=err.message + ) from err - for container in shared_store.container_names: - try: - container_instance = await docker.containers.get(container) - results[container] = await container_instance.show() - except aiodocker.exceptions.DockerError as err: - # FIXME: This is NOT a client error https://httpstatuses.com/400. More or a server problem? 5XX?? - - raise HTTPException( - status.HTTP_400_BAD_REQUEST, - detail=err.message, - ) from err - - return results + return results @containers_router.get( - "/containers/status", - responses={status.HTTP_400_BAD_REQUEST: {"description": "Errors in container"}}, + "/containers", + responses={ + status.HTTP_500_INTERNAL_SERVER_ERROR: {"description": "Errors in container"} + }, ) async def containers_docker_status( shared_store: SharedStore = Depends(get_shared_store), ) -> Dict[str, Any]: """ Returns the status of the containers """ - docker = aiodocker.Docker() + with docker_client() as docker: + container_names = shared_store.container_names - container_names = shared_store.container_names + # if containers are being pulled, return pulling (fake status) + if shared_store.is_pulling_containers: + # pulling is a fake state use to share more information with the frontend + return {x: {"Status": "pulling", "Error": ""} for x in container_names} - # if containers are being pulled, return pulling (fake status) - if shared_store.is_pulling_containers: - # pulling is a fake state use to share more information with the frontend - return {x: {"Status": "pulling", "Error": ""} for x in container_names} + results = {} - results = {} + for container in container_names: + try: + container_instance = await docker.containers.get(container) + container_inspect = await container_instance.show() + container_state = container_inspect.get("State", {}) - for container in container_names: - try: - container_instance = await docker.containers.get(container) - container_inspect = await container_instance.show() - container_state = container_inspect.get("State", {}) - - # pending is another fake state use to share more information with the frontend - results[container] = { - "Status": container_state.get("Status", "pending"), - "Error": container_state.get("Error", ""), - } - except aiodocker.exceptions.DockerError as err: - # FIXME: This is NOT a client error https://httpstatuses.com/400. More or a server problem? 5XX?? - raise HTTPException( - status.HTTP_400_BAD_REQUEST, - detail=err.message, - ) from err + # pending is another fake state use to share more information with the frontend + results[container] = { + "Status": container_state.get("Status", "pending"), + "Error": container_state.get("Error", ""), + } + except aiodocker.exceptions.DockerError as err: + logger.warning( + "An unexpected Docker error occurred:\n%s", + str(traceback.format_exc()), + ) + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, detail=err.message + ) from err - return results + return results @containers_router.get( - "/containers/{id}/logs", + "/containers/{id}:logs", responses={ - status.HTTP_400_BAD_REQUEST: {"description": "Container does not exists"} + status.HTTP_500_INTERNAL_SERVER_ERROR: { + "description": "Container does not exists" + } }, ) async def get_container_logs( @@ -115,40 +120,41 @@ async def get_container_logs( # TODO: remove from here and dump directly into the logs of this service # do this in PR#1887 if id not in shared_store.container_names: - raise HTTPException( - status.HTTP_400_BAD_REQUEST, - detail=f"No container '{id}' was started", - ) + message = f"No container '{id}' was started. Started containers '{shared_store.container_names}'" + logger.warning(message) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=message) - docker = aiodocker.Docker() - try: - container_instance = await docker.containers.get(id) - - args = dict(stdout=True, stderr=True) - if timestamps: - args["timestamps"] = True + with docker_client() as docker: + try: + container_instance = await docker.containers.get(id) - if since or until: - raise HTTPException( - status.HTTP_501_NOT_IMPLEMENTED, - detail="since and until options are still not implemented", - ) + args = dict(stdout=True, stderr=True) + if timestamps: + args["timestamps"] = True - container_logs: str = await container_instance.log(**args) - return container_logs - except aiodocker.exceptions.DockerError as err: - # FIXME: This is NOT a client error https://httpstatuses.com/400. More or a server problem? 5XX?? + if since or until: + raise HTTPException( + status.HTTP_501_NOT_IMPLEMENTED, + detail="since and until options are still not implemented", + ) - raise HTTPException( - status.HTTP_400_BAD_REQUEST, - detail=err.message, - ) from err + container_logs: str = await container_instance.log(**args) + return container_logs + except aiodocker.exceptions.DockerError as err: + logger.warning( + "An unexpected Docker error occurred:\n%s", str(traceback.format_exc()) + ) + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, detail=err.message + ) from err @containers_router.get( "/containers/{id}:inspect", responses={ - status.HTTP_400_BAD_REQUEST: {"description": "Container does not exist"} + status.HTTP_500_INTERNAL_SERVER_ERROR: { + "description": "Container does not exist" + } }, ) async def inspect_container( @@ -157,54 +163,53 @@ async def inspect_container( """ Returns information about the container, like docker inspect command """ if id not in shared_store.container_names: - raise HTTPException( - status.HTTP_400_BAD_REQUEST, - detail=f"No container '{id}' was started", - ) + message = f"No container '{id}' was started. Started containers '{shared_store.container_names}'" + logger.warning(message) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=message) - docker = aiodocker.Docker() - - try: - container_instance = await docker.containers.get(id) - inspect_result: Dict[str, Any] = await container_instance.show() - return inspect_result - except aiodocker.exceptions.DockerError as err: - # FIXME: This is NOT a client error https://httpstatuses.com/400. More or a server problem? 5XX?? - - raise HTTPException( - status.HTTP_400_BAD_REQUEST, - detail=f"No container '{id}' was started", - ) from err + with docker_client() as docker: + try: + container_instance = await docker.containers.get(id) + inspect_result: Dict[str, Any] = await container_instance.show() + return inspect_result + except aiodocker.exceptions.DockerError as err: + logger.warning( + "An unexpected Docker error occurred:\n%s", str(traceback.format_exc()) + ) + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, detail=err.message + ) from err @containers_router.delete( "/containers/{id}", status_code=status.HTTP_204_NO_CONTENT, responses={ - status.HTTP_400_BAD_REQUEST: {"description": "Container does not exist"} + status.HTTP_500_INTERNAL_SERVER_ERROR: { + "description": "Container does not exist" + } }, ) async def remove_container( id: str, shared_store: SharedStore = Depends(get_shared_store) -) -> None: +) -> Optional[Dict[str, Any]]: if id not in shared_store.container_names: - raise HTTPException( - status.HTTP_400_BAD_REQUEST, detail=f"No container '{id}' was started" - ) - - docker = aiodocker.Docker() - - try: - container_instance = await docker.containers.get(id) - await container_instance.delete() - - except aiodocker.exceptions.DockerError as err: - # FIXME: This is NOT a client error https://httpstatuses.com/400. More or a server problem? 5XX?? + message = f"No container '{id}' was started. Started containers '{shared_store.container_names}'" + logger.warning(message) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=message) - raise HTTPException( - status.HTTP_400_BAD_REQUEST, - detail=err.message, - ) from err + with docker_client() as docker: + try: + container_instance = await docker.containers.get(id) + await container_instance.delete() + return None + except aiodocker.exceptions.DockerError as err: + logger.warning( + "An unexpected Docker error occurred:\n%s", str(traceback.format_exc()) + ) + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, detail=err.message + ) from err __all__ = ["containers_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py index de0c6254de6..02c81bd9e9f 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py @@ -18,7 +18,7 @@ async def health_endpoint( ) -> ApplicationHealth: if not application_health.is_healthy: raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, detail="Marked as unhealthy" + status.HTTP_503_SERVICE_UNAVAILABLE, detail=application_health.dict() ) return application_health From 12408244ab2d6e137a1312a2cd152b53fadf6bd5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 14:06:17 +0200 Subject: [PATCH 066/102] extended utils --- .../src/simcore_service_dynamic_sidecar/utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py index 00dd1ea8447..3473480c367 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py @@ -2,9 +2,11 @@ import logging import tempfile import traceback +from contextlib import contextmanager from pathlib import Path -from typing import AsyncGenerator, List, Tuple +from typing import AsyncGenerator, Generator, List, Tuple +import aiodocker import aiofiles import yaml from async_generator import asynccontextmanager @@ -28,6 +30,15 @@ async def write_to_tmp_file(file_contents: str) -> AsyncGenerator[Path, None]: await aiofiles.os.remove(file_path) +@contextmanager +def docker_client() -> Generator[aiodocker.Docker, None, None]: + docker = aiodocker.Docker() + try: + yield docker + finally: + docker.close() + + async def async_command(command: str, command_timeout: float) -> Tuple[bool, str]: """Returns if the command exited correctly and the stdout of the command """ proc = await asyncio.create_subprocess_shell( From d03853e3dd48cdddeb1f2b47dd85ba0fec240fe3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 14:06:24 +0200 Subject: [PATCH 067/102] adapted new tests --- .../tests/unit/test_api_compose.py | 61 ++------- .../tests/unit/test_api_containers.py | 127 +++++++++++------- .../tests/unit/test_api_health.py | 19 ++- 3 files changed, 105 insertions(+), 102 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_compose.py b/services/dynamic-sidecar/tests/unit/test_api_compose.py index 4f647ef86ce..895dbedcd98 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_compose.py +++ b/services/dynamic-sidecar/tests/unit/test_api_compose.py @@ -7,8 +7,8 @@ import pytest from async_asgi_testclient import TestClient from faker import Faker +from fastapi import status from simcore_service_dynamic_sidecar._meta import api_vtag -from starlette.status import HTTP_200_OK, HTTP_500_INTERNAL_SERVER_ERROR DEFAULT_COMMAND_TIMEOUT = 10.0 @@ -30,86 +30,53 @@ async def test_compose_up( ) -> None: response = await test_client.post( - f"/{api_vtag}/containers:up", + f"/{api_vtag}/containers", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), data=compose_spec, ) - assert response.status_code == HTTP_200_OK, response.text + assert response.status_code == status.HTTP_201_CREATED, response.text + assert json.loads(response.text) is None async def test_compose_up_spec_not_provided(test_client: TestClient) -> None: response = await test_client.post( - f"/{api_vtag}/containers:up", + f"/{api_vtag}/containers", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) - assert response.status_code == HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text assert json.loads(response.text) == {"detail": "\nProvided yaml is not valid!"} async def test_compose_up_spec_invalid(test_client: TestClient) -> None: invalid_compose_spec = Faker().text() response = await test_client.post( - f"/{api_vtag}/containers:up", + f"/{api_vtag}/containers", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), data=invalid_compose_spec, ) - assert response.status_code == HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text assert "Provided yaml is not valid!" in response.text # 28+ characters means the compos spec is also present in the error message assert len(response.text) > 28 -async def test_containers_pull( +async def test_containers_down_after_starting( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: # store spec first response = await test_client.post( - f"/{api_vtag}/containers:up", + f"/{api_vtag}/containers", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), data=compose_spec, ) - assert response.status_code == HTTP_200_OK, response.text - assert response.text != "" - - # pull images for spec - response = await test_client.get( - f"/{api_vtag}/containers:pull", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - ) - assert response.status_code == HTTP_200_OK, response.text - assert response.text != "" - - -async def test_containers_pull_missing_spec( - test_client: TestClient, compose_spec: Dict[str, Any] -) -> None: - response = await test_client.get( - f"/{api_vtag}/containers:pull", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - ) - assert response.status_code == HTTP_500_INTERNAL_SERVER_ERROR, response.text - assert json.loads(response.text) == { - "detail": "No spec for docker-compose pull was found" - } - - -async def test_containers_down_after_stopping( - test_client: TestClient, compose_spec: Dict[str, Any] -) -> None: - # store spec first - response = await test_client.post( - f"/{api_vtag}/containers:up", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - data=compose_spec, - ) - assert response.status_code == HTTP_200_OK, response.text - assert response.text != "" + assert response.status_code == status.HTTP_201_CREATED, response.text + assert json.loads(response.text) is None response = await test_client.post( f"/{api_vtag}/containers:down", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) - assert response.status_code == HTTP_200_OK, response.text + assert response.status_code == status.HTTP_200_OK, response.text assert response.text != "" @@ -120,7 +87,7 @@ async def test_containers_down_missing_spec( f"/{api_vtag}/containers:down", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) - assert response.status_code == HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text assert json.loads(response.text) == { "detail": "No spec for docker-compose down was found" } diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index e60fe12e07b..2dc740b9533 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -1,13 +1,17 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument + import json from contextlib import contextmanager from typing import Any, Dict, Generator, List import pytest from async_asgi_testclient import TestClient +from fastapi import status from simcore_service_dynamic_sidecar._meta import api_vtag +from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings +from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command from simcore_service_dynamic_sidecar.shared_store import SharedStore pytestmark = pytest.mark.asyncio @@ -26,20 +30,38 @@ def compose_spec() -> str: ) +async def assert_compose_spec_pulled( + compose_spec: str, settings: DynamicSidecarSettings +) -> None: + """ensures all containers inside compose_spec are pulled""" + + command = ( + "docker-compose --project-name {project} --file {file_path} " + "up --no-build --detach" + ) + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=compose_spec, + command=command, + command_timeout=10.0, + ) + + assert finished_without_errors is True, stdout + + @pytest.fixture -async def started_containers( - test_client: TestClient, compose_spec: Dict[str, Any] -) -> List[str]: - # store spec first - response = await test_client.post(f"/{api_vtag}/compose:store", data=compose_spec) - assert response.status_code == 204, response.text - assert response.text == "" - - # pull images for spec +async def started_containers(test_client: TestClient, compose_spec: str) -> List[str]: + settings: DynamicSidecarSettings = test_client.application.state.settings + await assert_compose_spec_pulled(compose_spec, settings) + + # start containers response = await test_client.post( - f"/{api_vtag}/compose", query_string=dict(command_timeout=10.0) + f"/{api_vtag}/containers", + query_string=dict(command_timeout=10.0), + data=compose_spec, ) - assert response.status_code == 200, response.text + assert response.status_code == status.HTTP_201_CREATED, response.text + assert json.loads(response.text) is None shared_store: SharedStore = test_client.application.state.shared_store container_names = shared_store.container_names @@ -57,7 +79,7 @@ async def test_containers_get( test_client: TestClient, started_containers: List[str] ) -> None: response = await test_client.get(f"/{api_vtag}/containers") - assert response.status_code == 200, response.text + assert response.status_code == status.HTTP_200_OK, response.text assert set(json.loads(response.text)) == set(started_containers) @@ -68,7 +90,7 @@ async def test_containers_inspect( f"/{api_vtag}/containers:inspect", query_string=dict(container_names=started_containers), ) - assert response.status_code == 200, response.text + assert response.status_code == status.HTTP_200_OK, response.text assert set(json.loads(response.text).keys()) == set(started_containers) @@ -79,7 +101,7 @@ async def test_containers_inspect_docker_error( f"/{api_vtag}/containers:inspect", query_string=dict(container_names=started_containers), ) - assert response.status_code == 400, response.text + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text def assert_keys_exist(result: Dict[str, Any]) -> bool: @@ -93,10 +115,10 @@ async def test_containers_docker_status( test_client: TestClient, started_containers: List[str] ) -> None: response = await test_client.get( - f"/{api_vtag}/containers:docker-status", + f"/{api_vtag}/containers", query_string=dict(container_names=started_containers), ) - assert response.status_code == 200, response.text + assert response.status_code == status.HTTP_200_OK, response.text decoded_response = json.loads(response.text) assert set(decoded_response) == set(started_containers) assert assert_keys_exist(decoded_response) is True @@ -119,10 +141,10 @@ def mark_pulling(shared_store: SharedStore) -> Generator[None, None, None]: assert shared_store.is_pulling_containers is True response = await test_client.get( - f"/{api_vtag}/containers:docker-status", + f"/{api_vtag}/containers", query_string=dict(container_names=started_containers), ) - assert response.status_code == 200, response.text + assert response.status_code == status.HTTP_200_OK, response.text decoded_response = json.loads(response.text) assert assert_keys_exist(decoded_response) is True @@ -134,10 +156,10 @@ async def test_containers_docker_status_docker_error( test_client: TestClient, started_containers: List[str], mock_containers_get: None ) -> None: response = await test_client.get( - f"/{api_vtag}/containers:docker-status", + f"/{api_vtag}/containers", query_string=dict(container_names=started_containers), ) - assert response.status_code == 400, response.text + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text async def test_container_inspect_logs_remove( @@ -145,20 +167,19 @@ async def test_container_inspect_logs_remove( ) -> None: for container in started_containers: # get container logs - response = await test_client.get(f"/{api_vtag}/containers/{container}/logs") - assert response.status_code == 200, response.text + response = await test_client.get(f"/{api_vtag}/containers/{container}:logs") + assert response.status_code == status.HTTP_200_OK, response.text # inspect container - response = await test_client.get(f"/{api_vtag}/containers/{container}/inspect") - assert response.status_code == 200, response.text + response = await test_client.get(f"/{api_vtag}/containers/{container}:inspect") + assert response.status_code == status.HTTP_200_OK, response.text parsed_response = response.json() assert parsed_response["Name"] == f"/{container}" # delete container - response = await test_client.delete( - f"/{api_vtag}/containers/{container}/remove" - ) - assert response.status_code == 200, response.text + response = await test_client.delete(f"/{api_vtag}/containers/{container}") + assert response.status_code == status.HTTP_204_NO_CONTENT, response.text + assert json.loads(response.text) is None async def test_container_logs_with_timestamps( @@ -167,34 +188,40 @@ async def test_container_logs_with_timestamps( for container in started_containers: # get container logs response = await test_client.get( - f"/{api_vtag}/containers/{container}/logs", + f"/{api_vtag}/containers/{container}:logs", query_string=dict(timestamps=True), ) - assert response.status_code == 200, response.text + assert response.status_code == status.HTTP_200_OK, response.text async def test_container_missing_container( test_client: TestClient, not_started_containers: List[str] ) -> None: def _expected_error_string(container: str) -> Dict[str, str]: - return dict(error=f"No container '{container}' was started") + return dict( + detail=f"No container '{container}' was started. Started containers '[]'" + ) for container in not_started_containers: # get container logs - response = await test_client.get(f"/{api_vtag}/containers/{container}/logs") - assert response.status_code == 400, response.text + response = await test_client.get(f"/{api_vtag}/containers/{container}:logs") + assert ( + response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + ), response.text assert response.json() == _expected_error_string(container) # inspect container - response = await test_client.get(f"/{api_vtag}/containers/{container}/inspect") - assert response.status_code == 400, response.text + response = await test_client.get(f"/{api_vtag}/containers/{container}:inspect") + assert ( + response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + ), response.text assert response.json() == _expected_error_string(container) # delete container - response = await test_client.delete( - f"/{api_vtag}/containers/{container}/remove" - ) - assert response.status_code == 400, response.text + response = await test_client.delete(f"/{api_vtag}/containers/{container}") + assert ( + response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + ), response.text assert response.json() == _expected_error_string(container) @@ -204,22 +231,26 @@ async def test_container_docker_error( mock_containers_get: None, ) -> None: def _expected_error_string() -> Dict[str, str]: - return dict(error="aiodocker_mocked_error") + return dict(detail="aiodocker_mocked_error") for container in started_containers: # get container logs - response = await test_client.get(f"/{api_vtag}/containers/{container}/logs") - assert response.status_code == 400, response.text + response = await test_client.get(f"/{api_vtag}/containers/{container}:logs") + assert ( + response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + ), response.text assert response.json() == _expected_error_string() # inspect container - response = await test_client.get(f"/{api_vtag}/containers/{container}/inspect") - assert response.status_code == 400, response.text + response = await test_client.get(f"/{api_vtag}/containers/{container}:inspect") + assert ( + response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + ), response.text assert response.json() == _expected_error_string() # delete container - response = await test_client.delete( - f"/{api_vtag}/containers/{container}/remove" - ) - assert response.status_code == 400, response.text + response = await test_client.delete(f"/{api_vtag}/containers/{container}") + assert ( + response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + ), response.text assert response.json() == _expected_error_string() diff --git a/services/dynamic-sidecar/tests/unit/test_api_health.py b/services/dynamic-sidecar/tests/unit/test_api_health.py index 0f41070cca8..dab756db51f 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_health.py +++ b/services/dynamic-sidecar/tests/unit/test_api_health.py @@ -1,15 +1,20 @@ import pytest from async_asgi_testclient import TestClient +from fastapi import status from simcore_service_dynamic_sidecar.models import ApplicationHealth pytestmark = pytest.mark.asyncio -@pytest.mark.parametrize("is_healthy,status_code", [(True, 200), (False, 400)]) -async def test_is_healthy( - test_client: TestClient, is_healthy: bool, status_code: int -) -> None: - test_client.application.state.application_health.is_healthy = is_healthy +async def test_is_healthy(test_client: TestClient) -> None: + test_client.application.state.application_health.is_healthy = True response = await test_client.get("/health") - assert response.status_code == status_code, response - assert response.json() == ApplicationHealth(is_healthy=is_healthy).dict() + assert response.status_code == status.HTTP_200_OK, response + assert response.json() == ApplicationHealth(is_healthy=True).dict() + + +async def test_is_unhealthy(test_client: TestClient) -> None: + test_client.application.state.application_health.is_healthy = False + response = await test_client.get("/health") + assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE, response + assert response.json() == {"detail": ApplicationHealth(is_healthy=False).dict()} From f810a4851066f4e8ecf80397b5a23d15b9ba2c95 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 14:06:30 +0200 Subject: [PATCH 068/102] updated API spec --- services/dynamic-sidecar/openapi.json | 187 +++++++------------------- 1 file changed, 45 insertions(+), 142 deletions(-) diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index 7d02f603003..dd959da8218 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -19,125 +19,61 @@ } } } + }, + "503": { + "description": "Service is unhealthy" } } } }, - "/v1/compose:store": { - "post": { + "/v1/containers": { + "get": { "tags": [ - "docker-compose" - ], - "summary": "Validates Docker Compose Spec And Stores It", - "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", - "operationId": "validates_docker_compose_spec_and_stores_it_v1_compose_store_post", - "parameters": [ - { - "required": false, - "schema": { - "title": "Command Timeout", - "type": "number", - "default": 5.0 - }, - "name": "command_timeout", - "in": "query" - } + "containers" ], + "summary": "Containers Docker Status", + "description": "Returns the status of the containers ", + "operationId": "containers_docker_status_v1_containers_get", "responses": { "200": { "description": "Successful Response", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - }, - "204": { - "description": "No Content" - }, - "422": { - "description": "Validation Error", "content": { "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } + "schema": {} } } + }, + "500": { + "description": "Errors in container" } } - } - }, - "/v1/compose": { + }, "post": { "tags": [ "docker-compose" ], "summary": "Runs Docker Compose Up", "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", - "operationId": "runs_docker_compose_up_v1_compose_post", + "operationId": "runs_docker_compose_up_v1_containers_post", "parameters": [ { + "description": "docker-compose up also pulls images, this value needs to be big enough to account for that", "required": true, "schema": { "title": "Command Timeout", - "type": "number" + "type": "number", + "description": "docker-compose up also pulls images, this value needs to be big enough to account for that" }, "name": "command_timeout", "in": "query" } ], "responses": { - "200": { + "201": { "description": "Successful Response", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - }, - "422": { - "description": "Validation Error", "content": { "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - }, - "delete": { - "tags": [ - "docker-compose" - ], - "summary": "Runs Docker Compose Down", - "description": "Removes the previously started service\nand returns the docker-compose output", - "operationId": "runs_docker_compose_down_v1_compose_delete", - "parameters": [ - { - "required": true, - "schema": { - "title": "Command Timeout", - "type": "number" - }, - "name": "command_timeout", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "text/plain": { - "schema": { - "type": "string" - } + "schema": {} } } }, @@ -154,14 +90,14 @@ } } }, - "/v1/compose:pull": { - "get": { + "/v1/containers:down": { + "post": { "tags": [ "docker-compose" ], - "summary": "Runs Docker Compose Pull", - "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", - "operationId": "runs_docker_compose_pull_v1_compose_pull_get", + "summary": "Runs Docker Compose Down", + "description": "Removes the previously started service\nand returns the docker-compose output", + "operationId": "runs_docker_compose_down_v1_containers_down_post", "parameters": [ { "required": true, @@ -197,26 +133,6 @@ } } }, - "/v1/containers": { - "get": { - "tags": [ - "containers" - ], - "summary": "Get Spawned Container Names", - "description": "Returns a list of containers created using docker-compose ", - "operationId": "get_spawned_container_names_v1_containers_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - } - } - } - }, "/v1/containers:inspect": { "get": { "tags": [ @@ -233,31 +149,14 @@ "schema": {} } } + }, + "500": { + "description": "Erros in container" } } } }, - "/v1/containers:docker-status": { - "get": { - "tags": [ - "containers" - ], - "summary": "Containers Docker Status", - "description": "Returns the status of the containers ", - "operationId": "containers_docker_status_v1_containers_docker_status_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - } - } - } - }, - "/v1/containers/{id}/logs": { + "/v1/containers/{id}:logs": { "get": { "tags": [ "containers" @@ -279,7 +178,7 @@ "description": "Only return logs since this time, as a UNIX timestamp", "required": false, "schema": { - "title": "Timstamp", + "title": "Timestamp", "type": "integer", "description": "Only return logs since this time, as a UNIX timestamp", "default": 0 @@ -291,7 +190,7 @@ "description": "Only return logs before this time, as a UNIX timestamp", "required": false, "schema": { - "title": "Timstamp", + "title": "Timestamp", "type": "integer", "description": "Only return logs before this time, as a UNIX timestamp", "default": 0 @@ -321,6 +220,9 @@ } } }, + "500": { + "description": "Container does not exists" + }, "422": { "description": "Validation Error", "content": { @@ -334,7 +236,7 @@ } } }, - "/v1/containers/{id}/inspect": { + "/v1/containers/{id}:inspect": { "get": { "tags": [ "containers" @@ -362,6 +264,9 @@ } } }, + "500": { + "description": "Container does not exist" + }, "422": { "description": "Validation Error", "content": { @@ -375,13 +280,13 @@ } } }, - "/v1/containers/{id}/remove": { + "/v1/containers/{id}": { "delete": { "tags": [ "containers" ], "summary": "Remove Container", - "operationId": "remove_container_v1_containers__id__remove_delete", + "operationId": "remove_container_v1_containers__id__delete", "parameters": [ { "required": true, @@ -394,13 +299,11 @@ } ], "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } + "204": { + "description": "Successful Response" + }, + "500": { + "description": "Container does not exist" }, "422": { "description": "Validation Error", From 8e4156051d7131851b06cdf77c070d1a9fb353cc Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 14:12:55 +0200 Subject: [PATCH 069/102] moving routes under the same module --- .../api/_routing.py | 2 - .../api/compose.py | 124 ------------------ .../api/containers.py | 115 +++++++++++++++- 3 files changed, 112 insertions(+), 129 deletions(-) delete mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py index ebdb7478c68..4b4829063f4 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/_routing.py @@ -3,7 +3,6 @@ from fastapi import APIRouter from .._meta import api_vtag -from .compose import compose_router from .containers import containers_router from .health import health_router from .mocked import mocked_router @@ -11,7 +10,6 @@ # setup and register all routes here form different modules main_router = APIRouter() main_router.include_router(health_router) -main_router.include_router(compose_router, prefix=f"/{api_vtag}") main_router.include_router(containers_router, prefix=f"/{api_vtag}") main_router.include_router(mocked_router) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py deleted file mode 100644 index b8e810609d0..00000000000 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/compose.py +++ /dev/null @@ -1,124 +0,0 @@ -import logging -import traceback -from typing import Any, Dict, Optional, Union - -from fastapi import ( - APIRouter, - BackgroundTasks, - Depends, - HTTPException, - Query, - Request, - Response, - status, -) -from fastapi.responses import PlainTextResponse - -from ..dependencies import get_settings, get_shared_store -from ..settings import DynamicSidecarSettings -from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command -from ..shared_store import SharedStore -from ..utils import assemble_container_names -from ..validation import InvalidComposeSpec, validate_compose_spec - -logger = logging.getLogger(__name__) -compose_router = APIRouter(tags=["docker-compose"]) - - -async def task_docker_compose_up( - command_timeout: float, - settings: DynamicSidecarSettings, - shared_store: SharedStore, -) -> None: - # --no-build might be a security risk building is disabled - command = ( - "docker-compose --project-name {project} --file {file_path} " - "up --no-build --detach" - ) - finished_without_errors, stdout = await write_file_and_run_command( - settings=settings, - file_content=shared_store.compose_spec, - command=command, - command_timeout=command_timeout, - ) - message = f"Finished {command} with output\n{stdout}" - - if finished_without_errors: - logger.info(message) - else: - logger.error(message) - - return None - - -@compose_router.post("/containers", status_code=status.HTTP_201_CREATED) -async def runs_docker_compose_up( - request: Request, - background_tasks: BackgroundTasks, - command_timeout: float = Query( - ..., - description=( - "docker-compose up also pulls images, this value " - "needs to be big enough to account for that" - ), - ), - settings: DynamicSidecarSettings = Depends(get_settings), - shared_store: SharedStore = Depends(get_shared_store), -) -> Optional[Dict[str, Any]]: - """ Expects the docker-compose spec as raw-body utf-8 encoded text """ - - # stores the compose spec after validation - body_as_text = (await request.body()).decode("utf-8") - - try: - shared_store.compose_spec = await validate_compose_spec( - settings=settings, - compose_file_content=body_as_text, - command_timeout=command_timeout, - ) - shared_store.container_names = assemble_container_names( - shared_store.compose_spec - ) - except InvalidComposeSpec as e: - logger.warning("Error detected %s", traceback.format_exc()) - raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e - - # run docker-compose in a background queue and return early - background_tasks.add_task( - task_docker_compose_up, command_timeout, settings, shared_store - ) - return None - - -@compose_router.post("/containers:down", response_class=PlainTextResponse) -async def runs_docker_compose_down( - response: Response, - command_timeout: float, - settings: DynamicSidecarSettings = Depends(get_settings), - shared_store: SharedStore = Depends(get_shared_store), -) -> Union[str, Dict[str, Any]]: - """Removes the previously started service - and returns the docker-compose output""" - - stored_compose_content = shared_store.compose_spec - if stored_compose_content is None: - raise HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="No spec for docker-compose down was found", - ) - - finished_without_errors, stdout = await remove_the_compose_spec( - shared_store=shared_store, - settings=settings, - command_timeout=command_timeout, - ) - - response.status_code = ( - status.HTTP_200_OK - if finished_without_errors - else status.HTTP_500_INTERNAL_SERVER_ERROR - ) - return stdout - - -__all__ = ["compose_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index c85c64f328c..302e36ebba1 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -5,17 +5,126 @@ from typing import Any, Dict, Optional, Union import aiodocker -from fastapi import APIRouter, Depends, HTTPException, Query, status +from fastapi import ( + APIRouter, + BackgroundTasks, + Depends, + HTTPException, + Query, + Request, + Response, + status, +) +from fastapi.responses import PlainTextResponse -from ..dependencies import get_shared_store +from ..dependencies import get_settings, get_shared_store +from ..settings import DynamicSidecarSettings +from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command from ..shared_store import SharedStore -from ..utils import docker_client +from ..utils import assemble_container_names, docker_client +from ..validation import InvalidComposeSpec, validate_compose_spec logger = logging.getLogger(__name__) containers_router = APIRouter(tags=["containers"]) +async def task_docker_compose_up( + command_timeout: float, + settings: DynamicSidecarSettings, + shared_store: SharedStore, +) -> None: + # --no-build might be a security risk building is disabled + command = ( + "docker-compose --project-name {project} --file {file_path} " + "up --no-build --detach" + ) + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=shared_store.compose_spec, + command=command, + command_timeout=command_timeout, + ) + message = f"Finished {command} with output\n{stdout}" + + if finished_without_errors: + logger.info(message) + else: + logger.error(message) + + return None + + +@containers_router.post("/containers", status_code=status.HTTP_201_CREATED) +async def runs_docker_compose_up( + request: Request, + background_tasks: BackgroundTasks, + command_timeout: float = Query( + ..., + description=( + "docker-compose up also pulls images, this value " + "needs to be big enough to account for that" + ), + ), + settings: DynamicSidecarSettings = Depends(get_settings), + shared_store: SharedStore = Depends(get_shared_store), +) -> Optional[Dict[str, Any]]: + """ Expects the docker-compose spec as raw-body utf-8 encoded text """ + + # stores the compose spec after validation + body_as_text = (await request.body()).decode("utf-8") + + try: + shared_store.compose_spec = await validate_compose_spec( + settings=settings, + compose_file_content=body_as_text, + command_timeout=command_timeout, + ) + shared_store.container_names = assemble_container_names( + shared_store.compose_spec + ) + except InvalidComposeSpec as e: + logger.warning("Error detected %s", traceback.format_exc()) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e + + # run docker-compose in a background queue and return early + background_tasks.add_task( + task_docker_compose_up, command_timeout, settings, shared_store + ) + return None + + +@containers_router.post("/containers:down", response_class=PlainTextResponse) +async def runs_docker_compose_down( + response: Response, + command_timeout: float, + settings: DynamicSidecarSettings = Depends(get_settings), + shared_store: SharedStore = Depends(get_shared_store), +) -> Union[str, Dict[str, Any]]: + """Removes the previously started service + and returns the docker-compose output""" + + stored_compose_content = shared_store.compose_spec + if stored_compose_content is None: + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="No spec for docker-compose down was found", + ) + + finished_without_errors, stdout = await remove_the_compose_spec( + shared_store=shared_store, + settings=settings, + command_timeout=command_timeout, + ) + + response.status_code = ( + status.HTTP_200_OK + if finished_without_errors + else status.HTTP_500_INTERNAL_SERVER_ERROR + ) + return stdout + + @containers_router.get( "/containers:inspect", responses={ From 194a9e9f1cf440aaa906061fa1078b135d69f4c5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 14:15:40 +0200 Subject: [PATCH 070/102] moved tests and updated openapi.json --- services/dynamic-sidecar/openapi.json | 4 +- .../tests/unit/test_api_compose.py | 93 ------------------- .../tests/unit/test_api_containers.py | 71 ++++++++++++++ 3 files changed, 73 insertions(+), 95 deletions(-) delete mode 100644 services/dynamic-sidecar/tests/unit/test_api_compose.py diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index dd959da8218..bcec37a6dbe 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -50,7 +50,7 @@ }, "post": { "tags": [ - "docker-compose" + "containers" ], "summary": "Runs Docker Compose Up", "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", @@ -93,7 +93,7 @@ "/v1/containers:down": { "post": { "tags": [ - "docker-compose" + "containers" ], "summary": "Runs Docker Compose Down", "description": "Removes the previously started service\nand returns the docker-compose output", diff --git a/services/dynamic-sidecar/tests/unit/test_api_compose.py b/services/dynamic-sidecar/tests/unit/test_api_compose.py deleted file mode 100644 index 895dbedcd98..00000000000 --- a/services/dynamic-sidecar/tests/unit/test_api_compose.py +++ /dev/null @@ -1,93 +0,0 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument - -import json -from typing import Any, Dict - -import pytest -from async_asgi_testclient import TestClient -from faker import Faker -from fastapi import status -from simcore_service_dynamic_sidecar._meta import api_vtag - -DEFAULT_COMMAND_TIMEOUT = 10.0 - -pytestmark = pytest.mark.asyncio - - -@pytest.fixture -def compose_spec() -> str: - return json.dumps( - { - "version": "3", - "services": {"nginx": {"image": "busybox"}}, - } - ) - - -async def test_compose_up( - test_client: TestClient, compose_spec: Dict[str, Any] -) -> None: - - response = await test_client.post( - f"/{api_vtag}/containers", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - data=compose_spec, - ) - assert response.status_code == status.HTTP_201_CREATED, response.text - assert json.loads(response.text) is None - - -async def test_compose_up_spec_not_provided(test_client: TestClient) -> None: - response = await test_client.post( - f"/{api_vtag}/containers", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - ) - assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text - assert json.loads(response.text) == {"detail": "\nProvided yaml is not valid!"} - - -async def test_compose_up_spec_invalid(test_client: TestClient) -> None: - invalid_compose_spec = Faker().text() - response = await test_client.post( - f"/{api_vtag}/containers", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - data=invalid_compose_spec, - ) - assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text - assert "Provided yaml is not valid!" in response.text - # 28+ characters means the compos spec is also present in the error message - assert len(response.text) > 28 - - -async def test_containers_down_after_starting( - test_client: TestClient, compose_spec: Dict[str, Any] -) -> None: - # store spec first - response = await test_client.post( - f"/{api_vtag}/containers", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - data=compose_spec, - ) - assert response.status_code == status.HTTP_201_CREATED, response.text - assert json.loads(response.text) is None - - response = await test_client.post( - f"/{api_vtag}/containers:down", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - ) - assert response.status_code == status.HTTP_200_OK, response.text - assert response.text != "" - - -async def test_containers_down_missing_spec( - test_client: TestClient, compose_spec: Dict[str, Any] -) -> None: - response = await test_client.post( - f"/{api_vtag}/containers:down", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - ) - assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text - assert json.loads(response.text) == { - "detail": "No spec for docker-compose down was found" - } diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 2dc740b9533..4ff55a63c3c 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -8,12 +8,15 @@ import pytest from async_asgi_testclient import TestClient +from faker import Faker from fastapi import status from simcore_service_dynamic_sidecar._meta import api_vtag from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command from simcore_service_dynamic_sidecar.shared_store import SharedStore +DEFAULT_COMMAND_TIMEOUT = 10.0 + pytestmark = pytest.mark.asyncio @@ -75,6 +78,74 @@ def not_started_containers() -> List[str]: return [f"missing-container-{i}" for i in range(5)] +async def test_compose_up( + test_client: TestClient, compose_spec: Dict[str, Any] +) -> None: + + response = await test_client.post( + f"/{api_vtag}/containers", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), + data=compose_spec, + ) + assert response.status_code == status.HTTP_201_CREATED, response.text + assert json.loads(response.text) is None + + +async def test_compose_up_spec_not_provided(test_client: TestClient) -> None: + response = await test_client.post( + f"/{api_vtag}/containers", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), + ) + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert json.loads(response.text) == {"detail": "\nProvided yaml is not valid!"} + + +async def test_compose_up_spec_invalid(test_client: TestClient) -> None: + invalid_compose_spec = Faker().text() + response = await test_client.post( + f"/{api_vtag}/containers", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), + data=invalid_compose_spec, + ) + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert "Provided yaml is not valid!" in response.text + # 28+ characters means the compos spec is also present in the error message + assert len(response.text) > 28 + + +async def test_containers_down_after_starting( + test_client: TestClient, compose_spec: Dict[str, Any] +) -> None: + # store spec first + response = await test_client.post( + f"/{api_vtag}/containers", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), + data=compose_spec, + ) + assert response.status_code == status.HTTP_201_CREATED, response.text + assert json.loads(response.text) is None + + response = await test_client.post( + f"/{api_vtag}/containers:down", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), + ) + assert response.status_code == status.HTTP_200_OK, response.text + assert response.text != "" + + +async def test_containers_down_missing_spec( + test_client: TestClient, compose_spec: Dict[str, Any] +) -> None: + response = await test_client.post( + f"/{api_vtag}/containers:down", + query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), + ) + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert json.loads(response.text) == { + "detail": "No spec for docker-compose down was found" + } + + async def test_containers_get( test_client: TestClient, started_containers: List[str] ) -> None: From 4255341e025ddd4d3409c6ce7a59e9eaf97aa5f5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 14:20:22 +0200 Subject: [PATCH 071/102] updated script entryoint name --- services/dynamic-sidecar/docker/boot.sh | 2 +- services/dynamic-sidecar/setup.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/services/dynamic-sidecar/docker/boot.sh b/services/dynamic-sidecar/docker/boot.sh index cdc4b875ac1..2948d71c279 100755 --- a/services/dynamic-sidecar/docker/boot.sh +++ b/services/dynamic-sidecar/docker/boot.sh @@ -32,5 +32,5 @@ then # this way we can have reload in place as well exec uvicorn simcore_service_dynamic_sidecar.main:app --reload --host 0.0.0.0 else - exec simcore_service_dynamic_sidecar_startup + exec simcore-service-dynamic-sidecar fi diff --git a/services/dynamic-sidecar/setup.py b/services/dynamic-sidecar/setup.py index 486343f80c2..387172bfdb9 100644 --- a/services/dynamic-sidecar/setup.py +++ b/services/dynamic-sidecar/setup.py @@ -36,9 +36,7 @@ def read_reqs(reqs_path: Path): setup_requires=["setuptools_scm"], entry_points={ "console_scripts": [ - "simcore-service-dynamic-sidecar=simcore_service_dynamic_sidecar.main:main", - # alternative entry-points - "simcore_service_dynamic_sidecar_startup = simcore_service_dynamic_sidecar.main:main", + "simcore-service-dynamic-sidecar=simcore_service_dynamic_sidecar.main:main" ], }, ) From 4536b93d5bd8971a18fa84f6efffdb27c92a70a7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 15:23:48 +0200 Subject: [PATCH 072/102] renamed endpoint --- .../src/simcore_service_dynamic_sidecar/api/containers.py | 2 +- .../dynamic-sidecar/tests/unit/test_api_containers.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 302e36ebba1..aaeea934a6e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -199,7 +199,7 @@ async def containers_docker_status( @containers_router.get( - "/containers/{id}:logs", + "/containers/{id}/logs", responses={ status.HTTP_500_INTERNAL_SERVER_ERROR: { "description": "Container does not exists" diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 4ff55a63c3c..25602173e0c 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -238,7 +238,7 @@ async def test_container_inspect_logs_remove( ) -> None: for container in started_containers: # get container logs - response = await test_client.get(f"/{api_vtag}/containers/{container}:logs") + response = await test_client.get(f"/{api_vtag}/containers/{container}/logs") assert response.status_code == status.HTTP_200_OK, response.text # inspect container @@ -259,7 +259,7 @@ async def test_container_logs_with_timestamps( for container in started_containers: # get container logs response = await test_client.get( - f"/{api_vtag}/containers/{container}:logs", + f"/{api_vtag}/containers/{container}/logs", query_string=dict(timestamps=True), ) assert response.status_code == status.HTTP_200_OK, response.text @@ -275,7 +275,7 @@ def _expected_error_string(container: str) -> Dict[str, str]: for container in not_started_containers: # get container logs - response = await test_client.get(f"/{api_vtag}/containers/{container}:logs") + response = await test_client.get(f"/{api_vtag}/containers/{container}/logs") assert ( response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR ), response.text @@ -306,7 +306,7 @@ def _expected_error_string() -> Dict[str, str]: for container in started_containers: # get container logs - response = await test_client.get(f"/{api_vtag}/containers/{container}:logs") + response = await test_client.get(f"/{api_vtag}/containers/{container}/logs") assert ( response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR ), response.text From 74771845d9cdbe377e439cb1e832e1610b6b05be Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 15:26:20 +0200 Subject: [PATCH 073/102] refactor --- .../simcore_service_dynamic_sidecar/api/containers.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index aaeea934a6e..5776e883d02 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -96,7 +96,6 @@ async def runs_docker_compose_up( @containers_router.post("/containers:down", response_class=PlainTextResponse) async def runs_docker_compose_down( - response: Response, command_timeout: float, settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), @@ -117,11 +116,9 @@ async def runs_docker_compose_down( command_timeout=command_timeout, ) - response.status_code = ( - status.HTTP_200_OK - if finished_without_errors - else status.HTTP_500_INTERNAL_SERVER_ERROR - ) + if not finished_without_errors: + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=stdout) + return stdout From 9327327886b28599671720574a1f09ffcfb2650e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 15:36:34 +0200 Subject: [PATCH 074/102] refactor error raising and staus code for resource not found --- .../api/containers.py | 74 +++++++------------ .../tests/unit/test_api_containers.py | 12 +-- 2 files changed, 30 insertions(+), 56 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 5776e883d02..b53df3e2f1c 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -2,7 +2,7 @@ import traceback # pylint: disable=redefined-builtin -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Optional, Union, List import aiodocker from fastapi import ( @@ -12,7 +12,6 @@ HTTPException, Query, Request, - Response, status, ) from fastapi.responses import PlainTextResponse @@ -55,6 +54,24 @@ async def task_docker_compose_up( return None +def _raise_if_container_is_missing(id: str, container_names: List[str]) -> None: + if id not in container_names: + message = ( + f"No container '{id}' was started. Started containers '{container_names}'" + ) + logger.warning(message) + raise HTTPException(status.HTTP_404_NOT_FOUND, detail=message) + + +def _raise_from_docker_error(error: aiodocker.exceptions.DockerError) -> None: + logger.warning( + "An unexpected Docker error occurred:\n%s", str(traceback.format_exc()) + ) + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error.message + ) from error + + @containers_router.post("/containers", status_code=status.HTTP_201_CREATED) async def runs_docker_compose_up( request: Request, @@ -140,13 +157,7 @@ async def containers_inspect( container_instance = await docker.containers.get(container) results[container] = await container_instance.show() except aiodocker.exceptions.DockerError as err: - logger.warning( - "An unexpected Docker error occurred:\n%s", - str(traceback.format_exc()), - ) - raise HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, detail=err.message - ) from err + _raise_from_docker_error(err) return results @@ -184,13 +195,7 @@ async def containers_docker_status( "Error": container_state.get("Error", ""), } except aiodocker.exceptions.DockerError as err: - logger.warning( - "An unexpected Docker error occurred:\n%s", - str(traceback.format_exc()), - ) - raise HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, detail=err.message - ) from err + _raise_from_docker_error(err) return results @@ -225,10 +230,7 @@ async def get_container_logs( """ Returns the logs of a given container if found """ # TODO: remove from here and dump directly into the logs of this service # do this in PR#1887 - if id not in shared_store.container_names: - message = f"No container '{id}' was started. Started containers '{shared_store.container_names}'" - logger.warning(message) - raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=message) + _raise_if_container_is_missing(id, shared_store.container_names) with docker_client() as docker: try: @@ -247,12 +249,7 @@ async def get_container_logs( container_logs: str = await container_instance.log(**args) return container_logs except aiodocker.exceptions.DockerError as err: - logger.warning( - "An unexpected Docker error occurred:\n%s", str(traceback.format_exc()) - ) - raise HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, detail=err.message - ) from err + _raise_from_docker_error(err) @containers_router.get( @@ -267,11 +264,7 @@ async def inspect_container( id: str, shared_store: SharedStore = Depends(get_shared_store) ) -> Dict[str, Any]: """ Returns information about the container, like docker inspect command """ - - if id not in shared_store.container_names: - message = f"No container '{id}' was started. Started containers '{shared_store.container_names}'" - logger.warning(message) - raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=message) + _raise_if_container_is_missing(id, shared_store.container_names) with docker_client() as docker: try: @@ -279,12 +272,7 @@ async def inspect_container( inspect_result: Dict[str, Any] = await container_instance.show() return inspect_result except aiodocker.exceptions.DockerError as err: - logger.warning( - "An unexpected Docker error occurred:\n%s", str(traceback.format_exc()) - ) - raise HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, detail=err.message - ) from err + _raise_from_docker_error(err) @containers_router.delete( @@ -299,10 +287,7 @@ async def inspect_container( async def remove_container( id: str, shared_store: SharedStore = Depends(get_shared_store) ) -> Optional[Dict[str, Any]]: - if id not in shared_store.container_names: - message = f"No container '{id}' was started. Started containers '{shared_store.container_names}'" - logger.warning(message) - raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=message) + _raise_if_container_is_missing(id, shared_store.container_names) with docker_client() as docker: try: @@ -310,12 +295,7 @@ async def remove_container( await container_instance.delete() return None except aiodocker.exceptions.DockerError as err: - logger.warning( - "An unexpected Docker error occurred:\n%s", str(traceback.format_exc()) - ) - raise HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, detail=err.message - ) from err + _raise_from_docker_error(err) __all__ = ["containers_router"] diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 25602173e0c..eae48c956d7 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -276,23 +276,17 @@ def _expected_error_string(container: str) -> Dict[str, str]: for container in not_started_containers: # get container logs response = await test_client.get(f"/{api_vtag}/containers/{container}/logs") - assert ( - response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR - ), response.text + assert response.status_code == status.HTTP_404_NOT_FOUND, response.text assert response.json() == _expected_error_string(container) # inspect container response = await test_client.get(f"/{api_vtag}/containers/{container}:inspect") - assert ( - response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR - ), response.text + assert response.status_code == status.HTTP_404_NOT_FOUND, response.text assert response.json() == _expected_error_string(container) # delete container response = await test_client.delete(f"/{api_vtag}/containers/{container}") - assert ( - response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR - ), response.text + assert response.status_code == status.HTTP_404_NOT_FOUND, response.text assert response.json() == _expected_error_string(container) From e81a75f5edbf947a793eaaf250dbc2790d41312f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 15:36:43 +0200 Subject: [PATCH 075/102] regenerated openapi.json spec --- services/dynamic-sidecar/openapi.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index bcec37a6dbe..28cb26f364b 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -156,7 +156,7 @@ } } }, - "/v1/containers/{id}:logs": { + "/v1/containers/{id}/logs": { "get": { "tags": [ "containers" From 959e10667dba918b8e3dbe4f9250fef80e798c56 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 15:39:42 +0200 Subject: [PATCH 076/102] codestyle :\ --- .../src/simcore_service_dynamic_sidecar/api/containers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index b53df3e2f1c..9ee572f0bc1 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -2,7 +2,7 @@ import traceback # pylint: disable=redefined-builtin -from typing import Any, Dict, Optional, Union, List +from typing import Any, Dict, List, Optional, Union import aiodocker from fastapi import ( @@ -250,6 +250,7 @@ async def get_container_logs( return container_logs except aiodocker.exceptions.DockerError as err: _raise_from_docker_error(err) + return None @containers_router.get( @@ -273,6 +274,7 @@ async def inspect_container( return inspect_result except aiodocker.exceptions.DockerError as err: _raise_from_docker_error(err) + return None @containers_router.delete( @@ -296,6 +298,7 @@ async def remove_container( return None except aiodocker.exceptions.DockerError as err: _raise_from_docker_error(err) + return None __all__ = ["containers_router"] From ca5c02ef993e24be3db0f4edd856e295597b5430 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 16:45:37 +0200 Subject: [PATCH 077/102] merged /containers and /containers:inspect --- services/dynamic-sidecar/openapi.json | 63 +++++++++---------- .../api/containers.py | 59 +++++++---------- .../tests/unit/test_api_containers.py | 27 +++++--- 3 files changed, 71 insertions(+), 78 deletions(-) diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index 28cb26f364b..94e54b76433 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -31,9 +31,23 @@ "tags": [ "containers" ], - "summary": "Containers Docker Status", - "description": "Returns the status of the containers ", - "operationId": "containers_docker_status_v1_containers_get", + "summary": "Containers Docker Inspect", + "description": "Returns entire docker inspect data, if only_state is True,\nthe status of the containers is returned", + "operationId": "containers_docker_inspect_v1_containers_get", + "parameters": [ + { + "description": "if True only show the status of the container", + "required": false, + "schema": { + "title": "Only Status", + "type": "boolean", + "description": "if True only show the status of the container", + "default": true + }, + "name": "only_status", + "in": "query" + } + ], "responses": { "200": { "description": "Successful Response", @@ -45,6 +59,16 @@ }, "500": { "description": "Errors in container" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } } } }, @@ -133,29 +157,6 @@ } } }, - "/v1/containers:inspect": { - "get": { - "tags": [ - "containers" - ], - "summary": "Containers Inspect", - "description": "Returns information about the container, like docker inspect command ", - "operationId": "containers_inspect_v1_containers_inspect_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - }, - "500": { - "description": "Erros in container" - } - } - } - }, "/v1/containers/{id}/logs": { "get": { "tags": [ @@ -220,7 +221,7 @@ } } }, - "500": { + "404": { "description": "Container does not exists" }, "422": { @@ -236,14 +237,14 @@ } } }, - "/v1/containers/{id}:inspect": { + "/v1/containers/{id}": { "get": { "tags": [ "containers" ], "summary": "Inspect Container", "description": "Returns information about the container, like docker inspect command ", - "operationId": "inspect_container_v1_containers__id__inspect_get", + "operationId": "inspect_container_v1_containers__id__get", "parameters": [ { "required": true, @@ -278,9 +279,7 @@ } } } - } - }, - "/v1/containers/{id}": { + }, "delete": { "tags": [ "containers" diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 9ee572f0bc1..0e44c52b435 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -140,38 +140,33 @@ async def runs_docker_compose_down( @containers_router.get( - "/containers:inspect", + "/containers", responses={ - status.HTTP_500_INTERNAL_SERVER_ERROR: {"description": "Erros in container"} + status.HTTP_500_INTERNAL_SERVER_ERROR: {"description": "Errors in container"} }, ) -async def containers_inspect( +async def containers_docker_inspect( + only_status: bool = Query( + True, description="if True only show the status of the container" + ), shared_store: SharedStore = Depends(get_shared_store), ) -> Dict[str, Any]: - """ Returns information about the container, like docker inspect command """ - with docker_client() as docker: - results = {} + """ + Returns entire docker inspect data, if only_state is True, + the status of the containers is returned + """ - for container in shared_store.container_names: - try: - container_instance = await docker.containers.get(container) - results[container] = await container_instance.show() - except aiodocker.exceptions.DockerError as err: - _raise_from_docker_error(err) - - return results + def _format_result(container_inspect: Dict[str, Any]) -> Dict[str, Any]: + if only_status: + container_state = container_inspect.get("State", {}) + # pending is another fake state use to share more information with the frontend + return { + "Status": container_state.get("Status", "pending"), + "Error": container_state.get("Error", ""), + } -@containers_router.get( - "/containers", - responses={ - status.HTTP_500_INTERNAL_SERVER_ERROR: {"description": "Errors in container"} - }, -) -async def containers_docker_status( - shared_store: SharedStore = Depends(get_shared_store), -) -> Dict[str, Any]: - """ Returns the status of the containers """ + return container_inspect with docker_client() as docker: container_names = shared_store.container_names @@ -187,13 +182,7 @@ async def containers_docker_status( try: container_instance = await docker.containers.get(container) container_inspect = await container_instance.show() - container_state = container_inspect.get("State", {}) - - # pending is another fake state use to share more information with the frontend - results[container] = { - "Status": container_state.get("Status", "pending"), - "Error": container_state.get("Error", ""), - } + results[container] = _format_result(container_inspect) except aiodocker.exceptions.DockerError as err: _raise_from_docker_error(err) @@ -202,11 +191,7 @@ async def containers_docker_status( @containers_router.get( "/containers/{id}/logs", - responses={ - status.HTTP_500_INTERNAL_SERVER_ERROR: { - "description": "Container does not exists" - } - }, + responses={status.HTTP_404_NOT_FOUND: {"description": "Container does not exists"}}, ) async def get_container_logs( id: str, @@ -254,7 +239,7 @@ async def get_container_logs( @containers_router.get( - "/containers/{id}:inspect", + "/containers/{id}", responses={ status.HTTP_500_INTERNAL_SERVER_ERROR: { "description": "Container does not exist" diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index eae48c956d7..06a613b4c55 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -151,25 +151,34 @@ async def test_containers_get( ) -> None: response = await test_client.get(f"/{api_vtag}/containers") assert response.status_code == status.HTTP_200_OK, response.text - assert set(json.loads(response.text)) == set(started_containers) + response_dict = json.loads(response.text) + assert set(response_dict) == set(started_containers) + for entry in response_dict.values(): + assert "Status" not in entry + assert "Error" not in entry -async def test_containers_inspect( + +async def test_containers_get_status( test_client: TestClient, started_containers: List[str] ) -> None: response = await test_client.get( - f"/{api_vtag}/containers:inspect", - query_string=dict(container_names=started_containers), + f"/{api_vtag}/containers", query_string=dict(only_status=True) ) assert response.status_code == status.HTTP_200_OK, response.text - assert set(json.loads(response.text).keys()) == set(started_containers) + + response_dict = json.loads(response.text) + assert set(response_dict) == set(started_containers) + for entry in response_dict.values(): + assert "Status" in entry + assert "Error" in entry async def test_containers_inspect_docker_error( test_client: TestClient, started_containers: List[str], mock_containers_get: None ) -> None: response = await test_client.get( - f"/{api_vtag}/containers:inspect", + f"/{api_vtag}/containers", query_string=dict(container_names=started_containers), ) assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text @@ -242,7 +251,7 @@ async def test_container_inspect_logs_remove( assert response.status_code == status.HTTP_200_OK, response.text # inspect container - response = await test_client.get(f"/{api_vtag}/containers/{container}:inspect") + response = await test_client.get(f"/{api_vtag}/containers/{container}") assert response.status_code == status.HTTP_200_OK, response.text parsed_response = response.json() assert parsed_response["Name"] == f"/{container}" @@ -280,7 +289,7 @@ def _expected_error_string(container: str) -> Dict[str, str]: assert response.json() == _expected_error_string(container) # inspect container - response = await test_client.get(f"/{api_vtag}/containers/{container}:inspect") + response = await test_client.get(f"/{api_vtag}/containers/{container}") assert response.status_code == status.HTTP_404_NOT_FOUND, response.text assert response.json() == _expected_error_string(container) @@ -307,7 +316,7 @@ def _expected_error_string() -> Dict[str, str]: assert response.json() == _expected_error_string() # inspect container - response = await test_client.get(f"/{api_vtag}/containers/{container}:inspect") + response = await test_client.get(f"/{api_vtag}/containers/{container}") assert ( response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR ), response.text From ce31061f61b2ee7e9c7d0feeb79df4d6ad0007a8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 16:48:39 +0200 Subject: [PATCH 078/102] wrong default value --- .../src/simcore_service_dynamic_sidecar/api/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 0e44c52b435..1220c73637d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -147,7 +147,7 @@ async def runs_docker_compose_down( ) async def containers_docker_inspect( only_status: bool = Query( - True, description="if True only show the status of the container" + False, description="if True only show the status of the container" ), shared_store: SharedStore = Depends(get_shared_store), ) -> Dict[str, Any]: From c742c6b48261a44b65f871523f6b94e134127e40 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 16:53:05 +0200 Subject: [PATCH 079/102] test clenup --- .../tests/unit/test_api_containers.py | 51 ++++++------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 06a613b4c55..659a8930ff5 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -146,15 +146,22 @@ async def test_containers_down_missing_spec( } +def assert_keys_exist(result: Dict[str, Any]) -> bool: + for entry in result.values(): + assert "Status" in entry + assert "Error" in entry + return True + + async def test_containers_get( test_client: TestClient, started_containers: List[str] ) -> None: response = await test_client.get(f"/{api_vtag}/containers") assert response.status_code == status.HTTP_200_OK, response.text - response_dict = json.loads(response.text) - assert set(response_dict) == set(started_containers) - for entry in response_dict.values(): + decoded_response = json.loads(response.text) + assert set(decoded_response) == set(started_containers) + for entry in decoded_response.values(): assert "Status" not in entry assert "Error" not in entry @@ -167,43 +174,18 @@ async def test_containers_get_status( ) assert response.status_code == status.HTTP_200_OK, response.text - response_dict = json.loads(response.text) - assert set(response_dict) == set(started_containers) - for entry in response_dict.values(): - assert "Status" in entry - assert "Error" in entry + decoded_response = json.loads(response.text) + assert set(decoded_response) == set(started_containers) + assert assert_keys_exist(decoded_response) is True async def test_containers_inspect_docker_error( test_client: TestClient, started_containers: List[str], mock_containers_get: None ) -> None: - response = await test_client.get( - f"/{api_vtag}/containers", - query_string=dict(container_names=started_containers), - ) + response = await test_client.get(f"/{api_vtag}/containers") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text -def assert_keys_exist(result: Dict[str, Any]) -> bool: - for entry in result.values(): - assert "Status" in entry - assert "Error" in entry - return True - - -async def test_containers_docker_status( - test_client: TestClient, started_containers: List[str] -) -> None: - response = await test_client.get( - f"/{api_vtag}/containers", - query_string=dict(container_names=started_containers), - ) - assert response.status_code == status.HTTP_200_OK, response.text - decoded_response = json.loads(response.text) - assert set(decoded_response) == set(started_containers) - assert assert_keys_exist(decoded_response) is True - - async def test_containers_docker_status_pulling_containers( test_client: TestClient, started_containers: List[str] ) -> None: @@ -220,10 +202,7 @@ def mark_pulling(shared_store: SharedStore) -> Generator[None, None, None]: with mark_pulling(shared_store): assert shared_store.is_pulling_containers is True - response = await test_client.get( - f"/{api_vtag}/containers", - query_string=dict(container_names=started_containers), - ) + response = await test_client.get(f"/{api_vtag}/containers") assert response.status_code == status.HTTP_200_OK, response.text decoded_response = json.loads(response.text) assert assert_keys_exist(decoded_response) is True From 08fc26f675ff22533a9694338f911103c0ad2c54 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 16:54:59 +0200 Subject: [PATCH 080/102] updated openapi.json --- services/dynamic-sidecar/openapi.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index 94e54b76433..64112c12765 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -42,7 +42,7 @@ "title": "Only Status", "type": "boolean", "description": "if True only show the status of the container", - "default": true + "default": false }, "name": "only_status", "in": "query" From 88fbeacd15b7afb847bf393a49f659a269bd2cf2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 16:59:14 +0200 Subject: [PATCH 081/102] clarify comment --- .../src/simcore_service_dynamic_sidecar/api/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 1220c73637d..0d1340b40de 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -33,7 +33,7 @@ async def task_docker_compose_up( settings: DynamicSidecarSettings, shared_store: SharedStore, ) -> None: - # --no-build might be a security risk building is disabled + # building is a security risk hence is disabled via "--no-build" parameter command = ( "docker-compose --project-name {project} --file {file_path} " "up --no-build --detach" From 8a8eb2fc28d3fb50ccec7cdcadd95d9cda7d0758 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 16:59:23 +0200 Subject: [PATCH 082/102] removed unused API --- services/dynamic-sidecar/openapi.json | 38 +------------------ .../api/containers.py | 30 +-------------- .../tests/unit/test_api_containers.py | 17 --------- 3 files changed, 2 insertions(+), 83 deletions(-) diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index 64112c12765..809e73f759d 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -265,43 +265,7 @@ } } }, - "500": { - "description": "Container does not exist" - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - }, - "delete": { - "tags": [ - "containers" - ], - "summary": "Remove Container", - "operationId": "remove_container_v1_containers__id__delete", - "parameters": [ - { - "required": true, - "schema": { - "title": "Id", - "type": "string" - }, - "name": "id", - "in": "path" - } - ], - "responses": { - "204": { - "description": "Successful Response" - }, - "500": { + "404": { "description": "Container does not exist" }, "422": { diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 0d1340b40de..2725102db6f 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -240,11 +240,7 @@ async def get_container_logs( @containers_router.get( "/containers/{id}", - responses={ - status.HTTP_500_INTERNAL_SERVER_ERROR: { - "description": "Container does not exist" - } - }, + responses={status.HTTP_404_NOT_FOUND: {"description": "Container does not exist"}}, ) async def inspect_container( id: str, shared_store: SharedStore = Depends(get_shared_store) @@ -262,28 +258,4 @@ async def inspect_container( return None -@containers_router.delete( - "/containers/{id}", - status_code=status.HTTP_204_NO_CONTENT, - responses={ - status.HTTP_500_INTERNAL_SERVER_ERROR: { - "description": "Container does not exist" - } - }, -) -async def remove_container( - id: str, shared_store: SharedStore = Depends(get_shared_store) -) -> Optional[Dict[str, Any]]: - _raise_if_container_is_missing(id, shared_store.container_names) - - with docker_client() as docker: - try: - container_instance = await docker.containers.get(id) - await container_instance.delete() - return None - except aiodocker.exceptions.DockerError as err: - _raise_from_docker_error(err) - return None - - __all__ = ["containers_router"] diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 659a8930ff5..f06a54e7495 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -235,11 +235,6 @@ async def test_container_inspect_logs_remove( parsed_response = response.json() assert parsed_response["Name"] == f"/{container}" - # delete container - response = await test_client.delete(f"/{api_vtag}/containers/{container}") - assert response.status_code == status.HTTP_204_NO_CONTENT, response.text - assert json.loads(response.text) is None - async def test_container_logs_with_timestamps( test_client: TestClient, started_containers: List[str] @@ -272,11 +267,6 @@ def _expected_error_string(container: str) -> Dict[str, str]: assert response.status_code == status.HTTP_404_NOT_FOUND, response.text assert response.json() == _expected_error_string(container) - # delete container - response = await test_client.delete(f"/{api_vtag}/containers/{container}") - assert response.status_code == status.HTTP_404_NOT_FOUND, response.text - assert response.json() == _expected_error_string(container) - async def test_container_docker_error( test_client: TestClient, @@ -300,10 +290,3 @@ def _expected_error_string() -> Dict[str, str]: response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR ), response.text assert response.json() == _expected_error_string() - - # delete container - response = await test_client.delete(f"/{api_vtag}/containers/{container}") - assert ( - response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR - ), response.text - assert response.json() == _expected_error_string() From b37135728821090af9fb44eb3e45fa51c7ec67dc Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 17:08:13 +0200 Subject: [PATCH 083/102] removed command timeout when posting to createing services via docker-compose --- services/dynamic-sidecar/openapi.json | 23 ---------------- .../api/containers.py | 18 +++---------- .../validation.py | 4 +-- .../tests/unit/test_api_containers.py | 27 ++++--------------- 4 files changed, 10 insertions(+), 62 deletions(-) diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index 809e73f759d..1169a0f73bd 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -79,19 +79,6 @@ "summary": "Runs Docker Compose Up", "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", "operationId": "runs_docker_compose_up_v1_containers_post", - "parameters": [ - { - "description": "docker-compose up also pulls images, this value needs to be big enough to account for that", - "required": true, - "schema": { - "title": "Command Timeout", - "type": "number", - "description": "docker-compose up also pulls images, this value needs to be big enough to account for that" - }, - "name": "command_timeout", - "in": "query" - } - ], "responses": { "201": { "description": "Successful Response", @@ -100,16 +87,6 @@ "schema": {} } } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } } } } diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 2725102db6f..d41fbf764f3 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -29,7 +29,6 @@ async def task_docker_compose_up( - command_timeout: float, settings: DynamicSidecarSettings, shared_store: SharedStore, ) -> None: @@ -42,7 +41,7 @@ async def task_docker_compose_up( settings=settings, file_content=shared_store.compose_spec, command=command, - command_timeout=command_timeout, + command_timeout=None, ) message = f"Finished {command} with output\n{stdout}" @@ -76,13 +75,6 @@ def _raise_from_docker_error(error: aiodocker.exceptions.DockerError) -> None: async def runs_docker_compose_up( request: Request, background_tasks: BackgroundTasks, - command_timeout: float = Query( - ..., - description=( - "docker-compose up also pulls images, this value " - "needs to be big enough to account for that" - ), - ), settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), ) -> Optional[Dict[str, Any]]: @@ -93,9 +85,7 @@ async def runs_docker_compose_up( try: shared_store.compose_spec = await validate_compose_spec( - settings=settings, - compose_file_content=body_as_text, - command_timeout=command_timeout, + settings=settings, compose_file_content=body_as_text ) shared_store.container_names = assemble_container_names( shared_store.compose_spec @@ -105,9 +95,7 @@ async def runs_docker_compose_up( raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e # run docker-compose in a background queue and return early - background_tasks.add_task( - task_docker_compose_up, command_timeout, settings, shared_store - ) + background_tasks.add_task(task_docker_compose_up, settings, shared_store) return None diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py index a71ce777b13..c83a01c0ee2 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py @@ -145,7 +145,7 @@ def _inject_backend_networking( async def validate_compose_spec( - settings: DynamicSidecarSettings, compose_file_content: str, command_timeout: float + settings: DynamicSidecarSettings, compose_file_content: str ) -> str: """ Validates what looks like a docker compose spec and injects @@ -233,7 +233,7 @@ async def validate_compose_spec( settings=settings, file_content=compose_spec, command=command, - command_timeout=command_timeout, + command_timeout=None, ) if not finished_without_errors: message = ( diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index f06a54e7495..e23675880a4 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -58,11 +58,7 @@ async def started_containers(test_client: TestClient, compose_spec: str) -> List await assert_compose_spec_pulled(compose_spec, settings) # start containers - response = await test_client.post( - f"/{api_vtag}/containers", - query_string=dict(command_timeout=10.0), - data=compose_spec, - ) + response = await test_client.post(f"/{api_vtag}/containers", data=compose_spec) assert response.status_code == status.HTTP_201_CREATED, response.text assert json.loads(response.text) is None @@ -82,20 +78,13 @@ async def test_compose_up( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: - response = await test_client.post( - f"/{api_vtag}/containers", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - data=compose_spec, - ) + response = await test_client.post(f"/{api_vtag}/containers", data=compose_spec) assert response.status_code == status.HTTP_201_CREATED, response.text assert json.loads(response.text) is None async def test_compose_up_spec_not_provided(test_client: TestClient) -> None: - response = await test_client.post( - f"/{api_vtag}/containers", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - ) + response = await test_client.post(f"/{api_vtag}/containers") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text assert json.loads(response.text) == {"detail": "\nProvided yaml is not valid!"} @@ -103,9 +92,7 @@ async def test_compose_up_spec_not_provided(test_client: TestClient) -> None: async def test_compose_up_spec_invalid(test_client: TestClient) -> None: invalid_compose_spec = Faker().text() response = await test_client.post( - f"/{api_vtag}/containers", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - data=invalid_compose_spec, + f"/{api_vtag}/containers", data=invalid_compose_spec ) assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text assert "Provided yaml is not valid!" in response.text @@ -117,11 +104,7 @@ async def test_containers_down_after_starting( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: # store spec first - response = await test_client.post( - f"/{api_vtag}/containers", - query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), - data=compose_spec, - ) + response = await test_client.post(f"/{api_vtag}/containers", data=compose_spec) assert response.status_code == status.HTTP_201_CREATED, response.text assert json.loads(response.text) is None From 0723f8aa611f002ccb87252ccef1b367715719dc Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 17:09:23 +0200 Subject: [PATCH 084/102] removing unecessary timeouts --- services/dynamic-sidecar/tests/unit/test_api_containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index e23675880a4..4afd59730dd 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -15,7 +15,7 @@ from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command from simcore_service_dynamic_sidecar.shared_store import SharedStore -DEFAULT_COMMAND_TIMEOUT = 10.0 +DEFAULT_COMMAND_TIMEOUT = 5.0 pytestmark = pytest.mark.asyncio @@ -46,7 +46,7 @@ async def assert_compose_spec_pulled( settings=settings, file_content=compose_spec, command=command, - command_timeout=10.0, + command_timeout=None, ) assert finished_without_errors is True, stdout From faf26f136a283e51f871f7d8d4f353575ec62339 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 17:33:08 +0200 Subject: [PATCH 085/102] forgot to implement it --- .../src/simcore_service_dynamic_sidecar/api/containers.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index d41fbf764f3..c25e4a85124 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -209,16 +209,10 @@ async def get_container_logs( try: container_instance = await docker.containers.get(id) - args = dict(stdout=True, stderr=True) + args = dict(stdout=True, stderr=True, since=since, until=until) if timestamps: args["timestamps"] = True - if since or until: - raise HTTPException( - status.HTTP_501_NOT_IMPLEMENTED, - detail="since and until options are still not implemented", - ) - container_logs: str = await container_instance.log(**args) return container_logs except aiodocker.exceptions.DockerError as err: From d2f1146273fc6dec7fe31f64cfc5ce001bdbebcf Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 17:33:59 +0200 Subject: [PATCH 086/102] after the request is accepted the list of container names is returned --- .../simcore_service_dynamic_sidecar/api/containers.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index c25e4a85124..ab82d68d9d3 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -71,13 +71,13 @@ def _raise_from_docker_error(error: aiodocker.exceptions.DockerError) -> None: ) from error -@containers_router.post("/containers", status_code=status.HTTP_201_CREATED) +@containers_router.post("/containers", status_code=status.HTTP_202_ACCEPTED) async def runs_docker_compose_up( request: Request, background_tasks: BackgroundTasks, settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), -) -> Optional[Dict[str, Any]]: +) -> Union[List[str], Dict[str, Any]]: """ Expects the docker-compose spec as raw-body utf-8 encoded text """ # stores the compose spec after validation @@ -96,7 +96,8 @@ async def runs_docker_compose_up( # run docker-compose in a background queue and return early background_tasks.add_task(task_docker_compose_up, settings, shared_store) - return None + + return shared_store.container_names @containers_router.post("/containers:down", response_class=PlainTextResponse) @@ -111,7 +112,7 @@ async def runs_docker_compose_down( stored_compose_content = shared_store.compose_spec if stored_compose_content is None: raise HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, + status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No spec for docker-compose down was found", ) From 886727c5db04f1006aeb40760486e08a8a65c438 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 17:34:35 +0200 Subject: [PATCH 087/102] removed flag which is no longer used --- .../src/simcore_service_dynamic_sidecar/api/containers.py | 7 +------ .../src/simcore_service_dynamic_sidecar/shared_store.py | 3 --- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index ab82d68d9d3..2649f5d448d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -2,7 +2,7 @@ import traceback # pylint: disable=redefined-builtin -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Union import aiodocker from fastapi import ( @@ -160,11 +160,6 @@ def _format_result(container_inspect: Dict[str, Any]) -> Dict[str, Any]: with docker_client() as docker: container_names = shared_store.container_names - # if containers are being pulled, return pulling (fake status) - if shared_store.is_pulling_containers: - # pulling is a fake state use to share more information with the frontend - return {x: {"Status": "pulling", "Error": ""} for x in container_names} - results = {} for container in container_names: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py index ebf5af7c4ff..a956faddc27 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py @@ -10,6 +10,3 @@ class SharedStore(BaseModel): container_names: List[str] = Field( [], description="stores the container names from the compose_spec" ) - is_pulling_containers: bool = Field( - False, description="set to True while the containers are being pulled" - ) From cffe2ab9e1cc2f9a545ca8a45b36c113050f54ce Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 17:34:42 +0200 Subject: [PATCH 088/102] fixed tests --- services/dynamic-sidecar/openapi.json | 2 +- .../tests/unit/test_api_containers.py | 48 +++++-------------- 2 files changed, 13 insertions(+), 37 deletions(-) diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index 1169a0f73bd..8dab419ca65 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -80,7 +80,7 @@ "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", "operationId": "runs_docker_compose_up_v1_containers_post", "responses": { - "201": { + "202": { "description": "Successful Response", "content": { "application/json": { diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 4afd59730dd..54ea846bb79 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -59,12 +59,12 @@ async def started_containers(test_client: TestClient, compose_spec: str) -> List # start containers response = await test_client.post(f"/{api_vtag}/containers", data=compose_spec) - assert response.status_code == status.HTTP_201_CREATED, response.text - assert json.loads(response.text) is None + assert response.status_code == status.HTTP_202_ACCEPTED, response.text shared_store: SharedStore = test_client.application.state.shared_store container_names = shared_store.container_names assert len(container_names) == 2 + assert json.loads(response.text) == container_names return container_names @@ -79,8 +79,10 @@ async def test_compose_up( ) -> None: response = await test_client.post(f"/{api_vtag}/containers", data=compose_spec) - assert response.status_code == status.HTTP_201_CREATED, response.text - assert json.loads(response.text) is None + assert response.status_code == status.HTTP_202_ACCEPTED, response.text + shared_store: SharedStore = test_client.application.state.shared_store + container_names = shared_store.container_names + assert json.loads(response.text) == container_names async def test_compose_up_spec_not_provided(test_client: TestClient) -> None: @@ -105,8 +107,10 @@ async def test_containers_down_after_starting( ) -> None: # store spec first response = await test_client.post(f"/{api_vtag}/containers", data=compose_spec) - assert response.status_code == status.HTTP_201_CREATED, response.text - assert json.loads(response.text) is None + assert response.status_code == status.HTTP_202_ACCEPTED, response.text + shared_store: SharedStore = test_client.application.state.shared_store + container_names = shared_store.container_names + assert json.loads(response.text) == container_names response = await test_client.post( f"/{api_vtag}/containers:down", @@ -123,7 +127,7 @@ async def test_containers_down_missing_spec( f"/{api_vtag}/containers:down", query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) - assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY, response.text assert json.loads(response.text) == { "detail": "No spec for docker-compose down was found" } @@ -169,38 +173,10 @@ async def test_containers_inspect_docker_error( assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text -async def test_containers_docker_status_pulling_containers( - test_client: TestClient, started_containers: List[str] -) -> None: - @contextmanager - def mark_pulling(shared_store: SharedStore) -> Generator[None, None, None]: - try: - shared_store.is_pulling_containers = True - yield - finally: - shared_store.is_pulling_containers = False - - shared_store: SharedStore = test_client.application.state.shared_store - - with mark_pulling(shared_store): - assert shared_store.is_pulling_containers is True - - response = await test_client.get(f"/{api_vtag}/containers") - assert response.status_code == status.HTTP_200_OK, response.text - decoded_response = json.loads(response.text) - assert assert_keys_exist(decoded_response) is True - - for entry in decoded_response.values(): - assert entry["Status"] == "pulling" - - async def test_containers_docker_status_docker_error( test_client: TestClient, started_containers: List[str], mock_containers_get: None ) -> None: - response = await test_client.get( - f"/{api_vtag}/containers", - query_string=dict(container_names=started_containers), - ) + response = await test_client.get(f"/{api_vtag}/containers") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text From ac9fd174178ad4c2ebb8e7e51be5b2342511b857 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 17:43:15 +0200 Subject: [PATCH 089/102] moved ApplicationHealth and codestyle fixes --- .../src/simcore_service_dynamic_sidecar/models/__init__.py | 3 +++ .../{models.py => models/application_health.py} | 0 services/dynamic-sidecar/tests/unit/test_api_containers.py | 3 +-- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/__init__.py rename services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/{models.py => models/application_health.py} (100%) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/__init__.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/__init__.py new file mode 100644 index 00000000000..ce4c96a67c8 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/__init__.py @@ -0,0 +1,3 @@ +from .application_health import ApplicationHealth + +__all__ = ["ApplicationHealth"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/application_health.py similarity index 100% rename from services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models.py rename to services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/application_health.py diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 54ea846bb79..3407c646010 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -3,8 +3,7 @@ import json -from contextlib import contextmanager -from typing import Any, Dict, Generator, List +from typing import Any, Dict, List import pytest from async_asgi_testclient import TestClient From 703b7571b49b97ac24f43035d1e59c7a0e4b45e8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 17:54:06 +0200 Subject: [PATCH 090/102] added test to check it can run twice the same compos spec with different project-name --- .../tests/unit/test_api_containers.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 3407c646010..c6745b73a50 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -3,7 +3,7 @@ import json -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple import pytest from async_asgi_testclient import TestClient @@ -73,6 +73,16 @@ def not_started_containers() -> List[str]: return [f"missing-container-{i}" for i in range(5)] +async def test_start_same_space_twice( + test_client: TestClient, compose_spec: str +) -> None: + settings: DynamicSidecarSettings = test_client.application.state.settings + await assert_compose_spec_pulled(compose_spec, settings) + + settings.compose_namespace = "test_name_space" + await assert_compose_spec_pulled(compose_spec, settings) + + async def test_compose_up( test_client: TestClient, compose_spec: Dict[str, Any] ) -> None: From f7c25bcea8cb3a385f6ce977440cb461afcecade Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 23 Apr 2021 17:55:02 +0200 Subject: [PATCH 091/102] fixed codestyle --- services/dynamic-sidecar/tests/unit/test_api_containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index c6745b73a50..fa97a9a7670 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -3,7 +3,7 @@ import json -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List import pytest from async_asgi_testclient import TestClient From 3e5246e9cbdbb7bfb99d342c4820aa1aeb70a7a6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Apr 2021 08:24:21 +0200 Subject: [PATCH 092/102] setting default and documentation to argument --- .../src/simcore_service_dynamic_sidecar/api/containers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 2649f5d448d..a55bbcfbec6 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -102,7 +102,9 @@ async def runs_docker_compose_up( @containers_router.post("/containers:down", response_class=PlainTextResponse) async def runs_docker_compose_down( - command_timeout: float, + command_timeout: float = Query( + 10.0, description="docker-compose down command timeout default" + ), settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), ) -> Union[str, Dict[str, Any]]: From 30d3ac14e06df6a32f36004f1b83fc83837d42c6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Apr 2021 08:24:28 +0200 Subject: [PATCH 093/102] updated oepnapi.json --- services/dynamic-sidecar/openapi.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index 8dab419ca65..a98370a37a9 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -101,10 +101,13 @@ "operationId": "runs_docker_compose_down_v1_containers_down_post", "parameters": [ { - "required": true, + "description": "docker-compose down command timeout default", + "required": false, "schema": { "title": "Command Timeout", - "type": "number" + "type": "number", + "description": "docker-compose down command timeout default", + "default": 10.0 }, "name": "command_timeout", "in": "query" From 1f74e229462dcb9170c57df4c0b83fe29478de6d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Apr 2021 09:03:44 +0200 Subject: [PATCH 094/102] fixed test to properly work based on workspace --- .../tests/unit/test_api_containers.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index fa97a9a7670..11933fad9f8 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -2,6 +2,7 @@ # pylint: disable=unused-argument +import asyncio import json from typing import Any, Dict, List @@ -13,6 +14,7 @@ from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command from simcore_service_dynamic_sidecar.shared_store import SharedStore +from simcore_service_dynamic_sidecar.utils import async_command DEFAULT_COMMAND_TIMEOUT = 5.0 @@ -32,6 +34,16 @@ def compose_spec() -> str: ) +async def _docker_ps_a_container_names() -> List[str]: + command = 'docker ps -a --format "{{.Names}}"' + finished_without_errors, stdout = await async_command( + command=command, command_timeout=None + ) + + assert finished_without_errors is True, stdout + return stdout.split("\n") + + async def assert_compose_spec_pulled( compose_spec: str, settings: DynamicSidecarSettings ) -> None: @@ -50,6 +62,15 @@ async def assert_compose_spec_pulled( assert finished_without_errors is True, stdout + dict_compose_spec = json.loads(compose_spec) + expected_services_count = len(dict_compose_spec["services"]) + + docker_ps_names = await _docker_ps_a_container_names() + started_containers = [ + x for x in docker_ps_names if x.startswith(settings.compose_namespace) + ] + assert len(started_containers) == expected_services_count + @pytest.fixture async def started_containers(test_client: TestClient, compose_spec: str) -> List[str]: @@ -77,9 +98,10 @@ async def test_start_same_space_twice( test_client: TestClient, compose_spec: str ) -> None: settings: DynamicSidecarSettings = test_client.application.state.settings + settings.compose_namespace = "test_name_space_1" await assert_compose_spec_pulled(compose_spec, settings) - settings.compose_namespace = "test_name_space" + settings.compose_namespace = "test_name_space_2" await assert_compose_spec_pulled(compose_spec, settings) From 9cec746bd7e8850fff3367acd7edd5dec3ca9b55 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Apr 2021 09:22:33 +0200 Subject: [PATCH 095/102] refactoring application models structure --- .../src/simcore_service_dynamic_sidecar/api/containers.py | 2 +- .../src/simcore_service_dynamic_sidecar/api/health.py | 2 +- .../src/simcore_service_dynamic_sidecar/application.py | 4 ++-- .../src/simcore_service_dynamic_sidecar/dependencies.py | 4 ++-- .../src/simcore_service_dynamic_sidecar/models/__init__.py | 3 --- .../models/domains/__init__.py | 0 .../{ => models/domains}/shared_store.py | 0 .../models/schemas/__init__.py | 0 .../models/{ => schemas}/application_health.py | 0 .../src/simcore_service_dynamic_sidecar/shared_handlers.py | 2 +- services/dynamic-sidecar/tests/conftest.py | 2 +- services/dynamic-sidecar/tests/unit/test_api_containers.py | 3 +-- services/dynamic-sidecar/tests/unit/test_api_health.py | 4 +++- 13 files changed, 12 insertions(+), 14 deletions(-) create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/domains/__init__.py rename services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/{ => models/domains}/shared_store.py (100%) create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/__init__.py rename services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/{ => schemas}/application_health.py (100%) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index a55bbcfbec6..0a535e4cf77 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -17,9 +17,9 @@ from fastapi.responses import PlainTextResponse from ..dependencies import get_settings, get_shared_store +from ..models.domains.shared_store import SharedStore from ..settings import DynamicSidecarSettings from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command -from ..shared_store import SharedStore from ..utils import assemble_container_names, docker_client from ..validation import InvalidComposeSpec, validate_compose_spec diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py index 02c81bd9e9f..c37a7826c5f 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py @@ -1,7 +1,7 @@ from fastapi import APIRouter, Depends, HTTPException, status from ..dependencies import get_application_health -from ..models import ApplicationHealth +from ..models.schemas.application_health import ApplicationHealth health_router = APIRouter() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py index 5d7116dcff7..9f95725ab5b 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py @@ -4,11 +4,11 @@ from ._meta import api_vtag from .api import main_router -from .models import ApplicationHealth +from .models.domains.shared_store import SharedStore +from .models.schemas.application_health import ApplicationHealth from .remote_debug import setup as remote_debug_setup from .settings import DynamicSidecarSettings from .shared_handlers import on_shutdown_handler -from .shared_store import SharedStore logger = logging.getLogger(__name__) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py index d5b35a545fb..c1950224296 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py @@ -1,9 +1,9 @@ from fastapi import Depends, Request from fastapi.datastructures import State -from .models import ApplicationHealth +from .models.domains.shared_store import SharedStore +from .models.schemas.application_health import ApplicationHealth from .settings import DynamicSidecarSettings -from .shared_store import SharedStore def get_app_state(request: Request) -> State: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/__init__.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/__init__.py index ce4c96a67c8..e69de29bb2d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/__init__.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/__init__.py @@ -1,3 +0,0 @@ -from .application_health import ApplicationHealth - -__all__ = ["ApplicationHealth"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/domains/__init__.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/domains/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/domains/shared_store.py similarity index 100% rename from services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_store.py rename to services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/domains/shared_store.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/__init__.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/application_health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/application_health.py similarity index 100% rename from services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/application_health.py rename to services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/application_health.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py index 8a31a91c69e..5e4c1758d73 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py @@ -3,8 +3,8 @@ from fastapi import FastAPI +from .models.domains.shared_store import SharedStore from .settings import DynamicSidecarSettings -from .shared_store import SharedStore from .utils import async_command, write_to_tmp_file logger = logging.getLogger(__name__) diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index ee8181cc70c..30f45ee5325 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -13,9 +13,9 @@ from fastapi import FastAPI from pytest_mock.plugin import MockerFixture from simcore_service_dynamic_sidecar.application import assemble_application +from simcore_service_dynamic_sidecar.models.domains.shared_store import SharedStore from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command -from simcore_service_dynamic_sidecar.shared_store import SharedStore @pytest.fixture(scope="module", autouse=True) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 11933fad9f8..b791edc00c9 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -2,7 +2,6 @@ # pylint: disable=unused-argument -import asyncio import json from typing import Any, Dict, List @@ -11,9 +10,9 @@ from faker import Faker from fastapi import status from simcore_service_dynamic_sidecar._meta import api_vtag +from simcore_service_dynamic_sidecar.models.domains.shared_store import SharedStore from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command -from simcore_service_dynamic_sidecar.shared_store import SharedStore from simcore_service_dynamic_sidecar.utils import async_command DEFAULT_COMMAND_TIMEOUT = 5.0 diff --git a/services/dynamic-sidecar/tests/unit/test_api_health.py b/services/dynamic-sidecar/tests/unit/test_api_health.py index dab756db51f..2d0553a91e6 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_health.py +++ b/services/dynamic-sidecar/tests/unit/test_api_health.py @@ -1,7 +1,9 @@ import pytest from async_asgi_testclient import TestClient from fastapi import status -from simcore_service_dynamic_sidecar.models import ApplicationHealth +from simcore_service_dynamic_sidecar.models.schemas.application_health import ( + ApplicationHealth, +) pytestmark = pytest.mark.asyncio From 57dc0c1af5452e3eba412a4ac701376f3daa43b6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Apr 2021 09:29:00 +0200 Subject: [PATCH 096/102] rending applicaiton strucutre similar to other fastapi services --- .../simcore_service_dynamic_sidecar/api/containers.py | 10 +++++----- .../src/simcore_service_dynamic_sidecar/api/health.py | 2 +- .../simcore_service_dynamic_sidecar/core/__init__.py | 0 .../{ => core}/application.py | 8 ++++---- .../{ => core}/dependencies.py | 4 ++-- .../{ => core}/remote_debug.py | 0 .../{ => core}/settings.py | 0 .../{ => core}/shared_handlers.py | 2 +- .../{ => core}/utils.py | 0 .../{ => core}/validation.py | 0 .../src/simcore_service_dynamic_sidecar/main.py | 4 ++-- services/dynamic-sidecar/tests/conftest.py | 8 +++++--- .../dynamic-sidecar/tests/unit/test_api_containers.py | 8 +++++--- 13 files changed, 25 insertions(+), 21 deletions(-) create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/__init__.py rename services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/{ => core}/application.py (90%) rename services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/{ => core}/dependencies.py (84%) rename services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/{ => core}/remote_debug.py (100%) rename services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/{ => core}/settings.py (100%) rename services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/{ => core}/shared_handlers.py (97%) rename services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/{ => core}/utils.py (100%) rename services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/{ => core}/validation.py (100%) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 0a535e4cf77..d32a25f924d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -16,12 +16,12 @@ ) from fastapi.responses import PlainTextResponse -from ..dependencies import get_settings, get_shared_store +from ..core.dependencies import get_settings, get_shared_store +from ..core.settings import DynamicSidecarSettings +from ..core.shared_handlers import remove_the_compose_spec, write_file_and_run_command +from ..core.utils import assemble_container_names, docker_client +from ..core.validation import InvalidComposeSpec, validate_compose_spec from ..models.domains.shared_store import SharedStore -from ..settings import DynamicSidecarSettings -from ..shared_handlers import remove_the_compose_spec, write_file_and_run_command -from ..utils import assemble_container_names, docker_client -from ..validation import InvalidComposeSpec, validate_compose_spec logger = logging.getLogger(__name__) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py index c37a7826c5f..b3c53982e62 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/health.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, Depends, HTTPException, status -from ..dependencies import get_application_health +from ..core.dependencies import get_application_health from ..models.schemas.application_health import ApplicationHealth health_router = APIRouter() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/__init__.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py similarity index 90% rename from services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py rename to services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py index 9f95725ab5b..373e179289a 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py @@ -2,10 +2,10 @@ from fastapi import FastAPI -from ._meta import api_vtag -from .api import main_router -from .models.domains.shared_store import SharedStore -from .models.schemas.application_health import ApplicationHealth +from .._meta import api_vtag +from ..api import main_router +from ..models.domains.shared_store import SharedStore +from ..models.schemas.application_health import ApplicationHealth from .remote_debug import setup as remote_debug_setup from .settings import DynamicSidecarSettings from .shared_handlers import on_shutdown_handler diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/dependencies.py similarity index 84% rename from services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py rename to services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/dependencies.py index c1950224296..21c720e8def 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/dependencies.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/dependencies.py @@ -1,8 +1,8 @@ from fastapi import Depends, Request from fastapi.datastructures import State -from .models.domains.shared_store import SharedStore -from .models.schemas.application_health import ApplicationHealth +from ..models.domains.shared_store import SharedStore +from ..models.schemas.application_health import ApplicationHealth from .settings import DynamicSidecarSettings diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/remote_debug.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/remote_debug.py similarity index 100% rename from services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/remote_debug.py rename to services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/remote_debug.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py similarity index 100% rename from services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/settings.py rename to services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/shared_handlers.py similarity index 97% rename from services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py rename to services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/shared_handlers.py index 5e4c1758d73..deaf3bd4417 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/shared_handlers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/shared_handlers.py @@ -3,7 +3,7 @@ from fastapi import FastAPI -from .models.domains.shared_store import SharedStore +from ..models.domains.shared_store import SharedStore from .settings import DynamicSidecarSettings from .utils import async_command, write_to_tmp_file diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py similarity index 100% rename from services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/utils.py rename to services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/validation.py similarity index 100% rename from services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/validation.py rename to services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/validation.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py index e7a2bdabe30..40116a05242 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/main.py @@ -3,8 +3,8 @@ import uvicorn from fastapi import FastAPI -from simcore_service_dynamic_sidecar.application import assemble_application -from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings +from simcore_service_dynamic_sidecar.core.application import assemble_application +from simcore_service_dynamic_sidecar.core.settings import DynamicSidecarSettings current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 30f45ee5325..940ff7cf2cf 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -12,10 +12,12 @@ from async_asgi_testclient import TestClient from fastapi import FastAPI from pytest_mock.plugin import MockerFixture -from simcore_service_dynamic_sidecar.application import assemble_application +from simcore_service_dynamic_sidecar.core.application import assemble_application +from simcore_service_dynamic_sidecar.core.settings import DynamicSidecarSettings +from simcore_service_dynamic_sidecar.core.shared_handlers import ( + write_file_and_run_command, +) from simcore_service_dynamic_sidecar.models.domains.shared_store import SharedStore -from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings -from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command @pytest.fixture(scope="module", autouse=True) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index b791edc00c9..bbe845c46fc 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -10,10 +10,12 @@ from faker import Faker from fastapi import status from simcore_service_dynamic_sidecar._meta import api_vtag +from simcore_service_dynamic_sidecar.core.settings import DynamicSidecarSettings +from simcore_service_dynamic_sidecar.core.shared_handlers import ( + write_file_and_run_command, +) +from simcore_service_dynamic_sidecar.core.utils import async_command from simcore_service_dynamic_sidecar.models.domains.shared_store import SharedStore -from simcore_service_dynamic_sidecar.settings import DynamicSidecarSettings -from simcore_service_dynamic_sidecar.shared_handlers import write_file_and_run_command -from simcore_service_dynamic_sidecar.utils import async_command DEFAULT_COMMAND_TIMEOUT = 5.0 From a7ee79616c21e0fdd62d13be1f6d660400c1f6d5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Apr 2021 09:52:29 +0200 Subject: [PATCH 097/102] replacing with response.json() --- .../tests/unit/test_api_containers.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index bbe845c46fc..664cf219cc7 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -85,7 +85,7 @@ async def started_containers(test_client: TestClient, compose_spec: str) -> List shared_store: SharedStore = test_client.application.state.shared_store container_names = shared_store.container_names assert len(container_names) == 2 - assert json.loads(response.text) == container_names + assert response.json() == container_names return container_names @@ -114,13 +114,13 @@ async def test_compose_up( assert response.status_code == status.HTTP_202_ACCEPTED, response.text shared_store: SharedStore = test_client.application.state.shared_store container_names = shared_store.container_names - assert json.loads(response.text) == container_names + assert response.json() == container_names async def test_compose_up_spec_not_provided(test_client: TestClient) -> None: response = await test_client.post(f"/{api_vtag}/containers") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text - assert json.loads(response.text) == {"detail": "\nProvided yaml is not valid!"} + assert response.json() == {"detail": "\nProvided yaml is not valid!"} async def test_compose_up_spec_invalid(test_client: TestClient) -> None: @@ -142,7 +142,7 @@ async def test_containers_down_after_starting( assert response.status_code == status.HTTP_202_ACCEPTED, response.text shared_store: SharedStore = test_client.application.state.shared_store container_names = shared_store.container_names - assert json.loads(response.text) == container_names + assert response.json() == container_names response = await test_client.post( f"/{api_vtag}/containers:down", @@ -160,9 +160,7 @@ async def test_containers_down_missing_spec( query_string=dict(command_timeout=DEFAULT_COMMAND_TIMEOUT), ) assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY, response.text - assert json.loads(response.text) == { - "detail": "No spec for docker-compose down was found" - } + assert response.json() == {"detail": "No spec for docker-compose down was found"} def assert_keys_exist(result: Dict[str, Any]) -> bool: @@ -178,7 +176,7 @@ async def test_containers_get( response = await test_client.get(f"/{api_vtag}/containers") assert response.status_code == status.HTTP_200_OK, response.text - decoded_response = json.loads(response.text) + decoded_response = response.json() assert set(decoded_response) == set(started_containers) for entry in decoded_response.values(): assert "Status" not in entry @@ -193,7 +191,7 @@ async def test_containers_get_status( ) assert response.status_code == status.HTTP_200_OK, response.text - decoded_response = json.loads(response.text) + decoded_response = response.json() assert set(decoded_response) == set(started_containers) assert assert_keys_exist(decoded_response) is True From b8937f9ed28561a1de2102618d71ff5050d1f61c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Apr 2021 10:00:47 +0200 Subject: [PATCH 098/102] applied codestyle --- services/dynamic-sidecar/openapi.json | 6 +++--- .../src/simcore_service_dynamic_sidecar/api/containers.py | 6 +++--- .../simcore_service_dynamic_sidecar/core/shared_handlers.py | 2 +- .../src/simcore_service_dynamic_sidecar/core/utils.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index a98370a37a9..7a42e9d82a3 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -77,7 +77,7 @@ "containers" ], "summary": "Runs Docker Compose Up", - "description": "Expects the docker-compose spec as raw-body utf-8 encoded text ", + "description": "Expects the docker-compose spec as raw-body utf-8 encoded text", "operationId": "runs_docker_compose_up_v1_containers_post", "responses": { "202": { @@ -143,7 +143,7 @@ "containers" ], "summary": "Get Container Logs", - "description": "Returns the logs of a given container if found ", + "description": "Returns the logs of a given container if found", "operationId": "get_container_logs_v1_containers__id__logs_get", "parameters": [ { @@ -223,7 +223,7 @@ "containers" ], "summary": "Inspect Container", - "description": "Returns information about the container, like docker inspect command ", + "description": "Returns information about the container, like docker inspect command", "operationId": "inspect_container_v1_containers__id__get", "parameters": [ { diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index d32a25f924d..b9ecc9026f4 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -78,7 +78,7 @@ async def runs_docker_compose_up( settings: DynamicSidecarSettings = Depends(get_settings), shared_store: SharedStore = Depends(get_shared_store), ) -> Union[List[str], Dict[str, Any]]: - """ Expects the docker-compose spec as raw-body utf-8 encoded text """ + """Expects the docker-compose spec as raw-body utf-8 encoded text""" # stores the compose spec after validation body_as_text = (await request.body()).decode("utf-8") @@ -198,7 +198,7 @@ async def get_container_logs( ), shared_store: SharedStore = Depends(get_shared_store), ) -> Union[str, Dict[str, Any]]: - """ Returns the logs of a given container if found """ + """Returns the logs of a given container if found""" # TODO: remove from here and dump directly into the logs of this service # do this in PR#1887 _raise_if_container_is_missing(id, shared_store.container_names) @@ -225,7 +225,7 @@ async def get_container_logs( async def inspect_container( id: str, shared_store: SharedStore = Depends(get_shared_store) ) -> Dict[str, Any]: - """ Returns information about the container, like docker inspect command """ + """Returns information about the container, like docker inspect command""" _raise_if_container_is_missing(id, shared_store.container_names) with docker_client() as docker: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/shared_handlers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/shared_handlers.py index deaf3bd4417..8b10ebe0e83 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/shared_handlers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/shared_handlers.py @@ -16,7 +16,7 @@ async def write_file_and_run_command( command: str, command_timeout: float, ) -> Tuple[bool, str]: - """ The command which accepts {file_path} as an argument for string formatting """ + """The command which accepts {file_path} as an argument for string formatting""" # pylint: disable=not-async-context-manager async with write_to_tmp_file(file_content) as file_path: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py index 3473480c367..f64ab9a8d64 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py @@ -40,7 +40,7 @@ def docker_client() -> Generator[aiodocker.Docker, None, None]: async def async_command(command: str, command_timeout: float) -> Tuple[bool, str]: - """Returns if the command exited correctly and the stdout of the command """ + """Returns if the command exited correctly and the stdout of the command""" proc = await asyncio.create_subprocess_shell( command, stdin=asyncio.subprocess.PIPE, From e5464a04d46f85ad65347e3db128a84c6bb674f4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Apr 2021 10:00:55 +0200 Subject: [PATCH 099/102] comitting new updated dependencies --- services/dynamic-sidecar/requirements/_base.txt | 10 ++++------ services/dynamic-sidecar/requirements/_test.txt | 4 ++-- services/dynamic-sidecar/requirements/_tools.txt | 6 +++--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index 8ba0ccccfa7..b148ba36dfe 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -65,22 +65,20 @@ email-validator==1.1.2 # via pydantic fastapi==0.63.0 # via -r requirements/_base.in -greenlet==1.0.0 - # via sqlalchemy h11==0.12.0 # via uvicorn idna-ssl==1.1.0 # via aiohttp idna==2.10 # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/_base.in # email-validator # idna-ssl # requests # yarl importlib-metadata==4.0.1 - # via - # jsonschema - # sqlalchemy + # via jsonschema jsonschema==3.2.0 # via docker-compose multidict==5.1.0 @@ -122,7 +120,7 @@ six==1.15.0 # jsonschema # pynacl # websocket-client -sqlalchemy[postgresql_psycopg2binary]==1.4.11 +sqlalchemy[postgresql_psycopg2binary]==1.3.24 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt diff --git a/services/dynamic-sidecar/requirements/_test.txt b/services/dynamic-sidecar/requirements/_test.txt index 3efa909d812..1aee8548802 100644 --- a/services/dynamic-sidecar/requirements/_test.txt +++ b/services/dynamic-sidecar/requirements/_test.txt @@ -14,7 +14,7 @@ chardet==4.0.0 # via requests coverage==5.5 # via pytest-cov -faker==8.1.0 +faker==8.1.1 # via -r requirements/_test.in idna==2.10 # via requests @@ -38,7 +38,7 @@ pytest-asyncio==0.15.1 # via -r requirements/_test.in pytest-cov==2.11.1 # via -r requirements/_test.in -pytest-mock==3.5.1 +pytest-mock==3.6.0 # via -r requirements/_test.in pytest==6.2.3 # via diff --git a/services/dynamic-sidecar/requirements/_tools.txt b/services/dynamic-sidecar/requirements/_tools.txt index 9a446d986ef..9b8fc1f7277 100644 --- a/services/dynamic-sidecar/requirements/_tools.txt +++ b/services/dynamic-sidecar/requirements/_tools.txt @@ -8,9 +8,9 @@ appdirs==1.4.4 # via # black # virtualenv -astroid==2.5.3 +astroid==2.5.6 # via pylint -black==20.8b1 +black==21.4b0 # via # -r requirements/../../../requirements/devenv.txt # -r requirements/_tools.in @@ -69,7 +69,7 @@ pip-tools==6.1.0 # via -r requirements/../../../requirements/devenv.txt pre-commit==2.12.1 # via -r requirements/../../../requirements/devenv.txt -pylint==2.7.4 +pylint==2.8.1 # via -r requirements/_tools.in pyyaml==5.4.1 # via From 5b44da3577963bd06000d4d9882b6862b274711c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Apr 2021 11:32:08 +0200 Subject: [PATCH 100/102] refactoring error codes --- .../api/containers.py | 47 +++++-------------- .../core/utils.py | 8 ++++ .../tests/unit/test_api_containers.py | 4 +- 3 files changed, 23 insertions(+), 36 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index b9ecc9026f4..7996c46e61e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -4,7 +4,6 @@ # pylint: disable=redefined-builtin from typing import Any, Dict, List, Union -import aiodocker from fastapi import ( APIRouter, BackgroundTasks, @@ -62,15 +61,6 @@ def _raise_if_container_is_missing(id: str, container_names: List[str]) -> None: raise HTTPException(status.HTTP_404_NOT_FOUND, detail=message) -def _raise_from_docker_error(error: aiodocker.exceptions.DockerError) -> None: - logger.warning( - "An unexpected Docker error occurred:\n%s", str(traceback.format_exc()) - ) - raise HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error.message - ) from error - - @containers_router.post("/containers", status_code=status.HTTP_202_ACCEPTED) async def runs_docker_compose_up( request: Request, @@ -92,7 +82,7 @@ async def runs_docker_compose_up( ) except InvalidComposeSpec as e: logger.warning("Error detected %s", traceback.format_exc()) - raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)) from e # run docker-compose in a background queue and return early background_tasks.add_task(task_docker_compose_up, settings, shared_store) @@ -165,12 +155,9 @@ def _format_result(container_inspect: Dict[str, Any]) -> Dict[str, Any]: results = {} for container in container_names: - try: - container_instance = await docker.containers.get(container) - container_inspect = await container_instance.show() - results[container] = _format_result(container_inspect) - except aiodocker.exceptions.DockerError as err: - _raise_from_docker_error(err) + container_instance = await docker.containers.get(container) + container_inspect = await container_instance.show() + results[container] = _format_result(container_inspect) return results @@ -204,18 +191,14 @@ async def get_container_logs( _raise_if_container_is_missing(id, shared_store.container_names) with docker_client() as docker: - try: - container_instance = await docker.containers.get(id) + container_instance = await docker.containers.get(id) - args = dict(stdout=True, stderr=True, since=since, until=until) - if timestamps: - args["timestamps"] = True + args = dict(stdout=True, stderr=True, since=since, until=until) + if timestamps: + args["timestamps"] = True - container_logs: str = await container_instance.log(**args) - return container_logs - except aiodocker.exceptions.DockerError as err: - _raise_from_docker_error(err) - return None + container_logs: str = await container_instance.log(**args) + return container_logs @containers_router.get( @@ -229,13 +212,9 @@ async def inspect_container( _raise_if_container_is_missing(id, shared_store.container_names) with docker_client() as docker: - try: - container_instance = await docker.containers.get(id) - inspect_result: Dict[str, Any] = await container_instance.show() - return inspect_result - except aiodocker.exceptions.DockerError as err: - _raise_from_docker_error(err) - return None + container_instance = await docker.containers.get(id) + inspect_result: Dict[str, Any] = await container_instance.show() + return inspect_result __all__ = ["containers_router"] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py index f64ab9a8d64..15880b9a94e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py @@ -11,6 +11,7 @@ import yaml from async_generator import asynccontextmanager from async_timeout import timeout +from fastapi import HTTPException, status TEMPLATE_SEARCH_PATTERN = r"%%(.*?)%%" @@ -35,6 +36,13 @@ def docker_client() -> Generator[aiodocker.Docker, None, None]: docker = aiodocker.Docker() try: yield docker + except aiodocker.exceptions.DockerError as error: + logger.warning( + "An unexpected Docker error occurred:\n%s", str(traceback.format_exc()) + ) + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error.message + ) from error finally: docker.close() diff --git a/services/dynamic-sidecar/tests/unit/test_api_containers.py b/services/dynamic-sidecar/tests/unit/test_api_containers.py index 664cf219cc7..6de5ed4a00e 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_containers.py @@ -119,7 +119,7 @@ async def test_compose_up( async def test_compose_up_spec_not_provided(test_client: TestClient) -> None: response = await test_client.post(f"/{api_vtag}/containers") - assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY, response.text assert response.json() == {"detail": "\nProvided yaml is not valid!"} @@ -128,7 +128,7 @@ async def test_compose_up_spec_invalid(test_client: TestClient) -> None: response = await test_client.post( f"/{api_vtag}/containers", data=invalid_compose_spec ) - assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response.text + assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY, response.text assert "Provided yaml is not valid!" in response.text # 28+ characters means the compos spec is also present in the error message assert len(response.text) > 28 From 9ad7a7865fd1892a7ba21cd2b99650e432b9db1e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Apr 2021 11:40:16 +0200 Subject: [PATCH 101/102] clened up exception throwing --- services/dynamic-sidecar/openapi.json | 6 ++++++ .../api/containers.py | 15 ++++++++++++--- .../simcore_service_dynamic_sidecar/core/utils.py | 5 +---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index 7a42e9d82a3..afa260b06d8 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -204,6 +204,9 @@ "404": { "description": "Container does not exists" }, + "500": { + "description": "Errors in container" + }, "422": { "description": "Validation Error", "content": { @@ -248,6 +251,9 @@ "404": { "description": "Container does not exist" }, + "500": { + "description": "Errors in container" + }, "422": { "description": "Validation Error", "content": { diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 7996c46e61e..623da462cab 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -115,7 +115,8 @@ async def runs_docker_compose_down( ) if not finished_without_errors: - raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=stdout) + logger.warning("docker-compose command finished with errors\n%s", stdout) + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail=stdout) return stdout @@ -164,7 +165,12 @@ def _format_result(container_inspect: Dict[str, Any]) -> Dict[str, Any]: @containers_router.get( "/containers/{id}/logs", - responses={status.HTTP_404_NOT_FOUND: {"description": "Container does not exists"}}, + responses={ + status.HTTP_404_NOT_FOUND: { + "description": "Container does not exists", + }, + status.HTTP_500_INTERNAL_SERVER_ERROR: {"description": "Errors in container"}, + }, ) async def get_container_logs( id: str, @@ -203,7 +209,10 @@ async def get_container_logs( @containers_router.get( "/containers/{id}", - responses={status.HTTP_404_NOT_FOUND: {"description": "Container does not exist"}}, + responses={ + status.HTTP_404_NOT_FOUND: {"description": "Container does not exist"}, + status.HTTP_500_INTERNAL_SERVER_ERROR: {"description": "Errors in container"}, + }, ) async def inspect_container( id: str, shared_store: SharedStore = Depends(get_shared_store) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py index 15880b9a94e..fa0c1978022 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py @@ -11,7 +11,6 @@ import yaml from async_generator import asynccontextmanager from async_timeout import timeout -from fastapi import HTTPException, status TEMPLATE_SEARCH_PATTERN = r"%%(.*?)%%" @@ -40,9 +39,7 @@ def docker_client() -> Generator[aiodocker.Docker, None, None]: logger.warning( "An unexpected Docker error occurred:\n%s", str(traceback.format_exc()) ) - raise HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error.message - ) from error + raise error finally: docker.close() From b8ac2c6f9cc24f96ba5e687d0662594a3638b5e6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Apr 2021 11:45:51 +0200 Subject: [PATCH 102/102] restored error emssage --- .../src/simcore_service_dynamic_sidecar/core/utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py index fa0c1978022..c0e60b9154d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py @@ -11,6 +11,7 @@ import yaml from async_generator import asynccontextmanager from async_timeout import timeout +from fastapi import HTTPException, status TEMPLATE_SEARCH_PATTERN = r"%%(.*?)%%" @@ -36,10 +37,10 @@ def docker_client() -> Generator[aiodocker.Docker, None, None]: try: yield docker except aiodocker.exceptions.DockerError as error: - logger.warning( - "An unexpected Docker error occurred:\n%s", str(traceback.format_exc()) - ) - raise error + logger.exception("An unexpected Docker error occurred") + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error.message + ) from error finally: docker.close()