diff --git a/.dockerignore b/.dockerignore index 34ae3126ea5..a00f0b10508 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,7 +8,7 @@ ops/ *.py[cod] # virtualenv -.venv +**/.venv #python eggs **/*.egg-info diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index b70065c91d3..96524ffbfff 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -85,7 +85,7 @@ email-validator==2.2.0 # via pydantic fast-depends==2.4.12 # via faststream -faststream==0.5.23 +faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 # via diff --git a/packages/models-library/src/models_library/projects_state.py b/packages/models-library/src/models_library/projects_state.py index 9f067f8dc51..44d40152760 100644 --- a/packages/models-library/src/models_library/projects_state.py +++ b/packages/models-library/src/models_library/projects_state.py @@ -4,7 +4,14 @@ from enum import Enum, unique -from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationInfo, + field_validator, + model_validator, +) from .projects_access import Owner @@ -56,12 +63,12 @@ class ProjectStatus(str, Enum): class ProjectLocked(BaseModel): value: bool = Field(..., description="True if the project is locked") + status: ProjectStatus = Field(..., description="The status of the project") owner: Owner | None = Field( default=None, description="If locked, the user that owns the lock", validate_default=True, ) - status: ProjectStatus = Field(..., description="The status of the project") model_config = ConfigDict( extra="forbid", use_enum_values=True, @@ -81,15 +88,7 @@ class ProjectLocked(BaseModel): }, ) - @field_validator("owner", mode="before") - @classmethod - def check_not_null(cls, v, info: ValidationInfo): - if info.data["value"] is True and v is None: - msg = "value cannot be None when project is locked" - raise ValueError(msg) - return v - - @field_validator("status") + @field_validator("status", mode="after") @classmethod def check_status_compatible(cls, v, info: ValidationInfo): if info.data["value"] is False and v not in ["CLOSED", "OPENED"]: @@ -100,6 +99,23 @@ def check_status_compatible(cls, v, info: ValidationInfo): raise ValueError(msg) return v + @model_validator(mode="before") + @classmethod + def check_owner_compatible(cls, values): + if ( + values["value"] is True + and values.get("owner") is None + and values["status"] + in [ + status.value + for status in ProjectStatus + if status != ProjectStatus.MAINTAINING + ] + ): + msg = "Owner must be specified when the project is not in the 'MAINTAINING' status." + raise ValueError(msg) + return values + class ProjectRunningState(BaseModel): value: RunningState = Field( diff --git a/packages/models-library/tests/test_projects_state.py b/packages/models-library/tests/test_projects_state.py index 236d65a5538..a5e7f5ce7ea 100644 --- a/packages/models-library/tests/test_projects_state.py +++ b/packages/models-library/tests/test_projects_state.py @@ -8,6 +8,10 @@ def test_project_locked_with_missing_owner_raises(): ProjectLocked.model_validate({"value": False, "status": ProjectStatus.OPENED}) +def test_project_locked_with_missing_owner_ok_during_maintaining(): + ProjectLocked.parse_obj({"value": True, "status": ProjectStatus.MAINTAINING}) + + @pytest.mark.parametrize( "lock, status", [ diff --git a/packages/postgres-database/docker/Dockerfile b/packages/postgres-database/docker/Dockerfile index 7c3e74870b2..ba5a04156af 100644 --- a/packages/postgres-database/docker/Dockerfile +++ b/packages/postgres-database/docker/Dockerfile @@ -1,5 +1,9 @@ # syntax=docker/dockerfile:1 -FROM python:3.6-slim AS base +ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian +FROM python:${PYTHON_VERSION}-slim-bookworm AS base LABEL maintainer=sanderegg @@ -22,16 +26,14 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools @@ -44,6 +46,8 @@ RUN git clone --single-branch --branch ${GIT_BRANCH} ${GIT_REPOSITORY} osparc-si FROM base AS production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu # ensure home folder is read/writable for user scu diff --git a/packages/postgres-database/scripts/erd/Dockerfile b/packages/postgres-database/scripts/erd/Dockerfile index e18bec73e16..168c5522290 100644 --- a/packages/postgres-database/scripts/erd/Dockerfile +++ b/packages/postgres-database/scripts/erd/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base RUN apt-get update \ @@ -15,15 +20,14 @@ RUN apt-get update \ && apt-get clean -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --upgrade \ - pip~=24.0 \ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --upgrade \ wheel \ setuptools # devenv COPY requirements.txt requirements.txt -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install \ - -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements.txt diff --git a/packages/service-integration/Dockerfile b/packages/service-integration/Dockerfile index 15c1de6cc24..b41f4d07b0b 100644 --- a/packages/service-integration/Dockerfile +++ b/packages/service-integration/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base LABEL maintainer=pcrespov @@ -38,12 +43,13 @@ ENV LANG=C.UTF-8 # Turns off writing .pyc files; superfluous on an ephemeral container. ENV PYTHONDONTWRITEBYTECODE=1 \ VIRTUAL_ENV=/home/scu/.venv +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 # Ensures that the python and pip executables used # in the image will be those from our virtualenv. ENV PATH="${VIRTUAL_ENV}/bin:$PATH" - # -------------------------- Build stage ------------------- FROM base AS build @@ -55,15 +61,14 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ pip~=24.0 \ wheel \ @@ -74,7 +79,7 @@ WORKDIR /build/packages/service-integration RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=packages/service-integration,target=/build/packages/service-integration,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ + --mount=type=cache,target=/root/.cache/uv \ uv pip install \ --requirement requirements/prod.txt \ && uv pip list @@ -86,8 +91,6 @@ FROM base AS development # NOTE: this is necessary to allow to build development images but is the same as production here FROM base AS production -ENV PYTHONOPTIMIZE=TRUE - WORKDIR /home/scu # ensure home folder is read/writable for user scu RUN chown -R scu /home/scu diff --git a/packages/service-library/requirements/_base.txt b/packages/service-library/requirements/_base.txt index b97b15d0967..7118a1dba61 100644 --- a/packages/service-library/requirements/_base.txt +++ b/packages/service-library/requirements/_base.txt @@ -57,7 +57,7 @@ email-validator==2.2.0 # via pydantic fast-depends==2.4.12 # via faststream -faststream==0.5.23 +faststream==0.5.28 # via -r requirements/_base.in frozenlist==1.4.1 # via diff --git a/packages/service-library/src/servicelib/deferred_tasks/_deferred_manager.py b/packages/service-library/src/servicelib/deferred_tasks/_deferred_manager.py index 2427b25340e..b49990a7834 100644 --- a/packages/service-library/src/servicelib/deferred_tasks/_deferred_manager.py +++ b/packages/service-library/src/servicelib/deferred_tasks/_deferred_manager.py @@ -134,7 +134,9 @@ def __init__( ClassUniqueReference, type[BaseDeferredHandler] ] = {} - self.broker: RabbitBroker = RabbitBroker(rabbit_settings.dsn) + self.broker: RabbitBroker = RabbitBroker( + rabbit_settings.dsn, log_level=logging.DEBUG + ) self.router: RabbitRouter = RabbitRouter() # NOTE: do not move this to a function, must remain in constructor diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index af923d0c4f9..c5841c8dd74 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -83,7 +83,7 @@ email-validator==2.2.0 # via pydantic fast-depends==2.4.12 # via faststream -faststream==0.5.23 +faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in flexcache==0.3 # via pint diff --git a/requirements/tools/Dockerfile b/requirements/tools/Dockerfile index e6af4395ed6..0901b5e8302 100644 --- a/requirements/tools/Dockerfile +++ b/requirements/tools/Dockerfile @@ -9,6 +9,9 @@ # # ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base ENV VIRTUAL_ENV=/home/scu/.venv @@ -24,24 +27,21 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get clean -y -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools # devenv -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install \ - pip-tools \ pipreqs \ pipdeptree && \ uv pip list -vv diff --git a/scripts/erd/Dockerfile b/scripts/erd/Dockerfile index 9294fa23e1d..8289d6ab16c 100644 --- a/scripts/erd/Dockerfile +++ b/scripts/erd/Dockerfile @@ -8,6 +8,9 @@ # ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base RUN apt-get update \ @@ -22,13 +25,14 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --upgrade \ - pip~=24.0 \ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --upgrade \ wheel \ setuptools COPY requirements.txt . -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install -r requirements.txt diff --git a/scripts/maintenance/migrate_project/Dockerfile b/scripts/maintenance/migrate_project/Dockerfile index d4a17f9ba40..b215ed0886d 100644 --- a/scripts/maintenance/migrate_project/Dockerfile +++ b/scripts/maintenance/migrate_project/Dockerfile @@ -1,21 +1,27 @@ # syntax=docker/dockerfile:1 +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:3.11.9-buster RUN curl https://rclone.org/install.sh | bash && \ rclone --version +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ + WORKDIR /scripts COPY packages/postgres-database postgres-database -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - cd postgres-database && pip install . +RUN --mount=type=cache,target=/root/.cache/uv \ + cd postgres-database && uv pip install . COPY packages/settings-library settings-library -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - cd settings-library && pip install . +RUN --mount=type=cache,target=/root/.cache/uv \ + cd settings-library && uv pip install . COPY scripts/maintenance/migrate_project/requirements.txt /scripts/requirements.txt -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install -r /scripts/requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install -r /scripts/requirements.txt COPY scripts/maintenance/migrate_project/src/*.py /scripts/ diff --git a/scripts/openapi/oas_resolver/Dockerfile b/scripts/openapi/oas_resolver/Dockerfile index 5b0d51570ae..19ffedfdc16 100644 --- a/scripts/openapi/oas_resolver/Dockerfile +++ b/scripts/openapi/oas_resolver/Dockerfile @@ -2,6 +2,9 @@ # Usage: # docker build . -t oas_resolver # docker run -v /path/to/api:/input -v /path/to/compiled/file:/output oas_resolver /input/path/to/openapi.yaml /output/output_file.yaml +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:3.6-alpine LABEL maintainer=sanderegg @@ -11,16 +14,16 @@ VOLUME [ "/output" ] WORKDIR /src +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ + # update pip -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --upgrade \ - pip~=24.0 \ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --upgrade \ wheel \ setuptools -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install prance && \ - pip install click &&\ - pip install openapi_spec_validator +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install prance click openapi_spec_validator ENTRYPOINT [ "prance", "compile" , "--backend=openapi-spec-validator"] diff --git a/scripts/pydeps-docker/Dockerfile b/scripts/pydeps-docker/Dockerfile index 610fe9aa6f0..51d24208672 100644 --- a/scripts/pydeps-docker/Dockerfile +++ b/scripts/pydeps-docker/Dockerfile @@ -9,6 +9,9 @@ # # ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base @@ -23,18 +26,20 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ + ARG HOME_DIR RUN mkdir -p ${HOME_DIR} COPY .pydeps ${HOME_DIR}/.pydeps -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --upgrade \ - pip~=24.0 \ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --upgrade \ wheel \ setuptools # devenv -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install \ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install \ pydeps diff --git a/services/agent/Dockerfile b/services/agent/Dockerfile index fe6fef17c35..9b5d031affd 100644 --- a/services/agent/Dockerfile +++ b/services/agent/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -73,29 +78,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/agent/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -113,9 +111,9 @@ WORKDIR /build/services/agent RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/agent,target=/build/services/agent,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -132,6 +130,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/agent/docker/boot.sh b/services/agent/docker/boot.sh index 333a4af7469..259e1f54e71 100755 --- a/services/agent/docker/boot.sh +++ b/services/agent/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/agent || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/agent + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/agent/docker/entrypoint.sh b/services/agent/docker/entrypoint.sh index 87f1cc84da5..e89ad5408a3 100755 --- a/services/agent/docker/entrypoint.sh +++ b/services/agent/docker/entrypoint.sh @@ -63,27 +63,20 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock -if stat $DOCKER_MOUNT > /dev/null 2>&1 -then - echo "$INFO detected docker socket is mounted, adding user to group..." - GROUPID=$(stat --format=%g $DOCKER_MOUNT) - GROUPNAME=scdocker +if stat $DOCKER_MOUNT >/dev/null 2>&1; then + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker - if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 - then - echo "$WARNING docker group with $GROUPID already exists, getting group name..." - # if group already exists in container, then reuse name - GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) - echo "$WARNING docker group with $GROUPID has name $GROUPNAME" - fi - adduser "$SC_USER_NAME" "$GROUPNAME" + if ! addgroup --gid "$GROUPID" $GROUPNAME >/dev/null 2>&1; then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." + # if group already exists in container, then reuse name + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" + fi + adduser "$SC_USER_NAME" "$GROUPNAME" fi echo "$INFO Starting $* ..." diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt index b9838325ba9..1f4c324cbec 100644 --- a/services/agent/requirements/_base.txt +++ b/services/agent/requirements/_base.txt @@ -90,7 +90,8 @@ fastapi==0.115.2 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in -faststream==0.5.27 + # prometheus-fastapi-instrumentator +faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 # via diff --git a/services/api-server/Dockerfile b/services/api-server/Dockerfile index ced7167fe5c..933df727fc6 100644 --- a/services/api-server/Dockerfile +++ b/services/api-server/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # USAGE: @@ -69,29 +74,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/api-server/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -109,9 +107,9 @@ WORKDIR /build/services/api-server RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/api-server,target=/build/services/api-server,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/api-server/docker/boot.sh b/services/api-server/docker/boot.sh index 39f5867806a..38322883120 100755 --- a/services/api-server/docker/boot.sh +++ b/services/api-server/docker/boot.sh @@ -18,12 +18,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/api-server || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/api-server + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/api-server/docker/entrypoint.sh b/services/api-server/docker/entrypoint.sh index 68791d962fe..b579236b562 100755 --- a/services/api-server/docker/entrypoint.sh +++ b/services/api-server/docker/entrypoint.sh @@ -64,11 +64,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs ptvsd - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index 50006691f4e..51e2544cf88 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -159,7 +159,7 @@ fastapi==0.99.1 # prometheus-fastapi-instrumentator fastapi-pagination==0.12.31 # via -r requirements/_base.in -faststream==0.5.10 +faststream==0.5.28 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in @@ -556,7 +556,6 @@ typer==0.12.3 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in - # faststream types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.10.0 diff --git a/services/autoscaling/Dockerfile b/services/autoscaling/Dockerfile index 1821d873057..4ace322f5f9 100644 --- a/services/autoscaling/Dockerfile +++ b/services/autoscaling/Dockerfile @@ -1,10 +1,15 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # USAGE: -# cd sercices/autoscaling +# cd services/autoscaling # docker build -f Dockerfile -t autoscaling:prod --target production ../../ # docker run autoscaling:prod # @@ -89,31 +94,21 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ + # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/autoscaling/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt - - # --------------------------Prod-depends-only stage ------------------- # This stage is for production only dependencies that get partially wiped out afterwards (final docker image concerns) # @@ -129,9 +124,9 @@ WORKDIR /build/services/autoscaling RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/autoscaling,target=/build/services/autoscaling,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -148,6 +143,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu # ensure home folder is read/writable for user scu diff --git a/services/autoscaling/docker/boot.sh b/services/autoscaling/docker/boot.sh index 0e876652386..8fe0488daed 100755 --- a/services/autoscaling/docker/boot.sh +++ b/services/autoscaling/docker/boot.sh @@ -23,12 +23,15 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/autoscaling || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + cd services/autoscaling + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/autoscaling/docker/entrypoint.sh b/services/autoscaling/docker/entrypoint.sh index f928edbb4f5..ad982fd8d5c 100755 --- a/services/autoscaling/docker/entrypoint.sh +++ b/services/autoscaling/docker/entrypoint.sh @@ -70,11 +70,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock if stat $DOCKER_MOUNT >/dev/null 2>&1; then diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index c1fa0708c90..dd5f8f0ed20 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -138,7 +138,7 @@ fastapi==0.115.0 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator -faststream==0.5.10 +faststream==0.5.28 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -523,7 +523,6 @@ typer==0.12.3 # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in - # faststream types-aiobotocore==2.13.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in types-aiobotocore-ec2==2.13.0 diff --git a/services/catalog/Dockerfile b/services/catalog/Dockerfile index c4f48d2ec08..8fb6418b3ce 100644 --- a/services/catalog/Dockerfile +++ b/services/catalog/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # @@ -70,29 +75,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/catalog/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -110,9 +108,9 @@ WORKDIR /build/services/catalog RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/catalog,target=/build/services/catalog,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/catalog/docker/boot.sh b/services/catalog/docker/boot.sh index 69ce7ebb4ec..f7f01aec46e 100755 --- a/services/catalog/docker/boot.sh +++ b/services/catalog/docker/boot.sh @@ -18,12 +18,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/catalog || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/catalog + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/catalog/docker/entrypoint.sh b/services/catalog/docker/entrypoint.sh index 61529af46c6..63e9249be31 100755 --- a/services/catalog/docker/entrypoint.sh +++ b/services/catalog/docker/entrypoint.sh @@ -63,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs ptvsd - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt index 2e20eb89be0..d78334d994c 100644 --- a/services/catalog/requirements/_base.txt +++ b/services/catalog/requirements/_base.txt @@ -95,7 +95,7 @@ fastapi==0.99.1 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator -faststream==0.5.10 +faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 # via @@ -379,7 +379,6 @@ typer==0.12.3 # via # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in - # faststream types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.10.0 diff --git a/services/clusters-keeper/Dockerfile b/services/clusters-keeper/Dockerfile index 2dd08b4b44b..e75ba032ac0 100644 --- a/services/clusters-keeper/Dockerfile +++ b/services/clusters-keeper/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -89,29 +94,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/clusters-keeper/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -129,9 +127,9 @@ WORKDIR /build/services/clusters-keeper RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/clusters-keeper,target=/build/services/clusters-keeper,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -148,6 +146,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu # ensure home folder is read/writable for user scu diff --git a/services/clusters-keeper/docker/boot.sh b/services/clusters-keeper/docker/boot.sh index f0239f42e72..5aa4b0e8e1c 100755 --- a/services/clusters-keeper/docker/boot.sh +++ b/services/clusters-keeper/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/clusters-keeper || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/clusters-keeper + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 5b72b81e357..8d2bc14dbcc 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -136,7 +136,7 @@ fastapi==0.115.0 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator -faststream==0.5.10 +faststream==0.5.28 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -521,7 +521,6 @@ typer==0.12.3 # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in - # faststream types-aiobotocore==2.13.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in types-aiobotocore-ec2==2.13.0 diff --git a/services/dask-sidecar/Dockerfile b/services/dask-sidecar/Dockerfile index a85f653f5cc..5ea8e90d545 100644 --- a/services/dask-sidecar/Dockerfile +++ b/services/dask-sidecar/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM --platform=${TARGETPLATFORM} python:${PYTHON_VERSION}-slim-bookworm AS base ARG TARGETPLATFORM ARG BUILDPLATFORM @@ -80,27 +85,20 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install base 3rd party dependencies (NOTE: this speeds up devel mode) -RUN \ - --mount=type=bind,source=services/dask-sidecar/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- # This stage is for production only dependencies that get partially wiped out afterwards (final docker image concerns) @@ -117,9 +115,9 @@ WORKDIR /build/services/dask-sidecar RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/dask-sidecar,target=/build/services/dask-sidecar,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list # --------------------------Production stage ------------------- @@ -135,13 +133,17 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu - # ensure home folder is read/writable for user scu RUN chown -R scu /home/scu -# bring installed package without build tools + +# Starting from clean base image, copies pre-installed virtualenv from prod-only-deps COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +# Copies booting scripts COPY --chown=scu:scu services/dask-sidecar/docker services/dask-sidecar/docker RUN chmod +x services/dask-sidecar/docker/*.sh diff --git a/services/dask-sidecar/docker/boot.sh b/services/dask-sidecar/docker/boot.sh index 213377cbf0e..2cfdbf349d7 100755 --- a/services/dask-sidecar/docker/boot.sh +++ b/services/dask-sidecar/docker/boot.sh @@ -24,11 +24,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then print_info "Python :" python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/dask-sidecar || exit 1 - pip install --no-cache-dir -r requirements/dev.txt - cd - || exit 1 + cd services/dask-sidecar + uv pip sync --quiet --no-cache-dir requirements/dev.txt + cd - print_info "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/dask-sidecar/docker/entrypoint.sh b/services/dask-sidecar/docker/entrypoint.sh index 5c00a365b3a..f69fd1a71d8 100755 --- a/services/dask-sidecar/docker/entrypoint.sh +++ b/services/dask-sidecar/docker/entrypoint.sh @@ -22,93 +22,77 @@ update-ca-certificates # *runs* as non-root user [scu] # echo "$INFO" "Entrypoint for stage ${SC_BUILD_TARGET} ..." -echo User :"$(id "$(whoami)")" -echo Workdir :"$(pwd)" -echo scuUser :"$(id scu)" - - -if [ "${SC_BUILD_TARGET}" = "development" ] -then - echo "$INFO" "development mode detected..." - # NOTE: expects docker run ... -v $(pwd):/devel/services/dask-sidecar - DEVEL_MOUNT="/devel/services/dask-sidecar" - - stat $DEVEL_MOUNT > /dev/null 2>&1 || \ - (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) - - echo "setting correct user id/group id..." - HOST_USERID=$(stat --format=%u "${DEVEL_MOUNT}") - HOST_GROUPID=$(stat --format=%g "${DEVEL_MOUNT}") - CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) - if [ "$HOST_USERID" -eq 0 ] - then - echo "Warning: Folder mounted owned by root user... adding $SC_USER_NAME to root..." - adduser "$SC_USER_NAME" root +echo User :"$(id "$(whoami)")" +echo Workdir :"$(pwd)" +echo scuUser :"$(id scu)" + +if [ "${SC_BUILD_TARGET}" = "development" ]; then + echo "$INFO" "development mode detected..." + # NOTE: expects docker run ... -v $(pwd):/devel/services/dask-sidecar + DEVEL_MOUNT="/devel/services/dask-sidecar" + + stat $DEVEL_MOUNT >/dev/null 2>&1 || + (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) + + echo "setting correct user id/group id..." + HOST_USERID=$(stat --format=%u "${DEVEL_MOUNT}") + HOST_GROUPID=$(stat --format=%g "${DEVEL_MOUNT}") + CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) + if [ "$HOST_USERID" -eq 0 ]; then + echo "Warning: Folder mounted owned by root user... adding $SC_USER_NAME to root..." + adduser "$SC_USER_NAME" root + else + echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." + # take host's credentials in $SC_USER_NAME + if [ -z "$CONT_GROUPNAME" ]; then + echo "Creating new group my$SC_USER_NAME" + CONT_GROUPNAME=my$SC_USER_NAME + addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" else - echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." - # take host's credentials in $SC_USER_NAME - if [ -z "$CONT_GROUPNAME" ] - then - echo "Creating new group my$SC_USER_NAME" - CONT_GROUPNAME=my$SC_USER_NAME - addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" - else - echo "group already exists" - fi - echo "adding $SC_USER_NAME to group $CONT_GROUPNAME..." - adduser "$SC_USER_NAME" "$CONT_GROUPNAME" - - echo "changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" - usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" - - echo "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" - find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; - # change user property of files already around - echo "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" - find / -path /proc -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; + echo "group already exists" fi -fi + echo "adding $SC_USER_NAME to group $CONT_GROUPNAME..." + adduser "$SC_USER_NAME" "$CONT_GROUPNAME" + echo "changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" + usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" -if [ "${SC_BOOT_MODE}" = "debug" ] -then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy + echo "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; + # change user property of files already around + echo "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; + fi fi - if [ ${DASK_START_AS_SCHEDULER+x} ]; then echo "$INFO Starting $* as SCHEDULER ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" - else # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock - if stat $DOCKER_MOUNT > /dev/null 2>&1 - then - echo "$INFO detected docker socket is mounted, adding user to group..." - GROUPID=$(stat --format=%g $DOCKER_MOUNT) - GROUPNAME=scdocker - - if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 - then - echo "$WARNING docker group with $GROUPID already exists, getting group name..." - # if group already exists in container, then reuse name - GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) - echo "$WARNING docker group with $GROUPID has name $GROUPNAME" - fi - adduser "$SC_USER_NAME" "$GROUPNAME" + if stat $DOCKER_MOUNT >/dev/null 2>&1; then + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker + + if ! addgroup --gid "$GROUPID" $GROUPNAME >/dev/null 2>&1; then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." + # if group already exists in container, then reuse name + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" + fi + adduser "$SC_USER_NAME" "$GROUPNAME" fi echo "$INFO ensuring write rights on computational shared folder ..." mkdir --parents "${SIDECAR_COMP_SERVICES_SHARED_FOLDER}" chown --recursive "$SC_USER_NAME":"$GROUPNAME" "${SIDECAR_COMP_SERVICES_SHARED_FOLDER}" - echo "$INFO Starting $* as WORKER ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt index 9c36e1d2e6e..33eff58ebae 100644 --- a/services/dask-sidecar/requirements/_base.txt +++ b/services/dask-sidecar/requirements/_base.txt @@ -128,7 +128,7 @@ email-validator==2.1.1 # via pydantic fast-depends==2.4.12 # via faststream -faststream==0.5.10 +faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 # via @@ -465,7 +465,6 @@ typer==0.12.3 # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in - # faststream types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.11.0 diff --git a/services/datcore-adapter/Dockerfile b/services/datcore-adapter/Dockerfile index bdd52d81841..e83bf01c37e 100644 --- a/services/datcore-adapter/Dockerfile +++ b/services/datcore-adapter/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -70,29 +75,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/datcore-adapter/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -110,9 +108,9 @@ WORKDIR /build/services/datcore-adapter RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/datcore-adapter,target=/build/services/datcore-adapter,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/datcore-adapter/docker/boot.sh b/services/datcore-adapter/docker/boot.sh index 1002a32de8a..74f8d84a7aa 100755 --- a/services/datcore-adapter/docker/boot.sh +++ b/services/datcore-adapter/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/datcore-adapter || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/datcore-adapter + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/datcore-adapter/docker/entrypoint.sh b/services/datcore-adapter/docker/entrypoint.sh index 3ac31b9b815..25153a6b2a2 100755 --- a/services/datcore-adapter/docker/entrypoint.sh +++ b/services/datcore-adapter/docker/entrypoint.sh @@ -20,7 +20,6 @@ echo "$INFO" "User : $(id scu)" echo "$INFO" "python : $(command -v python)" echo "$INFO" "pip : $(command -v pip)" - # # DEVELOPMENT MODE # - expects docker run ... -v $(pwd):$SC_DEVEL_MOUNT @@ -64,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/datcore-adapter/requirements/_base.txt b/services/datcore-adapter/requirements/_base.txt index b572351f2e2..1608048db0f 100644 --- a/services/datcore-adapter/requirements/_base.txt +++ b/services/datcore-adapter/requirements/_base.txt @@ -100,7 +100,7 @@ fastapi==0.115.3 # prometheus-fastapi-instrumentator fastapi-pagination==0.12.31 # via -r requirements/_base.in -faststream==0.5.10 +faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 # via @@ -395,7 +395,6 @@ typer==0.12.3 # via # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in - # faststream types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.10.0 diff --git a/services/director-v2/Dockerfile b/services/director-v2/Dockerfile index 4780eb62e33..5388dbb6f73 100644 --- a/services/director-v2/Dockerfile +++ b/services/director-v2/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -70,29 +75,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/director-v2/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -110,9 +108,9 @@ WORKDIR /build/services/director-v2 RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/director-v2,target=/build/services/director-v2,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/director-v2/docker/boot.sh b/services/director-v2/docker/boot.sh index 348b976149c..a96161d9660 100755 --- a/services/director-v2/docker/boot.sh +++ b/services/director-v2/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/director-v2 || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/director-v2 + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/director-v2/docker/entrypoint.sh b/services/director-v2/docker/entrypoint.sh index 57b0d0ac2ff..42995c6ae78 100755 --- a/services/director-v2/docker/entrypoint.sh +++ b/services/director-v2/docker/entrypoint.sh @@ -27,7 +27,6 @@ echo "$INFO" "User : $(id scu)" echo "$INFO" "python : $(command -v python)" echo "$INFO" "pip : $(command -v pip)" - # # DEVELOPMENT MODE # - expects docker run ... -v $(pwd):$SC_DEVEL_MOUNT @@ -71,27 +70,20 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock -if stat $DOCKER_MOUNT > /dev/null 2>&1 -then - echo "$INFO detected docker socket is mounted, adding user to group..." - GROUPID=$(stat --format=%g $DOCKER_MOUNT) - GROUPNAME=scdocker +if stat $DOCKER_MOUNT >/dev/null 2>&1; then + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker - if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 - then - echo "$WARNING docker group with $GROUPID already exists, getting group name..." - # if group already exists in container, then reuse name - GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) - echo "$WARNING docker group with $GROUPID has name $GROUPNAME" - fi - adduser "$SC_USER_NAME" "$GROUPNAME" + if ! addgroup --gid "$GROUPID" $GROUPNAME >/dev/null 2>&1; then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." + # if group already exists in container, then reuse name + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" + fi + adduser "$SC_USER_NAME" "$GROUPNAME" fi echo "$INFO Starting $* ..." diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index d4a5c75eb69..42c06dd93de 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -178,7 +178,7 @@ fastapi==0.99.1 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator -faststream==0.5.10 +faststream==0.5.28 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in @@ -674,7 +674,6 @@ typer==0.12.3 # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in - # faststream types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.11.0 diff --git a/services/director/requirements/_tools.txt b/services/director/requirements/_tools.txt index 4c0c50f666d..821e63f1a10 100644 --- a/services/director/requirements/_tools.txt +++ b/services/director/requirements/_tools.txt @@ -25,7 +25,7 @@ pyyaml==5.4 # watchdog regex==2022.1.18 # via black -setuptools==75.1.0 +setuptools==75.2.0 # via pip-tools toml==0.10.2 # via diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index 78c72e08a9a..f0254834105 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -28,6 +28,8 @@ services: ports: - "8007:8000" - "3012:3000" + networks: + - default catalog: environment: diff --git a/services/dynamic-scheduler/Dockerfile b/services/dynamic-scheduler/Dockerfile index bc94b83125a..b3e9119c898 100644 --- a/services/dynamic-scheduler/Dockerfile +++ b/services/dynamic-scheduler/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -69,29 +74,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/dynamic-scheduler/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + @@ -110,9 +108,9 @@ WORKDIR /build/services/dynamic-scheduler RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/dynamic-scheduler,target=/build/services/dynamic-scheduler,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/dynamic-scheduler/docker/boot.sh b/services/dynamic-scheduler/docker/boot.sh index 7164715d7dc..0660b8c8cc0 100755 --- a/services/dynamic-scheduler/docker/boot.sh +++ b/services/dynamic-scheduler/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/dynamic-scheduler || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/dynamic-scheduler + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/dynamic-scheduler/docker/entrypoint.sh b/services/dynamic-scheduler/docker/entrypoint.sh index 908e8f9e64b..25153a6b2a2 100755 --- a/services/dynamic-scheduler/docker/entrypoint.sh +++ b/services/dynamic-scheduler/docker/entrypoint.sh @@ -63,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index 495550bffb1..8ecd0fef578 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -102,7 +102,8 @@ fastapi==0.115.2 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in -faststream==0.5.27 + # prometheus-fastapi-instrumentator +faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 # via diff --git a/services/dynamic-sidecar/Dockerfile b/services/dynamic-sidecar/Dockerfile index 3be8bae74d0..5eb6662593f 100644 --- a/services/dynamic-sidecar/Dockerfile +++ b/services/dynamic-sidecar/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # USAGE: @@ -53,14 +58,14 @@ RUN \ ./install_rclone.bash RUN AWS_CLI_VERSION="2.11.11" \ - && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip" -o "awscliv2.zip" \ - && apt-get update && apt-get install -y unzip \ - && unzip awscliv2.zip \ - && ./aws/install \ - && apt-get remove --purge -y unzip \ - && rm awscliv2.zip \ - && rm -rf awscliv2 \ - && aws --version + && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip" -o "awscliv2.zip" \ + && apt-get update && apt-get install -y unzip \ + && unzip awscliv2.zip \ + && ./aws/install \ + && apt-get remove --purge -y unzip \ + && rm awscliv2.zip \ + && rm -rf awscliv2 \ + && aws --version # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) ENV SC_USER_ID=8004 \ @@ -110,31 +115,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" \ && mkdir -p "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/dynamic-sidecar/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/pip \ - uv pip install \ - --requirement _base.txt # copy utility devops scripts COPY --chown=scu:scu services/dynamic-sidecar/scripts/Makefile /home/scu @@ -155,9 +151,9 @@ WORKDIR /build/services/dynamic-sidecar RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/dynamic-sidecar,target=/build/services/dynamic-sidecar,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list # --------------------------Production stage ------------------- @@ -173,6 +169,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/dynamic-sidecar/docker/boot.sh b/services/dynamic-sidecar/docker/boot.sh index ff63ce74cfb..d21e87cb070 100755 --- a/services/dynamic-sidecar/docker/boot.sh +++ b/services/dynamic-sidecar/docker/boot.sh @@ -25,12 +25,17 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then # NOTE: uv does not like this requirement file... cd /devel/services/dynamic-sidecar - pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" pip list | sed 's/^/ /' fi +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy +fi + # # RUNNING application # diff --git a/services/dynamic-sidecar/docker/entrypoint.sh b/services/dynamic-sidecar/docker/entrypoint.sh index e326c25825b..9ccf61d8f67 100755 --- a/services/dynamic-sidecar/docker/entrypoint.sh +++ b/services/dynamic-sidecar/docker/entrypoint.sh @@ -74,11 +74,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock if stat $DOCKER_MOUNT >/dev/null 2>&1; then diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index 6439dd71e97..e531f0ac0ba 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -157,7 +157,7 @@ fastapi==0.115.2 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator -faststream==0.5.10 +faststream==0.5.28 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in @@ -673,7 +673,6 @@ typer==0.12.3 # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in - # faststream types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.11.0 diff --git a/services/efs-guardian/Dockerfile b/services/efs-guardian/Dockerfile index 229e43563c4..82174799ca5 100644 --- a/services/efs-guardian/Dockerfile +++ b/services/efs-guardian/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -107,29 +112,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/efs-guardian/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -147,9 +145,9 @@ WORKDIR /build/services/efs-guardian RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/efs-guardian,target=/build/services/efs-guardian,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -166,6 +164,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/efs diff --git a/services/efs-guardian/docker/boot.sh b/services/efs-guardian/docker/boot.sh index d4d046d5a14..0575c7679fb 100755 --- a/services/efs-guardian/docker/boot.sh +++ b/services/efs-guardian/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/efs-guardian || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/efs-guardian + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index 35f30a99e5a..9caea149b29 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -129,7 +129,7 @@ fastapi==0.115.0 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator -faststream==0.5.27 +faststream==0.5.28 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/background_tasks.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/background_tasks.py index e6a9ee6ee15..8ac475a4742 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/background_tasks.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/background_tasks.py @@ -34,6 +34,10 @@ async def removal_policy_task(app: FastAPI) -> None: efs_project_ids: list[ ProjectID ] = await efs_manager.list_projects_across_whole_efs() + _logger.info( + "Number of projects that are currently in the EFS file system: %s", + len(efs_project_ids), + ) projects_repo = ProjectsRepo(app.state.engine) for project_id in efs_project_ids: diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/background_tasks_setup.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/background_tasks_setup.py index 81920ab2310..5874de42b0c 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/background_tasks_setup.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/background_tasks_setup.py @@ -14,9 +14,9 @@ _logger = logging.getLogger(__name__) -_SEC = 1000 # in ms -_MIN = 60 * _SEC # in ms -_HOUR = 60 * _MIN # in ms +_SEC = 1 # in s +_MIN = 60 * _SEC # in s +_HOUR = 60 * _MIN # in s class EfsGuardianBackgroundTask(TypedDict): diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages_setup.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages_setup.py index d879d189157..deb151fbda5 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages_setup.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages_setup.py @@ -39,7 +39,7 @@ async def _subscribe_to_rabbitmq(app) -> str: def _on_app_startup(app: FastAPI) -> Callable[[], Awaitable[None]]: async def _startup() -> None: with log_context( - _logger, logging.INFO, msg="setup resource tracker" + _logger, logging.INFO, msg="setup efs guardian process messages" ), log_catch(_logger, reraise=False): app_settings: ApplicationSettings = app.state.settings app.state.efs_guardian_rabbitmq_consumer = None diff --git a/services/invitations/Dockerfile b/services/invitations/Dockerfile index 5d935426c48..0422ca73cbd 100644 --- a/services/invitations/Dockerfile +++ b/services/invitations/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -69,29 +74,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/invitations/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + @@ -110,9 +108,9 @@ WORKDIR /build/services/invitations RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/invitations,target=/build/services/invitations,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/invitations/docker/boot.sh b/services/invitations/docker/boot.sh index b6ed691759f..7e771439871 100755 --- a/services/invitations/docker/boot.sh +++ b/services/invitations/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/invitations || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/invitations + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/invitations/docker/entrypoint.sh b/services/invitations/docker/entrypoint.sh index 908e8f9e64b..25153a6b2a2 100755 --- a/services/invitations/docker/entrypoint.sh +++ b/services/invitations/docker/entrypoint.sh @@ -63,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/invitations/requirements/_base.txt b/services/invitations/requirements/_base.txt index 636facc1b1f..3f4569b50b9 100644 --- a/services/invitations/requirements/_base.txt +++ b/services/invitations/requirements/_base.txt @@ -87,7 +87,7 @@ fastapi==0.115.0 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator -faststream==0.5.10 +faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 # via @@ -326,7 +326,6 @@ typer==0.12.3 # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in - # faststream types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.10.0 diff --git a/services/migration/Dockerfile b/services/migration/Dockerfile index d70eec97f93..30a20c6df93 100644 --- a/services/migration/Dockerfile +++ b/services/migration/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base LABEL maintainer=sanderegg @@ -43,16 +48,14 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ build-essential \ git -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools @@ -61,7 +64,7 @@ WORKDIR /build/packages/postgres-database # install only base 3rd party dependencies RUN \ --mount=type=bind,source=packages/postgres-database,target=/build/packages/postgres-database,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ + --mount=type=cache,target=/root/.cache/uv \ uv pip install \ --requirement requirements/prod.txt \ && uv pip list @@ -71,6 +74,8 @@ RUN \ FROM base AS production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 # testing defaults ENV POSTGRES_USER=scu \ diff --git a/services/osparc-gateway-server/Dockerfile b/services/osparc-gateway-server/Dockerfile index 59f55c8b3b6..899ef0fb434 100644 --- a/services/osparc-gateway-server/Dockerfile +++ b/services/osparc-gateway-server/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bullseye AS base ARG TARGETPLATFORM ARG BUILDPLATFORM @@ -72,26 +77,19 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ git \ golang-go -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install base 3rd party dependencies (NOTE: this speeds up devel mode) -RUN \ - --mount=type=bind,source=services/osparc-gateway-server/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # in ARM64 mode there is a catch: the piwheels package does not contain the dask-gateway-proxy executable in 64-bit RUN dpkgArch="$(dpkg --print-architecture)";\ @@ -117,7 +115,7 @@ WORKDIR /build/services/osparc-gateway-server RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/osparc-gateway-server,target=/build/services/osparc-gateway-server,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ + --mount=type=cache,target=/root/.cache/uv \ uv pip install \ --requirement requirements/prod.txt @@ -134,6 +132,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu # ensure home folder is read/writable for user scu diff --git a/services/osparc-gateway-server/docker/boot.sh b/services/osparc-gateway-server/docker/boot.sh index d1daccc6195..d2b912eb3ba 100755 --- a/services/osparc-gateway-server/docker/boot.sh +++ b/services/osparc-gateway-server/docker/boot.sh @@ -18,22 +18,27 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then echo "$INFO" "Python :" python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/osparc-gateway-server || exit 1 - pip install --no-cache-dir -r requirements/dev.txt - cd - || exit 1 + cd services/osparc-gateway-server + uv pip install --no-cache-dir -r requirements/dev.txt + cd - echo "$INFO" "PIP :" pip list | sed 's/^/ /' fi +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy +fi + if [ "${SC_BOOT_MODE}" = "debug" ]; then exec python -m debugpy --listen 0.0.0.0:"${OSPARC_GATEWAY_SERVER_DEBUGGING_PORT}" -m watchmedo auto-restart \ - --recursive \ - --pattern="*.py;*/src/*" \ - --ignore-patterns="*test*;pytest_simcore/*;setup.py;*ignore*" \ - --ignore-directories -- \ - osparc-gateway-server \ - --config "${GATEWAY_SERVER_CONFIG_FILE_CONTAINER}" \ - --debug + --recursive \ + --pattern="*.py;*/src/*" \ + --ignore-patterns="*test*;pytest_simcore/*;setup.py;*ignore*" \ + --ignore-directories -- \ + osparc-gateway-server \ + --config "${GATEWAY_SERVER_CONFIG_FILE_CONTAINER}" \ + --debug else exec osparc-gateway-server \ --config "${GATEWAY_SERVER_CONFIG_FILE_CONTAINER}" diff --git a/services/osparc-gateway-server/docker/entrypoint.sh b/services/osparc-gateway-server/docker/entrypoint.sh index 91204dd9b60..cd8eb9a01ef 100755 --- a/services/osparc-gateway-server/docker/entrypoint.sh +++ b/services/osparc-gateway-server/docker/entrypoint.sh @@ -15,74 +15,62 @@ ERROR="ERROR: [$(basename "$0")] " # *runs* as non-root user [scu] # echo "$INFO" "Entrypoint for stage ${SC_BUILD_TARGET} ..." -echo User :"$(id "$(whoami)")" -echo Workdir :"$(pwd)" -echo scuUser :"$(id scu)" +echo User :"$(id "$(whoami)")" +echo Workdir :"$(pwd)" +echo scuUser :"$(id scu)" +if [ "${SC_BUILD_TARGET}" = "development" ]; then + echo "$INFO" "development mode detected..." + # NOTE: expects docker run ... -v $(pwd):/devel/services/osparc-gateway-server + DEVEL_MOUNT=${DEVEL_MOUNT:="/devel/services/osparc-gateway-server"} -if [ "${SC_BUILD_TARGET}" = "development" ] -then - echo "$INFO" "development mode detected..." - # NOTE: expects docker run ... -v $(pwd):/devel/services/osparc-gateway-server - DEVEL_MOUNT=${DEVEL_MOUNT:="/devel/services/osparc-gateway-server"} + stat $DEVEL_MOUNT >/dev/null 2>&1 || + (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) - stat $DEVEL_MOUNT > /dev/null 2>&1 || \ - (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) - - echo "setting correct user id/group id..." - HOST_USERID=$(stat --format=%u "${DEVEL_MOUNT}") - HOST_GROUPID=$(stat --format=%g "${DEVEL_MOUNT}") - CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) - if [ "$HOST_USERID" -eq 0 ] - then - echo "Warning: Folder mounted owned by root user... adding $SC_USER_NAME to root..." - adduser "$SC_USER_NAME" root + echo "setting correct user id/group id..." + HOST_USERID=$(stat --format=%u "${DEVEL_MOUNT}") + HOST_GROUPID=$(stat --format=%g "${DEVEL_MOUNT}") + CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) + if [ "$HOST_USERID" -eq 0 ]; then + echo "Warning: Folder mounted owned by root user... adding $SC_USER_NAME to root..." + adduser "$SC_USER_NAME" root + else + echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." + # take host's credentials in $SC_USER_NAME + if [ -z "$CONT_GROUPNAME" ]; then + echo "Creating new group my$SC_USER_NAME" + CONT_GROUPNAME=my$SC_USER_NAME + addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" else - echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." - # take host's credentials in $SC_USER_NAME - if [ -z "$CONT_GROUPNAME" ] - then - echo "Creating new group my$SC_USER_NAME" - CONT_GROUPNAME=my$SC_USER_NAME - addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" - else - echo "group already exists" - fi - echo "adding $SC_USER_NAME to group $CONT_GROUPNAME..." - adduser "$SC_USER_NAME" "$CONT_GROUPNAME" - - echo "changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" - usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" - - echo "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" - find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; - # change user property of files already around - echo "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" - find / -path /proc -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; + echo "group already exists" fi -fi + echo "adding $SC_USER_NAME to group $CONT_GROUPNAME..." + adduser "$SC_USER_NAME" "$CONT_GROUPNAME" + echo "changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" + usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" -if [ "${SC_BOOT_MODE}" = "debug" ] -then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy + echo "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; + # change user property of files already around + echo "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; + fi fi DOCKER_MOUNT=/var/run/docker.sock if stat $DOCKER_MOUNT >/dev/null 2>&1; then - echo "$INFO detected docker socket is mounted, adding user to group..." - GROUPID=$(stat --format=%g $DOCKER_MOUNT) - GROUPNAME=scdocker + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker - if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 - then - echo "$WARNING docker group with $GROUPID already exists, getting group name..." - # if group already exists in container, then reuse name - GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) - echo "$WARNING docker group with $GROUPID has name $GROUPNAME" - fi - adduser "$SC_USER_NAME" "$GROUPNAME" + if ! addgroup --gid "$GROUPID" $GROUPNAME >/dev/null 2>&1; then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." + # if group already exists in container, then reuse name + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" + fi + adduser "$SC_USER_NAME" "$GROUPNAME" fi echo "$INFO Starting osparc-gateway-server ..." diff --git a/services/payments/Dockerfile b/services/payments/Dockerfile index 90cbb1c908e..32fada11361 100644 --- a/services/payments/Dockerfile +++ b/services/payments/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -69,29 +74,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/payments/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + @@ -110,9 +108,9 @@ WORKDIR /build/services/payments RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/payments,target=/build/services/payments,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/payments/docker/boot.sh b/services/payments/docker/boot.sh index bf5b6881fa1..be92e7ac4fc 100755 --- a/services/payments/docker/boot.sh +++ b/services/payments/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/payments || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/payments + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/payments/docker/entrypoint.sh b/services/payments/docker/entrypoint.sh index 908e8f9e64b..25153a6b2a2 100755 --- a/services/payments/docker/entrypoint.sh +++ b/services/payments/docker/entrypoint.sh @@ -63,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt index 812797c67e2..f4beebc5341 100644 --- a/services/payments/requirements/_base.txt +++ b/services/payments/requirements/_base.txt @@ -125,7 +125,7 @@ fastapi==0.115.2 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator -faststream==0.5.15 +faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 # via @@ -515,7 +515,6 @@ typer==0.12.3 # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in - # faststream types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.12.2 diff --git a/services/resource-usage-tracker/Dockerfile b/services/resource-usage-tracker/Dockerfile index 55976c00c53..528ecabfd3e 100644 --- a/services/resource-usage-tracker/Dockerfile +++ b/services/resource-usage-tracker/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -70,29 +75,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/resource-usage-tracker/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -110,9 +108,9 @@ WORKDIR /build/services/resource-usage-tracker RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/resource-usage-tracker,target=/build/services/resource-usage-tracker,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/resource-usage-tracker/docker/boot.sh b/services/resource-usage-tracker/docker/boot.sh index 878b164a05e..d6e54f8f347 100755 --- a/services/resource-usage-tracker/docker/boot.sh +++ b/services/resource-usage-tracker/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/resource-usage-tracker || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/resource-usage-tracker + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/resource-usage-tracker/docker/entrypoint.sh b/services/resource-usage-tracker/docker/entrypoint.sh index a1a8c160032..e89ad5408a3 100755 --- a/services/resource-usage-tracker/docker/entrypoint.sh +++ b/services/resource-usage-tracker/docker/entrypoint.sh @@ -63,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock if stat $DOCKER_MOUNT >/dev/null 2>&1; then diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index 857c0f0d741..6124ad0de12 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -135,7 +135,7 @@ fastapi==0.115.0 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator -faststream==0.5.10 +faststream==0.5.28 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -545,7 +545,6 @@ typer==0.12.3 # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in - # faststream types-aiobotocore==2.12.1 # via -r requirements/../../../packages/aws-library/requirements/_base.in types-aiobotocore-ec2==2.12.1 diff --git a/services/static-webserver/client/source/class/osparc/Application.js b/services/static-webserver/client/source/class/osparc/Application.js index 8435f9e42ed..c5f760188e5 100644 --- a/services/static-webserver/client/source/class/osparc/Application.js +++ b/services/static-webserver/client/source/class/osparc/Application.js @@ -63,14 +63,13 @@ qx.Class.define("osparc.Application", { return; } - const intlTelInput = osparc.wrapper.IntlTelInput.getInstance(); - intlTelInput.init(); + // libs + osparc.wrapper.IntlTelInput.getInstance().init(); + osparc.wrapper.Three.getInstance().init(); - const threejs = osparc.wrapper.Three.getInstance(); - threejs.init(); - - const announcementsTracker = osparc.announcement.Tracker.getInstance(); - announcementsTracker.startTracker(); + // trackers + osparc.announcement.Tracker.getInstance().startTracker(); + osparc.WindowSizeTracker.getInstance().startTracker(); const webSocket = osparc.wrapper.WebSocket.getInstance(); webSocket.addListener("connect", () => osparc.WatchDog.getInstance().setOnline(true)); @@ -467,6 +466,7 @@ qx.Class.define("osparc.Application", { __loadMainPage: function(studyId = null) { // logged in + osparc.WindowSizeTracker.getInstance().evaluateTooSmallDialog(); osparc.data.Resources.getOne("profile") .then(profile => { if (profile) { diff --git a/services/static-webserver/client/source/class/osparc/TooSmallDialog.js b/services/static-webserver/client/source/class/osparc/TooSmallDialog.js new file mode 100644 index 00000000000..b60219820f6 --- /dev/null +++ b/services/static-webserver/client/source/class/osparc/TooSmallDialog.js @@ -0,0 +1,82 @@ +/* ************************************************************************ + + osparc - the simcore frontend + + https://osparc.io + + Copyright: + 2024 IT'IS Foundation, https://itis.swiss + + License: + MIT: https://opensource.org/licenses/MIT + + Authors: + * Odei Maiz (odeimaiz) + +************************************************************************ */ + +qx.Class.define("osparc.TooSmallDialog", { + extend: osparc.ui.window.SingletonWindow, + + construct: function() { + this.base(arguments, "too-small-logout", this.tr("Window too small")); + + this.set({ + layout: new qx.ui.layout.VBox(10), + contentPadding: 15, + modal: true, + showMaximize: false, + showMinimize: false, + }); + + this.__buildLayout(); + }, + + statics: { + openWindow: function() { + const orgsWindow = new osparc.TooSmallDialog(); + orgsWindow.center(); + orgsWindow.open(); + return orgsWindow; + } + }, + + members: { + __buildLayout: function() { + const message = this.__createMessage(); + this.add(message); + + // if the user is logged in, let them log out, the user menu might be unreachable + const logoutButton = this.__createLogoutButton(); + this.add(logoutButton); + }, + + __createMessage: function() { + const introText = this.tr("The application can't perform in such a small window."); + const introLabel = new qx.ui.basic.Label(introText); + return introLabel; + }, + + __createLogoutButton: function() { + const layout = new qx.ui.container.Composite(new qx.ui.layout.HBox().set({ + alignX: "right" + })); + + const button = new qx.ui.form.Button().set({ + allowGrowX: false + }); + button.addListener("execute", () => qx.core.Init.getApplication().logout()); + layout.add(button); + + const authData = osparc.auth.Data.getInstance(); + authData.bind("loggedIn", layout, "visibility", { + converter: isLoggedIn => isLoggedIn ? "visible" : "excluded" + }); + authData.bind("guest", button, "label", { + converter: isGuest => isGuest ? this.tr("Exit") : this.tr("Log out") + }); + + return layout; + }, + } +}); diff --git a/services/static-webserver/client/source/class/osparc/WindowSizeTracker.js b/services/static-webserver/client/source/class/osparc/WindowSizeTracker.js index 671ec04e865..569120b3472 100644 --- a/services/static-webserver/client/source/class/osparc/WindowSizeTracker.js +++ b/services/static-webserver/client/source/class/osparc/WindowSizeTracker.js @@ -40,7 +40,7 @@ qx.Class.define("osparc.WindowSizeTracker", { }, tooSmall: { - check: [null, "shortText", "longText"], // display short message, long one or none + check: [null, "logout", "shortText", "longText"], init: null, nullable: true, apply: "__applyTooSmall" @@ -48,12 +48,14 @@ qx.Class.define("osparc.WindowSizeTracker", { }, statics: { + WIDTH_LOGOUT_BREAKPOINT: 600, + WIDTH_COMPACT_BREAKPOINT: 1100, WIDTH_BREAKPOINT: 1180, // - iPad Pro 11" 1194x834 inclusion HEIGHT_BREAKPOINT: 720, // - iPad Pro 11" 1194x834 inclusion - WIDTH_COMPACT_BREAKPOINT: 1100 }, members: { + __tooSmallDialog: null, __lastRibbonMessage: null, startTracker: function() { @@ -69,8 +71,12 @@ qx.Class.define("osparc.WindowSizeTracker", { this.setCompactVersion(width < this.self().WIDTH_COMPACT_BREAKPOINT); - if (width < this.self().WIDTH_BREAKPOINT || height < this.self().HEIGHT_BREAKPOINT) { - this.setTooSmall(width < this.self().WIDTH_COMPACT_BREAKPOINT ? "shortText" : "longText"); + if (width < this.self().WIDTH_LOGOUT_BREAKPOINT) { + this.setTooSmall("logout"); + } else if (width < this.self().WIDTH_COMPACT_BREAKPOINT) { + this.setTooSmall("shortText"); + } else if (width < this.self().WIDTH_BREAKPOINT) { + this.setTooSmall("longText"); } else { this.setTooSmall(null); } @@ -89,7 +95,7 @@ qx.Class.define("osparc.WindowSizeTracker", { } let notification = null; - if (tooSmall === "shortText") { + if (tooSmall === "logout" || tooSmall === "shortText") { notification = new osparc.notification.RibbonNotification(null, "smallWindow", true); } else if (tooSmall === "longText") { const text = this.__getLongText(true); @@ -97,6 +103,8 @@ qx.Class.define("osparc.WindowSizeTracker", { } osparc.notification.RibbonNotifications.getInstance().addNotification(notification); this.__lastRibbonMessage = notification; + + this.evaluateTooSmallDialog(); }, __getLongText: function() { @@ -111,6 +119,21 @@ qx.Class.define("osparc.WindowSizeTracker", { osparc.notification.RibbonNotifications.getInstance().removeNotification(this.__lastRibbonMessage); this.__lastRibbonMessage = null; } + }, + + evaluateTooSmallDialog: function() { + const tooSmall = this.getTooSmall(); + if (tooSmall === "logout") { + if (this.__tooSmallDialog) { + this.__tooSmallDialog.center(); + this.__tooSmallDialog.open(); + } else { + this.__tooSmallDialog = osparc.TooSmallDialog.openWindow(); + this.__tooSmallDialog.addListener("close", () => this.__tooSmallDialog = null, this); + } + } else if (this.__tooSmallDialog) { + this.__tooSmallDialog.close(); + } } } }); diff --git a/services/static-webserver/client/source/class/osparc/auth/Data.js b/services/static-webserver/client/source/class/osparc/auth/Data.js index 013a6f8ee51..276de264a4c 100644 --- a/services/static-webserver/client/source/class/osparc/auth/Data.js +++ b/services/static-webserver/client/source/class/osparc/auth/Data.js @@ -58,7 +58,8 @@ qx.Class.define("osparc.auth.Data", { auth: { init: null, nullable: true, - check: "osparc.io.request.authentication.Token" + check: "osparc.io.request.authentication.Token", + apply: "__applyAuth" }, /** @@ -104,10 +105,21 @@ qx.Class.define("osparc.auth.Data", { nullable: true, check: "Date", event: "changeExpirationDate" + }, + + loggedIn: { + check: "Boolean", + nullable: false, + init: false, + event: "changeLoggedIn", } }, members: { + __applyAuth: function(auth) { + this.setLoggedIn(auth !== null && auth instanceof osparc.io.request.authentication.Token); + }, + __applyRole: function(role) { if (role && ["user", "tester", "product_owner", "admin"].includes(role)) { this.setGuest(false); diff --git a/services/static-webserver/client/source/class/osparc/auth/Manager.js b/services/static-webserver/client/source/class/osparc/auth/Manager.js index c5c94bdad10..d45f7ce15a5 100644 --- a/services/static-webserver/client/source/class/osparc/auth/Manager.js +++ b/services/static-webserver/client/source/class/osparc/auth/Manager.js @@ -26,23 +26,10 @@ qx.Class.define("osparc.auth.Manager", { extend: qx.core.Object, type: "singleton", - /* - ***************************************************************************** - EVENTS - ***************************************************************************** - */ - events: { "loggedOut": "qx.event.type.Event" }, - - /* - ***************************************************************************** - MEMBERS - ***************************************************************************** - */ - members: { register: function(userData) { const params = { @@ -130,11 +117,7 @@ qx.Class.define("osparc.auth.Manager", { }, isLoggedIn: function() { - // TODO: how to store this locally?? See http://www.qooxdoo.org/devel/pages/data_binding/stores.html#offline-store - // TODO: check if expired?? - // TODO: request server if token is still valid (e.g. expired, etc) - const auth = osparc.auth.Data.getInstance().getAuth(); - return auth !== null && auth instanceof osparc.io.request.authentication.Token; + return osparc.auth.Data.getInstance().isLoggedIn(); }, /* diff --git a/services/static-webserver/client/source/class/osparc/dashboard/FolderWithSharedIcon.js b/services/static-webserver/client/source/class/osparc/dashboard/FolderWithSharedIcon.js index af6573fb16f..5e5b3dda20d 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/FolderWithSharedIcon.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/FolderWithSharedIcon.js @@ -23,27 +23,28 @@ qx.Class.define("osparc.dashboard.FolderWithSharedIcon", { this._setLayout(new qx.ui.layout.Canvas()); - this._createChildControlImpl("folder-icon"); - this._createChildControlImpl("shared-icon"); + this.getChildControl("folder-icon"); + this.getChildControl("shared-icon"); }, members: { _createChildControlImpl: function(id) { let control; switch (id) { - case "folder-icon": { - control = new qx.ui.basic.Image().set({ - source: "@FontAwesome5Solid/folder/26" - }); - const iconContainer = new qx.ui.container.Composite(new qx.ui.layout.HBox().set({ + case "icon-container": + control = new qx.ui.container.Composite(new qx.ui.layout.HBox().set({ alignY: "middle" })); - iconContainer.add(control); - this._add(iconContainer, { + this._add(control, { height: "100%" }); break; - } + case "folder-icon": + control = new qx.ui.basic.Image().set({ + source: "@FontAwesome5Solid/folder/26" + }); + this.getChildControl("icon-container").add(control); + break; case "shared-icon": control = new qx.ui.basic.Image().set({ textColor: "strong-main", diff --git a/services/static-webserver/client/source/class/osparc/dashboard/ResourceBrowserBase.js b/services/static-webserver/client/source/class/osparc/dashboard/ResourceBrowserBase.js index 9082c3a5c2b..9334861f11c 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/ResourceBrowserBase.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/ResourceBrowserBase.js @@ -282,7 +282,7 @@ qx.Class.define("osparc.dashboard.ResourceBrowserBase", { this._addToLayout(resourcesContainer); }, - _groupByChanged: function(groupBy) { + __groupByChanged: function(groupBy) { // if cards are grouped they need to be in grid mode this._resourcesContainer.setMode("grid"); this.__viewModeLayout.setVisibility(groupBy ? "excluded" : "visible"); @@ -290,7 +290,7 @@ qx.Class.define("osparc.dashboard.ResourceBrowserBase", { this._reloadCards(); }, - _viewByChanged: function(viewMode) { + __viewModeChanged: function(viewMode) { this._resourcesContainer.setMode(viewMode); this._reloadCards(); @@ -314,14 +314,14 @@ qx.Class.define("osparc.dashboard.ResourceBrowserBase", { const dontGroup = new qx.ui.menu.RadioButton(this.tr("None")); osparc.utils.Utils.setIdToWidget(dontGroup, "groupByNone"); - dontGroup.addListener("execute", () => this._groupByChanged(null)); + dontGroup.addListener("execute", () => this.__groupByChanged(null)); groupByMenu.add(dontGroup); groupOptions.add(dontGroup); if (this._resourceType === "template") { const tagByGroup = new qx.ui.menu.RadioButton(this.tr("Tags")); - tagByGroup.addListener("execute", () => this._groupByChanged("tags")); + tagByGroup.addListener("execute", () => this.__groupByChanged("tags")); groupByMenu.add(tagByGroup); groupOptions.add(tagByGroup); if ( @@ -334,7 +334,7 @@ qx.Class.define("osparc.dashboard.ResourceBrowserBase", { } const groupByShared = new qx.ui.menu.RadioButton(this.tr("Shared with")); - groupByShared.addListener("execute", () => this._groupByChanged("shared")); + groupByShared.addListener("execute", () => this.__groupByChanged("shared")); groupByMenu.add(groupByShared); groupOptions.add(groupByShared); @@ -343,10 +343,10 @@ qx.Class.define("osparc.dashboard.ResourceBrowserBase", { _addViewModeButton: function() { const gridBtn = this.self().createToolbarRadioButton(null, "@FontAwesome5Solid/th/14", this.tr("Grid view"), "left"); - gridBtn.addListener("execute", () => this._viewByChanged("grid")); + gridBtn.addListener("execute", () => this.__viewModeChanged("grid")); const listBtn = this.self().createToolbarRadioButton(null, "@FontAwesome5Solid/bars/14", this.tr("List view"), "right"); - listBtn.addListener("execute", () => this._viewByChanged("list")); + listBtn.addListener("execute", () => this.__viewModeChanged("list")); const viewModeLayout = this.__viewModeLayout; const radioGroup = new qx.ui.form.RadioGroup(); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/ResourceContainerManager.js b/services/static-webserver/client/source/class/osparc/dashboard/ResourceContainerManager.js index f0d854b2222..df41744539d 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/ResourceContainerManager.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/ResourceContainerManager.js @@ -59,8 +59,7 @@ qx.Class.define("osparc.dashboard.ResourceContainerManager", { check: ["grid", "list"], init: "grid", nullable: false, - event: "changeMode", - apply: "__reloadCards" + event: "changeMode" }, groupBy: { @@ -277,10 +276,6 @@ qx.Class.define("osparc.dashboard.ResourceContainerManager", { this._removeAll(); }, - __reloadCards: function(mode) { - this.reloadCards(); - }, - __addFoldersContainer: function() { // add foldersContainer dynamically [ @@ -301,7 +296,7 @@ qx.Class.define("osparc.dashboard.ResourceContainerManager", { }); }, - reloadCards: function(resourceType) { + __rebuildLayout: function(resourceType) { this.__cleanAll(); if (osparc.utils.DisabledPlugins.isFoldersEnabled()) { this.__addFoldersContainer(); @@ -326,8 +321,12 @@ qx.Class.define("osparc.dashboard.ResourceContainerManager", { }); this._add(this.__nonGroupedContainer); } + }, - let cards = []; + reloadCards: function(resourceType) { + this.__rebuildLayout(resourceType); + + const cards = []; this.__resourcesList.forEach(resourceData => { Array.prototype.push.apply(cards, this.__resourceToCards(resourceData)); }); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js index 44fab7818ee..165b2dff7e0 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js @@ -104,6 +104,7 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { __workspaceHeader: null, __workspacesList: null, __foldersList: null, + __loadingFolders: null, // overridden initResources: function() { @@ -170,27 +171,35 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { }, __reloadFolders: function() { - if (osparc.utils.DisabledPlugins.isFoldersEnabled()) { - const folderId = this.getCurrentFolderId(); - const workspaceId = this.getCurrentWorkspaceId(); - if (workspaceId === -1 || workspaceId === -2) { - return; - } - this.__setFoldersToList([]); - osparc.store.Folders.getInstance().fetchFolders(folderId, workspaceId, this.getOrderBy()) - .then(folders => { - this.__setFoldersToList(folders); - }) - .catch(console.error); + if ( + !osparc.auth.Manager.getInstance().isLoggedIn() || + !osparc.utils.DisabledPlugins.isFoldersEnabled() || + this.__loadingFolders) { + return; + } + const workspaceId = this.getCurrentWorkspaceId(); + if (workspaceId === -1 || workspaceId === -2) { + return; } + + this.__loadingFolders = true; + this.__setFoldersToList([]); + const folderId = this.getCurrentFolderId(); + osparc.store.Folders.getInstance().fetchFolders(folderId, workspaceId, this.getOrderBy()) + .then(folders => { + this.__setFoldersToList(folders); + }) + .catch(console.error) + .finally(() => this.__loadingFolders = null); }, __reloadStudies: function() { - if (this._loadingResourcesBtn.isFetching()) { - return; - } const workspaceId = this.getCurrentWorkspaceId(); - if (workspaceId === -1) { // shared workspace listing + if ( + !osparc.auth.Manager.getInstance().isLoggedIn() || + workspaceId === -1 || // listing workspaces + this._loadingResourcesBtn.isFetching() + ) { return; } @@ -279,6 +288,7 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { __resetStudiesList: function() { this._resourcesList = []; + // It will remove the study cards this._reloadCards(); }, @@ -689,7 +699,9 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { invalidateStudies: function() { osparc.store.Store.getInstance().invalidate("studies"); this.__resetStudiesList(); - this._resourcesContainer.getFlatList().nextRequest = null; + if (this._resourcesContainer.getFlatList()) { + this._resourcesContainer.getFlatList().nextRequest = null; + } }, __addNewStudyButtons: function() { @@ -934,6 +946,7 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { return; } + this._loadingResourcesBtn.setFetching(false); this.resetSelection(); this.setMultiSelection(false); this.set({ @@ -1229,6 +1242,7 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { } else { this._resourcesList[index] = studyData; } + // it will render the studies in the right order this._reloadCards(); }, diff --git a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonBase.js b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonBase.js index 22f238b0fd1..c0c93cc9508 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonBase.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonBase.js @@ -68,14 +68,14 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonBase", { ITEM_WIDTH: 190, ITEM_HEIGHT: 190, PADDING: 10, - SPACING_IN: 5, SPACING: 15, HEADER_MAX_HEIGHT: 40, // two lines in Manrope ICON_SIZE: 60, POS: { - HEADER: 0, - BODY: 1, - FOOTER: 2 + FOLDER_LOOK: 0, + HEADER: 1, + BODY: 2, + FOOTER: 3 }, HPOS: { SHARED: 0, @@ -107,10 +107,12 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonBase", { let control; switch (id) { case "main-layout": { - control = new qx.ui.container.Composite(new qx.ui.layout.VBox(this.self().SPACING_IN)); + control = new qx.ui.container.Composite(new qx.ui.layout.VBox()); + const folderLook = this.getChildControl("folder-look"); const header = this.getChildControl("header"); const body = this.getChildControl("body"); const footer = this.getChildControl("footer"); + control.addAt(folderLook, this.self().POS.FOLDER_LOOK); control.addAt(header, this.self().POS.HEADER); control.addAt(body, this.self().POS.BODY, { flex: 1 @@ -124,9 +126,14 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonBase", { }); break; } + case "folder-look": { + control = this.__createFolderLookHeader(); + break; + } case "header": control = new qx.ui.container.Composite(new qx.ui.layout.HBox(5)).set({ - backgroundColor: "background-card-overlay", + backgroundColor: "background-workspace-card-overlay", + opacity: 0.8, anonymous: true, maxWidth: this.self().ITEM_WIDTH, maxHeight: this.self().HEADER_MAX_HEIGHT, @@ -135,7 +142,7 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonBase", { }); break; case "body": - control = new qx.ui.container.Composite(new qx.ui.layout.VBox(5)).set({ + control = new qx.ui.container.Composite(new qx.ui.layout.VBox()).set({ decorator: "main", allowGrowY: true, allowGrowX: true, @@ -162,6 +169,7 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonBase", { textColor: "contrasted-text-light", font: "text-14", allowGrowX: true, + alignY: "middle", maxHeight: this.self().HEADER_MAX_HEIGHT }); layout = this.getChildControl("header"); @@ -190,6 +198,70 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonBase", { return control || this.base(arguments, id); }, + __createFolderLookHeader: function() { + const topHeight = 8; + const grid = new qx.ui.layout.Grid(0, 0); + grid.setColumnFlex(0, 1); + grid.setColumnFlex(2, 1); + grid.setRowHeight(0, topHeight); + grid.setRowHeight(1, 4); + const layout = new qx.ui.container.Composite(grid).set({ + backgroundColor: "background-main", + }); + const spacer00 = new qx.ui.core.Widget().set({ + backgroundColor: "background-workspace-card-overlay" + }); + const triangle = new qx.ui.core.Widget().set({ + width: topHeight, + height: topHeight, + }); + triangle.getContentElement().setStyles({ + "width": "0", + "height": "0", + "border-right": topHeight + "px solid transparent", + }); + const colorTriangle = () => { + const color = qx.theme.manager.Color.getInstance().resolve("background-workspace-card-overlay"); + triangle.getContentElement().setStyles({ + "border-bottom": topHeight + "px solid " + color, + }); + }; + colorTriangle(); + qx.theme.manager.Color.getInstance().addListener("changeTheme", colorTriangle); + const spacer01 = new qx.ui.core.Widget(); + const spacer10 = new qx.ui.core.Widget().set({ + backgroundColor: "background-workspace-card-overlay" + }); + const spacer11 = new qx.ui.core.Widget().set({ + backgroundColor: "background-workspace-card-overlay" + }); + spacer11.getContentElement().setStyles({ + "border-top-right-radius": "4px", + }); + layout.add(spacer00, { + row: 0, + column: 0, + }); + layout.add(triangle, { + row: 0, + column: 1, + }); + layout.add(spacer01, { + row: 0, + column: 2, + }); + layout.add(spacer10, { + row: 1, + column: 0, + colSpan: 2, + }); + layout.add(spacer11, { + row: 1, + column: 2, + }); + return layout; + }, + // overridden _applyIcon: function(value) { const image = this.getChildControl("icon").getChildControl("image"); diff --git a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonNew.js b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonNew.js index 4bdaf22a465..fc1526b387d 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonNew.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/WorkspaceButtonNew.js @@ -39,6 +39,9 @@ qx.Class.define("osparc.dashboard.WorkspaceButtonNew", { this.setIcon(osparc.dashboard.CardBase.NEW_ICON); + this.getChildControl("header").set({ + opacity: 1 + }); this.getChildControl("footer").exclude(); }, diff --git a/services/static-webserver/client/source/class/osparc/data/Resources.js b/services/static-webserver/client/source/class/osparc/data/Resources.js index 3c2d4d500f6..9d591c0d1ee 100644 --- a/services/static-webserver/client/source/class/osparc/data/Resources.js +++ b/services/static-webserver/client/source/class/osparc/data/Resources.js @@ -1370,7 +1370,7 @@ qx.Class.define("osparc.data.Resources", { params["url"] = {}; } params["url"]["offset"] = offset; - params["url"]["limit"] = 20; + params["url"]["limit"] = 10; const endpoint = "getPage"; const options = { resolveWResponse: true diff --git a/services/static-webserver/client/source/class/osparc/data/model/Study.js b/services/static-webserver/client/source/class/osparc/data/model/Study.js index 2d4633dd584..912f838486f 100644 --- a/services/static-webserver/client/source/class/osparc/data/model/Study.js +++ b/services/static-webserver/client/source/class/osparc/data/model/Study.js @@ -443,6 +443,39 @@ qx.Class.define("osparc.data.model.Study", { }); }, + __getFoldersPath: function(childId, foldersPath = []) { + foldersPath.unshift(childId); + const childFolder = osparc.store.Folders.getInstance().getFolder(childId); + if (childFolder) { + const parentFolder = osparc.store.Folders.getInstance().getFolder(childFolder.getParentFolderId()); + if (parentFolder) { + this.__getFoldersPath(parentFolder.getFolderId(), foldersPath); + } + } + return foldersPath; + }, + + getLocationString: function() { + const location = []; + + if (this.getWorkspaceId()) { + const workspace = osparc.store.Workspaces.getInstance().getWorkspace(this.getWorkspaceId()); + location.push(workspace.getName()); + } else { + location.push(qx.locale.Manager.tr("My Workspace")); + } + + const foldersPathIds = this.__getFoldersPath(this.getFolderId()); + foldersPathIds.forEach(folderId => { + const folder = osparc.store.Folders.getInstance().getFolder(folderId); + if (folder) { + location.push(folder.getName()); + } + }); + + return location.join(" / "); + }, + // Used for updating some node data through the "nodeUpdated" websocket event nodeUpdated: function(nodeUpdatedData) { const studyId = nodeUpdatedData["project_id"]; diff --git a/services/static-webserver/client/source/class/osparc/desktop/MainPage.js b/services/static-webserver/client/source/class/osparc/desktop/MainPage.js index f84d436ffd5..d2b72acfdcc 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/MainPage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/MainPage.js @@ -54,7 +54,6 @@ qx.Class.define("osparc.desktop.MainPage", { this._add(navBar); // Some resources request before building the main stack - osparc.WindowSizeTracker.getInstance().startTracker(); osparc.MaintenanceTracker.getInstance().startTracker(); osparc.CookieExpirationTracker.getInstance().startTracker(); osparc.NewUITracker.getInstance().startTracker(); diff --git a/services/static-webserver/client/source/class/osparc/info/StudyLarge.js b/services/static-webserver/client/source/class/osparc/info/StudyLarge.js index d058f584aca..3351ed0fc96 100644 --- a/services/static-webserver/client/source/class/osparc/info/StudyLarge.js +++ b/services/static-webserver/client/source/class/osparc/info/StudyLarge.js @@ -189,6 +189,16 @@ qx.Class.define("osparc.info.StudyLarge", { }; } + if (osparc.utils.DisabledPlugins.isFoldersEnabled() && !this.__isTemplate) { + const pathLabel = new qx.ui.basic.Label(); + pathLabel.setValue(this.getStudy().getLocationString()); + extraInfo["LOCATION"] = { + label: this.tr("Location:"), + view: pathLabel, + action: null + }; + } + return extraInfo; }, diff --git a/services/static-webserver/client/source/class/osparc/info/StudyUtils.js b/services/static-webserver/client/source/class/osparc/info/StudyUtils.js index 0e10bc93d1f..95ea7f20b7f 100644 --- a/services/static-webserver/client/source/class/osparc/info/StudyUtils.js +++ b/services/static-webserver/client/source/class/osparc/info/StudyUtils.js @@ -299,7 +299,12 @@ qx.Class.define("osparc.info.StudyUtils", { inline: true, column: 0, row: 6, - } + }, + LOCATION: { + inline: true, + column: 0, + row: 7, + }, }; const grid = new qx.ui.layout.Grid(15, 5); diff --git a/services/static-webserver/client/source/class/osparc/theme/ColorDark.js b/services/static-webserver/client/source/class/osparc/theme/ColorDark.js index 0e179de8059..0d4085c3153 100644 --- a/services/static-webserver/client/source/class/osparc/theme/ColorDark.js +++ b/services/static-webserver/client/source/class/osparc/theme/ColorDark.js @@ -32,6 +32,7 @@ qx.Theme.define("osparc.theme.ColorDark", { "background-main-5": "c06", "background-card-overlay": "rgba(25, 33, 37, 0.8)", + "background-workspace-card-overlay": "rgb(35, 93, 122)", "primary-background-color": "rgba(0, 20, 46, 1)", "navigation_bar_background_color": "rgba(1, 18, 26, 0.8)", diff --git a/services/static-webserver/client/source/class/osparc/theme/ColorLight.js b/services/static-webserver/client/source/class/osparc/theme/ColorLight.js index 4af8f46d180..6feedc258b6 100644 --- a/services/static-webserver/client/source/class/osparc/theme/ColorLight.js +++ b/services/static-webserver/client/source/class/osparc/theme/ColorLight.js @@ -32,6 +32,7 @@ qx.Theme.define("osparc.theme.ColorLight", { "background-main-5": "c06", "background-card-overlay": "rgba(229, 229, 229, 0.8)", + "background-workspace-card-overlay": "rgb(165, 223, 252)", "primary-background-color": "rgba(255, 255, 255, 1)", "navigation_bar_background_color": "rgba(229, 229, 229, 0.8)", diff --git a/services/static-webserver/client/source/class/osparc/utils/Utils.js b/services/static-webserver/client/source/class/osparc/utils/Utils.js index d4b8aa9de48..8124b6818e1 100644 --- a/services/static-webserver/client/source/class/osparc/utils/Utils.js +++ b/services/static-webserver/client/source/class/osparc/utils/Utils.js @@ -804,11 +804,10 @@ qx.Class.define("osparc.utils.Utils", { loadedCb(); } const blob = new Blob([xhr.response]); - const urlBlob = window.URL.createObjectURL(blob); if (!fileName) { fileName = this.self().filenameFromContentDisposition(xhr); } - this.self().downloadContent(urlBlob, fileName); + this.self().downloadBlobContent(blob, fileName); resolve(); } else { reject(xhr); @@ -820,9 +819,9 @@ qx.Class.define("osparc.utils.Utils", { }); }, - downloadContent: function(content, filename = "file") { + downloadBlobContent: function(blob, filename = "file") { let downloadAnchorNode = document.createElement("a"); - downloadAnchorNode.setAttribute("href", content); + downloadAnchorNode.setAttribute("href", window.URL.createObjectURL(blob)); downloadAnchorNode.setAttribute("download", filename); downloadAnchorNode.click(); downloadAnchorNode.remove(); diff --git a/services/static-webserver/client/source/class/osparc/viewer/MainPage.js b/services/static-webserver/client/source/class/osparc/viewer/MainPage.js index 073d7d249ef..1586023b062 100644 --- a/services/static-webserver/client/source/class/osparc/viewer/MainPage.js +++ b/services/static-webserver/client/source/class/osparc/viewer/MainPage.js @@ -29,7 +29,6 @@ qx.Class.define("osparc.viewer.MainPage", { navBar.populateLayout(); this._add(navBar); - osparc.WindowSizeTracker.getInstance().startTracker(); osparc.MaintenanceTracker.getInstance().startTracker(); const nodeViewer = new osparc.viewer.NodeViewer(studyId, viewerNodeId); diff --git a/services/static-webserver/client/source/class/osparc/widget/logger/LoggerView.js b/services/static-webserver/client/source/class/osparc/widget/logger/LoggerView.js index 3ad1b437301..e77ff8c6840 100644 --- a/services/static-webserver/client/source/class/osparc/widget/logger/LoggerView.js +++ b/services/static-webserver/client/source/class/osparc/widget/logger/LoggerView.js @@ -317,9 +317,10 @@ qx.Class.define("osparc.widget.logger.LoggerView", { }, __getLogsString: function() { + const newLine = "\n"; let logs = ""; this.__loggerModel.getFilteredRows().forEach(rowData => { - logs += this.self().printRow(rowData) + "\n"; + logs += this.self().printRow(rowData) + newLine; }); return logs; }, @@ -338,7 +339,8 @@ qx.Class.define("osparc.widget.logger.LoggerView", { downloadLogs: function() { const logs = this.__getLogsString(); - osparc.utils.Utils.downloadContent("data:text/plain;charset=utf-8," + logs, "logs.log"); + const blob = new Blob([logs], {type: "text/plain"}); + osparc.utils.Utils.downloadBlobContent(blob, "logs.log"); }, debug: function(nodeId, msg = "") { diff --git a/services/static-webserver/client/source/resource/osparc/S4LEngine_ComingSoon.html b/services/static-webserver/client/source/resource/osparc/S4LEngine_ComingSoon.html index e45e9bc2d79..3b40620bd6e 100644 --- a/services/static-webserver/client/source/resource/osparc/S4LEngine_ComingSoon.html +++ b/services/static-webserver/client/source/resource/osparc/S4LEngine_ComingSoon.html @@ -9,7 +9,7 @@ position: fixed; top: 0; left: 0; - /* Preserve aspet ratio */ + /* Preserve aspect ratio */ min-width: 100%; min-height: 100%; } diff --git a/services/storage/Dockerfile b/services/storage/Dockerfile index 12e60feb78d..8c9f0e381c0 100644 --- a/services/storage/Dockerfile +++ b/services/storage/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # USAGE: @@ -78,28 +83,21 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ build-essential \ git -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install only base 3rd party dependencies -RUN \ - --mount=type=bind,source=services/storage/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # -------------------------- ------------------------------- @@ -122,9 +120,9 @@ WORKDIR /build/services/storage RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/storage,target=/build/services/storage,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list # -------------------------- ------------------------------- @@ -141,6 +139,8 @@ ENV SC_BUILD_TARGET=production \ SC_HEALTHCHECK_RETRY=3 ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index 8f4c4f64346..ea9bf8bd877 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -18,12 +18,11 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/storage || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/storage + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list echo "$INFO" "Setting entrypoint to use watchmedo autorestart..." entrypoint='watchmedo auto-restart --recursive --pattern="*.py;*/src/*" --ignore-patterns="*test*;pytest_simcore/*;setup.py;*ignore*" --ignore-directories --' @@ -32,6 +31,11 @@ elif [ "${SC_BUILD_TARGET}" = "production" ]; then entrypoint="" fi +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy +fi + APP_LOG_LEVEL=${STORAGE_LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL:-INFO}}} SERVER_LOG_LEVEL=$(echo "${APP_LOG_LEVEL}" | tr '[:upper:]' '[:lower:]') diff --git a/services/storage/docker/entrypoint.sh b/services/storage/docker/entrypoint.sh index 076dd425e67..11684a3a591 100755 --- a/services/storage/docker/entrypoint.sh +++ b/services/storage/docker/entrypoint.sh @@ -15,7 +15,6 @@ ERROR="ERROR: [$(basename "$0")] " # It needs to be executed as root. update-ca-certificates - # This entrypoint script: # # - Executes *inside* of the container upon start as --user [default root] @@ -29,7 +28,6 @@ echo "$INFO" "User : $(id scu)" echo "$INFO" "python : $(command -v python)" echo "$INFO" "pip : $(command -v pip)" - if [ "${SC_BUILD_TARGET}" = "development" ]; then echo "$INFO" "development mode detected..." # NOTE: expects docker run ... -v $(pwd):$DEVEL_MOUNT @@ -70,11 +68,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index 0e2728ec7a0..d5a191d6e58 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -127,7 +127,7 @@ email-validator==2.1.1 # via pydantic fast-depends==2.4.12 # via faststream -faststream==0.5.10 +faststream==0.5.28 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -497,7 +497,6 @@ typer==0.12.3 # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in - # faststream types-aiobotocore==2.12.1 # via # -r requirements/../../../packages/aws-library/requirements/_base.in diff --git a/services/web/Dockerfile b/services/web/Dockerfile index 1a1a443333a..e7ff6b3fe41 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # USAGE: @@ -81,29 +86,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ build-essential \ libffi-dev -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install only base 3rd party dependencies -RUN \ - --mount=type=bind,source=services/web/server/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- # This stage is for production only dependencies that get partially wiped out afterwards (final docker image concerns) @@ -121,9 +119,9 @@ WORKDIR /build/services/web/server RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/web/server,target=/build/services/web/server,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list # --------------------------Production stage ------------------- @@ -139,6 +137,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/web/server/docker/boot.sh b/services/web/server/docker/boot.sh index e6b36ce5788..08a812f7bb0 100755 --- a/services/web/server/docker/boot.sh +++ b/services/web/server/docker/boot.sh @@ -18,18 +18,22 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/web/server || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/web/server + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list APP_CONFIG=server-docker-dev.yaml elif [ "${SC_BUILD_TARGET}" = "production" ]; then APP_CONFIG=server-docker-prod.yaml fi +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy +fi + APP_LOG_LEVEL=${WEBSERVER_LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL:-INFO}}} SERVER_LOG_LEVEL=$(echo "${APP_LOG_LEVEL}" | tr '[:upper:]' '[:lower:]') diff --git a/services/web/server/docker/entrypoint.sh b/services/web/server/docker/entrypoint.sh index d488a0ef2c7..d7e4f13bb7a 100755 --- a/services/web/server/docker/entrypoint.sh +++ b/services/web/server/docker/entrypoint.sh @@ -71,11 +71,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index 2eab661ddef..a98c8f71307 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -162,7 +162,7 @@ faker==19.6.1 # via -r requirements/_base.in fast-depends==2.4.12 # via faststream -faststream==0.5.10 +faststream==0.5.28 # via # -r requirements/../../../../packages/service-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in @@ -564,7 +564,6 @@ typer==0.12.3 # -r requirements/../../../../packages/settings-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in - # faststream typing-extensions==4.12.0 # via # aiodebug diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index 467c300de83..0a0e6a66fc1 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -145,7 +145,7 @@ email-validator==2.2.0 # via pydantic fast-depends==2.4.12 # via faststream -faststream==0.5.23 +faststream==0.5.28 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in