♻️Maintenance: Improve UV usage (ITISFoundation#6566)

mrnicegyu11 · Oct 22, 2024 · ea88f43 · ea88f43
1 parent e5dca04
commit ea88f43
Show file tree

Hide file tree

Showing 60 changed files with 636 additions and 615 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -8,7 +8,7 @@ ops/
 *.py[cod]
 
 # virtualenv
-.venv
+**/.venv
 
 #python eggs
 **/*.egg-info

diff --git a/packages/postgres-database/docker/Dockerfile b/packages/postgres-database/docker/Dockerfile
@@ -1,5 +1,9 @@
 # syntax=docker/dockerfile:1
-FROM python:3.6-slim AS base
+ARG PYTHON_VERSION="3.11.9"
+ARG UV_VERSION="0.4"
+FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build
+# we docker image is built based on debian
+FROM python:${PYTHON_VERSION}-slim-bookworm AS base
 
 LABEL maintainer=sanderegg
 
@@ -22,16 +26,14 @@ RUN apt-get update \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
-# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install uv~=0.2
+# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv
+COPY --from=uv_build /uv /uvx /bin/
 
 # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv
 RUN uv venv "${VIRTUAL_ENV}"
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \
+RUN --mount=type=cache,target=/root/.cache/uv \
   uv pip install --upgrade \
-  pip~=24.0  \
   wheel \
   setuptools
 
@@ -44,6 +46,8 @@ RUN git clone --single-branch --branch ${GIT_BRANCH} ${GIT_REPOSITORY} osparc-si
 FROM base AS production
 
 ENV PYTHONOPTIMIZE=TRUE
+# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode
+ENV UV_COMPILE_BYTECODE=1
 
 WORKDIR /home/scu
 # ensure home folder is read/writable for user scu

diff --git a/packages/postgres-database/scripts/erd/Dockerfile b/packages/postgres-database/scripts/erd/Dockerfile
@@ -1,5 +1,10 @@
 # syntax=docker/dockerfile:1
+
+# Define arguments in the global scope
 ARG PYTHON_VERSION="3.11.9"
+ARG UV_VERSION="0.4"
+FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build
+# we docker image is built based on debian
 FROM python:${PYTHON_VERSION}-slim-bookworm AS base
 
 RUN apt-get update \
@@ -15,15 +20,14 @@ RUN apt-get update \
   && apt-get clean
 
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install --upgrade \
-  pip~=24.0  \
+RUN --mount=type=cache,target=/root/.cache/uv \
+  uv pip install --upgrade \
   wheel \
   setuptools
 
 
 # devenv
 COPY requirements.txt requirements.txt
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install \
-  -r requirements.txt
+RUN --mount=type=cache,target=/root/.cache/uv \
+  uv pip sync \
+  requirements.txt
diff --git a/packages/service-integration/Dockerfile b/packages/service-integration/Dockerfile
@@ -1,5 +1,10 @@
 # syntax=docker/dockerfile:1
+
+# Define arguments in the global scope
 ARG PYTHON_VERSION="3.11.9"
+ARG UV_VERSION="0.4"
+FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build
+# we docker image is built based on debian
 FROM python:${PYTHON_VERSION}-slim-bookworm AS base
 
 LABEL maintainer=pcrespov
@@ -38,12 +43,13 @@ ENV LANG=C.UTF-8
 # Turns off writing .pyc files; superfluous on an ephemeral container.
 ENV PYTHONDONTWRITEBYTECODE=1 \
   VIRTUAL_ENV=/home/scu/.venv
+# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode
+ENV UV_COMPILE_BYTECODE=1
 
 # Ensures that the python and pip executables used
 # in the image will be those from our virtualenv.
 ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
 
-
 # -------------------------- Build stage -------------------
 
 FROM base AS build
@@ -55,15 +61,14 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
   && apt-get install -y --no-install-recommends \
   build-essential
 
-# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install uv~=0.2
+# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv
+COPY --from=uv_build /uv /uvx /bin/
 
 # NOTE: python virtualenv is used here such that installed
 # packages may be moved to production image easily by copying the venv
 RUN uv venv "${VIRTUAL_ENV}"
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \
+RUN --mount=type=cache,target=/root/.cache/uv \
   uv pip install --upgrade \
   pip~=24.0  \
   wheel \
@@ -74,7 +79,7 @@ WORKDIR /build/packages/service-integration
 RUN \
   --mount=type=bind,source=packages,target=/build/packages,rw \
   --mount=type=bind,source=packages/service-integration,target=/build/packages/service-integration,rw \
-  --mount=type=cache,mode=0755,target=/root/.cache/uv \
+  --mount=type=cache,target=/root/.cache/uv \
   uv pip install \
   --requirement requirements/prod.txt \
   && uv pip list
@@ -86,8 +91,6 @@ FROM base AS development
 # NOTE: this is necessary to allow to build development images but is the same as production here
 FROM base AS production
 
-ENV PYTHONOPTIMIZE=TRUE
-
 WORKDIR /home/scu
 # ensure home folder is read/writable for user scu
 RUN chown -R scu /home/scu

diff --git a/requirements/tools/Dockerfile b/requirements/tools/Dockerfile
@@ -9,6 +9,9 @@
 #
 #
 ARG PYTHON_VERSION="3.11.9"
+ARG UV_VERSION="0.4"
+FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build
+# we docker image is built based on debian
 FROM python:${PYTHON_VERSION}-slim-bookworm AS base
 
 ENV VIRTUAL_ENV=/home/scu/.venv
@@ -24,24 +27,21 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
   && apt-get clean -y
 
 
-# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install uv~=0.2
+# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv
+COPY --from=uv_build /uv /uvx /bin/
 
 RUN uv venv "${VIRTUAL_ENV}"
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \
+RUN --mount=type=cache,target=/root/.cache/uv \
   uv pip install --upgrade  \
-  pip~=24.0  \
   wheel \
   setuptools
 
 
 
 # devenv
-RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \
+RUN --mount=type=cache,target=/root/.cache/uv \
   uv pip install \
-  pip-tools \
   pipreqs \
   pipdeptree && \
   uv pip list -vv
diff --git a/scripts/erd/Dockerfile b/scripts/erd/Dockerfile
@@ -8,6 +8,9 @@
 #
 
 ARG PYTHON_VERSION="3.11.9"
+ARG UV_VERSION="0.4"
+FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build
+# we docker image is built based on debian
 FROM python:${PYTHON_VERSION}-slim-bookworm AS base
 
 RUN apt-get update \
@@ -22,13 +25,14 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/* \
   && apt-get clean
 
+# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv
+COPY --from=uv_build /uv /uvx /bin/
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install --upgrade \
-  pip~=24.0  \
+RUN --mount=type=cache,target=/root/.cache/uv \
+  uv pip install --upgrade \
   wheel \
   setuptools
 
 COPY requirements.txt .
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install -r requirements.txt
+RUN --mount=type=cache,target=/root/.cache/uv \
+  uv pip install -r requirements.txt
diff --git a/scripts/maintenance/migrate_project/Dockerfile b/scripts/maintenance/migrate_project/Dockerfile
@@ -1,21 +1,27 @@
 # syntax=docker/dockerfile:1
+ARG UV_VERSION="0.4"
+FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build
+# we docker image is built based on debian
 FROM python:3.11.9-buster
 
 RUN curl https://rclone.org/install.sh | bash && \
   rclone --version
 
+# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv
+COPY --from=uv_build /uv /uvx /bin/
+
 WORKDIR /scripts
 
 COPY packages/postgres-database postgres-database
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  cd postgres-database && pip install .
+RUN --mount=type=cache,target=/root/.cache/uv \
+  cd postgres-database && uv pip install .
 
 COPY packages/settings-library settings-library
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  cd settings-library && pip install .
+RUN --mount=type=cache,target=/root/.cache/uv \
+  cd settings-library && uv pip install .
 
 COPY scripts/maintenance/migrate_project/requirements.txt /scripts/requirements.txt
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install -r /scripts/requirements.txt
+RUN --mount=type=cache,target=/root/.cache/uv \
+  uv pip install -r /scripts/requirements.txt
 
 COPY scripts/maintenance/migrate_project/src/*.py /scripts/
diff --git a/scripts/openapi/oas_resolver/Dockerfile b/scripts/openapi/oas_resolver/Dockerfile
@@ -2,6 +2,9 @@
 # Usage:
 # docker build . -t oas_resolver
 # docker run -v /path/to/api:/input -v /path/to/compiled/file:/output oas_resolver /input/path/to/openapi.yaml /output/output_file.yaml
+ARG UV_VERSION="0.4"
+FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build
+# we docker image is built based on debian
 FROM python:3.6-alpine
 
 LABEL maintainer=sanderegg
@@ -11,16 +14,16 @@ VOLUME [ "/output" ]
 
 WORKDIR /src
 
+# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv
+COPY --from=uv_build /uv /uvx /bin/
+
 # update pip
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install --upgrade \
-  pip~=24.0  \
+RUN --mount=type=cache,target=/root/.cache/uv \
+  uv pip install --upgrade \
   wheel \
   setuptools
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install prance && \
-  pip install click &&\
-  pip install openapi_spec_validator
+RUN --mount=type=cache,target=/root/.cache/uv \
+  uv pip install prance click openapi_spec_validator
 
 ENTRYPOINT [ "prance", "compile" , "--backend=openapi-spec-validator"]
diff --git a/scripts/pydeps-docker/Dockerfile b/scripts/pydeps-docker/Dockerfile
@@ -9,6 +9,9 @@
 #
 #
 ARG PYTHON_VERSION="3.11.9"
+ARG UV_VERSION="0.4"
+FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build
+# we docker image is built based on debian
 FROM python:${PYTHON_VERSION}-slim-bookworm AS base
 
 
@@ -23,18 +26,20 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/* \
   && apt-get clean
 
+# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv
+COPY --from=uv_build /uv /uvx /bin/
+
 ARG HOME_DIR
 RUN mkdir -p ${HOME_DIR}
 COPY .pydeps ${HOME_DIR}/.pydeps
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install --upgrade \
-  pip~=24.0  \
+RUN --mount=type=cache,target=/root/.cache/uv \
+  uv pip install --upgrade \
   wheel \
   setuptools
 
 
 # devenv
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install \
+RUN --mount=type=cache,target=/root/.cache/uv \
+  uv pip install \
   pydeps
diff --git a/services/agent/Dockerfile b/services/agent/Dockerfile
@@ -1,5 +1,10 @@
 # syntax=docker/dockerfile:1
+
+# Define arguments in the global scope
 ARG PYTHON_VERSION="3.11.9"
+ARG UV_VERSION="0.4"
+FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build
+# we docker image is built based on debian
 FROM python:${PYTHON_VERSION}-slim-bookworm AS base
 
 #
@@ -73,29 +78,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
   && apt-get install -y --no-install-recommends \
   build-essential
 
-# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-  pip install uv~=0.2
+# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv
+COPY --from=uv_build /uv /uvx /bin/
 
 # NOTE: python virtualenv is used here such that installed
 # packages may be moved to production image easily by copying the venv
 RUN uv venv "${VIRTUAL_ENV}"
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \
+RUN --mount=type=cache,target=/root/.cache/uv \
   uv pip install --upgrade  \
-  pip~=24.0  \
   wheel \
   setuptools
 
 WORKDIR /build
 
 # install base 3rd party dependencies
-# NOTE: copies to /build to avoid overwriting later which would invalidate this layer
-RUN \
-  --mount=type=bind,source=services/agent/requirements/_base.txt,target=_base.txt \
-  --mount=type=cache,mode=0755,target=/root/.cache/uv \
-  uv pip install \
-  --requirement _base.txt
+
 
 
 # --------------------------Prod-depends-only stage -------------------
@@ -113,9 +111,9 @@ WORKDIR /build/services/agent
 RUN \
   --mount=type=bind,source=packages,target=/build/packages,rw \
   --mount=type=bind,source=services/agent,target=/build/services/agent,rw \
-  --mount=type=cache,mode=0755,target=/root/.cache/uv \
-  uv pip install \
-  --requirement requirements/prod.txt \
+  --mount=type=cache,target=/root/.cache/uv \
+  uv pip sync \
+  requirements/prod.txt \
   && uv pip list
 
 
@@ -132,6 +130,8 @@ ENV SC_BUILD_TARGET=production \
   SC_BOOT_MODE=production
 
 ENV PYTHONOPTIMIZE=TRUE
+# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode
+ENV UV_COMPILE_BYTECODE=1
 
 WORKDIR /home/scu