From f4f596365233c412201daca4b89b242802f8ff63 Mon Sep 17 00:00:00 2001 From: nate nowack Date: Thu, 12 Dec 2024 11:46:25 -0600 Subject: [PATCH] improve `Dockerfile` build time and add CI to catch future slow downs (#16348) --- .dockerignore | 9 +- .github/workflows/time-docker-build.yaml | 110 +++++++++++++++++++++++ Dockerfile | 20 +++-- 3 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/time-docker-build.yaml diff --git a/.dockerignore b/.dockerignore index 30b96ab92540..a5fb96ae22d3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -30,12 +30,13 @@ env/ venv/ # Documentation artifacts -docs/api-ref/schema.json +docs/ site/ # UI artifacts src/prefect/server/ui/* ui/node_modules +ui-v2/ # Databases *.db @@ -49,3 +50,9 @@ dask-worker-space/ # Editors .idea/ .vscode/ + +# Other +tests/ +compat-tests/ +benches/ +build/ diff --git a/.github/workflows/time-docker-build.yaml b/.github/workflows/time-docker-build.yaml new file mode 100644 index 000000000000..7156ac170aa2 --- /dev/null +++ b/.github/workflows/time-docker-build.yaml @@ -0,0 +1,110 @@ +name: Docker Build Time Benchmark + +on: + push: + paths: + - "Dockerfile" + - ".dockerignore" + pull_request: + paths: + - "Dockerfile" + - ".dockerignore" + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # For PRs, checkout the base branch to compare against + - name: Checkout base branch + if: github.base_ref + uses: actions/checkout@v4 + with: + ref: ${{ github.base_ref }} + clean: true + + - name: Clean Docker system + run: | + docker system prune -af + docker builder prune -af + + - name: Build base branch image + if: github.base_ref + id: base_build_time + run: | + start_time=$(date +%s) + DOCKER_BUILDKIT=1 docker build . --no-cache --progress=plain + end_time=$(date +%s) + base_time=$((end_time - start_time)) + echo "base_time=$base_time" >> $GITHUB_OUTPUT + + # For PRs, checkout back to the PR's HEAD + - name: Checkout PR branch + if: github.base_ref + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + clean: true + + - name: Clean Docker system again + run: | + docker system prune -af + docker builder prune -af + + - name: Build and time Docker image + id: build_time + run: | + start_time=$(date +%s) + DOCKER_BUILDKIT=1 docker build . --no-cache --progress=plain + end_time=$(date +%s) + build_time=$((end_time - start_time)) + echo "build_time=$build_time" >> $GITHUB_OUTPUT + + - name: Compare build times + run: | + CURRENT_TIME=${{ steps.build_time.outputs.build_time }} + + if [ "${{ github.base_ref }}" != "" ]; then + BASE_TIME=${{ steps.base_build_time.outputs.base_time }} + + echo "### 🏗️ Docker Build Time Comparison" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Branch | Build Time | Difference |" >> $GITHUB_STEP_SUMMARY + echo "|--------|------------|------------|" >> $GITHUB_STEP_SUMMARY + echo "| base (${{ github.base_ref }}) | ${BASE_TIME}s | - |" >> $GITHUB_STEP_SUMMARY + + DIFF=$((CURRENT_TIME - BASE_TIME)) + PERCENT=$(echo "scale=2; ($CURRENT_TIME - $BASE_TIME) * 100 / $BASE_TIME" | bc) + + if [ $DIFF -gt 0 ]; then + DIFF_TEXT="⬆️ +${DIFF}s (+${PERCENT}%)" + elif [ $DIFF -lt 0 ]; then + DIFF_TEXT="⬇️ ${DIFF}s (${PERCENT}%)" + else + DIFF_TEXT="✨ No change" + fi + + echo "| current (${{ github.head_ref }}) | ${CURRENT_TIME}s | $DIFF_TEXT |" >> $GITHUB_STEP_SUMMARY + + # Fail if build time increased by more than 5% + if (( $(echo "$PERCENT > 5" | bc -l) )); then + echo "" >> $GITHUB_STEP_SUMMARY + echo "❌ **Build time increased by more than 5%!**" >> $GITHUB_STEP_SUMMARY + echo "This change significantly increases the build time. Please review the Dockerfile changes." >> $GITHUB_STEP_SUMMARY + exit 1 + elif (( $(echo "$PERCENT < 0" | bc -l) )); then + echo "" >> $GITHUB_STEP_SUMMARY + echo "✅ **Build time decreased!**" >> $GITHUB_STEP_SUMMARY + echo "Great job optimizing the build!" >> $GITHUB_STEP_SUMMARY + fi + else + echo "### 🏗️ Docker Build Time" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Build completed in ${CURRENT_TIME} seconds" >> $GITHUB_STEP_SUMMARY + fi diff --git a/Dockerfile b/Dockerfile index 9534db5507a4..542296331793 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,6 +81,9 @@ FROM ${BASE_IMAGE} AS final ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 +ENV UV_LINK_MODE=copy +ENV UV_SYSTEM_PYTHON=1 + LABEL maintainer="help@prefect.io" \ io.prefect.python-version=${PYTHON_VERSION} \ org.label-schema.schema-version="1.0" \ @@ -95,32 +98,33 @@ RUN apt-get update && \ tini=0.19.* \ build-essential \ git=1:2.* \ - curl \ - ca-certificates \ && apt-get clean && rm -rf /var/lib/apt/lists/* -# Install UV from official image -COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +# Install UV from official image - pin to specific version for build caching +COPY --from=ghcr.io/astral-sh/uv:0.5.8 /uv /uvx /bin/ # Install dependencies using a temporary mount for requirements files RUN --mount=type=bind,source=requirements-client.txt,target=/tmp/requirements-client.txt \ --mount=type=bind,source=requirements.txt,target=/tmp/requirements.txt \ - uv pip install --system -r /tmp/requirements.txt + --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -r /tmp/requirements.txt -r /tmp/requirements-client.txt # Install prefect from the sdist COPY --from=python-builder /opt/prefect/dist ./dist # Extras to include during installation ARG PREFECT_EXTRAS -RUN uv pip install --system "./dist/prefect.tar.gz${PREFECT_EXTRAS:-""}" && \ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install "./dist/prefect.tar.gz${PREFECT_EXTRAS:-""}" && \ rm -rf dist/ # Remove setuptools -RUN uv pip uninstall --system setuptools +RUN uv pip uninstall setuptools # Install any extra packages ARG EXTRA_PIP_PACKAGES -RUN [ -z "${EXTRA_PIP_PACKAGES:-""}" ] || uv pip install --system "${EXTRA_PIP_PACKAGES}" +RUN --mount=type=cache,target=/root/.cache/uv \ + [ -z "${EXTRA_PIP_PACKAGES:-""}" ] || uv pip install "${EXTRA_PIP_PACKAGES}" # Smoke test RUN prefect version