⚗️CI: Separate image building of frontend (#6918)

ITISFoundation · Dec 9, 2024 · 66a4217 · 66a4217
1 parent 186c07b
commit 66a4217
Show file tree

Hide file tree

Showing 22 changed files with 108 additions and 111 deletions.
diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml
@@ -272,20 +272,43 @@ jobs:
         python: ["3.11"]
         os: [ubuntu-22.04]
       fail-fast: false
-    name: "[build] docker images"
+    name: "[build] docker images (excluding frontend)"
     steps:
-      - name: Remove unused software
+      - uses: actions/checkout@v4
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+      - name: expose github runtime for buildx
+        uses: crazy-max/ghaction-github-runtime@v3
+      - name: show system environs
+        run: ./ci/helpers/show_system_versions.bash
+      - name: build images
         run: |
-          echo "Available storage before:"
-          sudo df -h
-          echo
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /usr/local/lib/android
-          sudo rm -rf /opt/ghc
-          sudo rm -rf /opt/hostedtoolcache/CodeQL
-          echo "Available storage after:"
-          sudo df -h
-          echo
+          export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
+          mkdir --parents /${{ runner.temp }}/build
+          make build local-dest=/${{ runner.temp }}/build exclude=static-webserver
+      - name: upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
+          path: /${{ runner.temp }}/build
+
+  build-test-images-frontend:
+    # this step comes first, so that it is executed as first job in push calls
+    # in PR calls this step is anyway skipped
+    needs: changes
+    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 30
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.11"]
+        os: [ubuntu-22.04]
+      fail-fast: false
+    name: "[build] docker images (frontend-only)"
+    steps:
       - uses: actions/checkout@v4
       - name: setup docker buildx
         id: buildx
@@ -300,11 +323,11 @@ jobs:
         run: |
           export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
           mkdir --parents /${{ runner.temp }}/build
-          make build local-dest=/${{ runner.temp }}/build
+          make build local-dest=/${{ runner.temp }}/build target=static-webserver
       - name: upload build artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-frontend
           path: /${{ runner.temp }}/build
 
   unit-test-webserver-01:
@@ -1868,7 +1891,7 @@ jobs:
         with:
           action: actions/download-artifact@v4
           with: |
-            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
             path: /${{ runner.temp }}/build
           attempt_limit: 5
           attempt_delay: 1000
@@ -1932,7 +1955,7 @@ jobs:
         with:
           action: actions/download-artifact@v4
           with: |
-            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
             path: /${{ runner.temp }}/build
           attempt_limit: 5
           attempt_delay: 1000
@@ -1996,7 +2019,7 @@ jobs:
         with:
           action: actions/download-artifact@v4
           with: |
-            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
             path: /${{ runner.temp }}/build
           attempt_limit: 5
           attempt_delay: 1000
@@ -2067,7 +2090,7 @@ jobs:
         with:
           action: actions/download-artifact@v4
           with: |
-            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
             path: /${{ runner.temp }}/build
           attempt_limit: 5
           attempt_delay: 1000
@@ -2133,7 +2156,7 @@ jobs:
         with:
           action: actions/download-artifact@v4
           with: |
-            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
             path: /${{ runner.temp }}/build
           attempt_limit: 5
           attempt_delay: 1000
@@ -2199,7 +2222,7 @@ jobs:
         with:
           action: actions/download-artifact@v4
           with: |
-            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
             path: /${{ runner.temp }}/build
           attempt_limit: 5
           attempt_delay: 1000
@@ -2257,7 +2280,7 @@ jobs:
         run: echo "::notice All good!"
 
   system-test-public-api:
-    needs: [changes, build-test-images]
+    needs: [changes, build-test-images, build-test-images-frontend]
     if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
     timeout-minutes: 25 # if this timeout gets too small, then split the tests
     name: "[sys] public api"
@@ -2286,7 +2309,7 @@ jobs:
         with:
           action: actions/download-artifact@v4
           with: |
-            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+            pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-*
             path: /${{ runner.temp }}/build
           attempt_limit: 5
           attempt_delay: 1000
@@ -2317,7 +2340,7 @@ jobs:
         run: ./ci/github/system-testing/public-api.bash clean_up
 
   system-test-swarm-deploy:
-    needs: [changes, build-test-images]
+    needs: [changes, build-test-images, build-test-images-frontend]
     if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
     timeout-minutes: 30 # if this timeout gets too small, then split the tests
     name: "[sys] deploy simcore"
@@ -2346,7 +2369,7 @@ jobs:
         with:
           action: actions/download-artifact@v4
           with: |
-            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+            pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-*
             path: /${{ runner.temp }}/build
           attempt_limit: 5
           attempt_delay: 1000
@@ -2382,7 +2405,7 @@ jobs:
         run: ./ci/github/system-testing/swarm-deploy.bash clean_up
 
   system-test-e2e:
-    needs: [changes, build-test-images]
+    needs: [changes, build-test-images, build-test-images-frontend]
     if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
     timeout-minutes: 30 # if this timeout gets too small, then split the tests
     name: "[sys] e2e"
@@ -2417,7 +2440,7 @@ jobs:
         with:
           action: actions/download-artifact@v4
           with: |
-            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+            pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-*
             path: /${{ runner.temp }}/build
           attempt_limit: 5
           attempt_delay: 1000
@@ -2462,7 +2485,7 @@ jobs:
         run: ./ci/github/system-testing/e2e.bash clean_up
 
   system-test-e2e-playwright:
-    needs: [changes, build-test-images]
+    needs: [changes, build-test-images, build-test-images-frontend]
     if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
     timeout-minutes: 30 # if this timeout gets too small, then split the tests
     name: "[sys] e2e-playwright"
@@ -2497,7 +2520,7 @@ jobs:
       - name: download docker images
         uses: actions/download-artifact@v4
         with:
-          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+          pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-*
           path: /${{ runner.temp }}/build
       - name: load docker images
         run: make load-images local-src=/${{ runner.temp }}/build
@@ -2616,7 +2639,7 @@ jobs:
         with:
           action: actions/download-artifact@v4
           with: |
-            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
+            pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-*
             path: /${{ runner.temp }}/build
           attempt_limit: 5
           attempt_delay: 1000

diff --git a/Makefile b/Makefile
@@ -153,14 +153,15 @@ DOCKER_TARGET_PLATFORMS ?= linux/amd64
 comma := ,
 
 define _docker_compose_build
+$(eval INCLUDED_SERVICES := $(filter-out $(exclude), $(SERVICES_NAMES_TO_BUILD))) \
 export BUILD_TARGET=$(if $(findstring -devel,$@),development,production) &&\
 pushd services &&\
 $(foreach service, $(SERVICES_NAMES_TO_BUILD),\
 	$(if $(push),\
 		export $(subst -,_,$(shell echo $(service) | tr a-z A-Z))_VERSION=$(shell cat services/$(service)/VERSION);\
 	,) \
 )\
-docker buildx bake \
+docker buildx bake --allow=fs.read=.. \
 	$(if $(findstring -devel,$@),,\
 	--set *.platform=$(DOCKER_TARGET_PLATFORMS) \
 	)\
@@ -172,7 +173,7 @@ docker buildx bake \
 		)\
 	)\
 	$(if $(push),--push,) \
-	$(if $(push),--file docker-bake.hcl,) --file docker-compose-build.yml $(if $(target),$(target),) \
+	$(if $(push),--file docker-bake.hcl,) --file docker-compose-build.yml $(if $(target),$(target),$(INCLUDED_SERVICES)) \
 	$(if $(findstring -nc,$@),--no-cache,\
 		$(foreach service, $(SERVICES_NAMES_TO_BUILD),\
 			--set $(service).cache-to=type=gha$(comma)mode=max$(comma)scope=$(service) \
@@ -183,22 +184,20 @@ endef
 
 rebuild: build-nc # alias
 build build-nc: .env ## Builds production images and tags them as 'local/{service-name}:production'. For single target e.g. 'make target=webserver build'. To export to a folder: `make local-dest=/tmp/build`
-	# Building service$(if $(target),,s) $(target)
+	# Building service$(if $(target),,s) $(target) $(if $(exclude),excluding,) $(exclude)
 	@$(_docker_compose_build)
 	# List production images
 	@docker images --filter="reference=local/*:production"
 
 load-images: guard-local-src ## loads images from local-src
-	# loading from images from $(local-src)...
-	@$(foreach service, $(SERVICES_NAMES_TO_BUILD),\
-		docker load --input $(local-src)/$(service).tar; \
-	)
+	# loading from any tar images from $(local-src)...
+	@find $(local-src) -name '*.tar' -print0 | xargs -0 -n1 -P $(shell nproc) --no-run-if-empty --verbose docker load --input
 	# all images loaded
 	@docker images
 
 build-devel build-devel-nc: .env ## Builds development images and tags them as 'local/{service-name}:development'. For single target e.g. 'make target=webserver build-devel'
 ifeq ($(target),)
-	# Building services
+	# Building services $(if $(exclude),excluding,) $(exclude)
 	@$(_docker_compose_build)
 else
 ifeq ($(findstring static-webserver,$(target)),static-webserver)

diff --git a/packages/service-integration/Dockerfile b/packages/service-integration/Dockerfile
@@ -12,8 +12,7 @@ LABEL maintainer=pcrespov
 # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/]
 RUN rm -f /etc/apt/apt.conf.d/docker-clean && \
   echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
-RUN  --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux \
   && apt-get update \
   && apt-get install -y --no-install-recommends \
@@ -54,8 +53,7 @@ ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
 
 FROM base AS build
 
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux && \
   apt-get update \
   && apt-get install -y --no-install-recommends \

diff --git a/requirements/tools/Dockerfile b/requirements/tools/Dockerfile
@@ -16,8 +16,7 @@ FROM python:${PYTHON_VERSION}-slim-bookworm AS base
 
 ENV VIRTUAL_ENV=/home/scu/.venv
 
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux && \
   apt-get update \
   && apt-get -y install --no-install-recommends\

diff --git a/services/agent/Dockerfile b/services/agent/Dockerfile
@@ -20,8 +20,7 @@ LABEL maintainer=GitHK
 # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/]
 RUN rm -f /etc/apt/apt.conf.d/docker-clean && \
   echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux && \
   apt-get update && \
   apt-get install -y --no-install-recommends \
@@ -71,8 +70,7 @@ FROM base AS build
 
 ENV SC_BUILD_TARGET=build
 
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux \
   && apt-get update \
   && apt-get install -y --no-install-recommends \

diff --git a/services/api-server/Dockerfile b/services/api-server/Dockerfile
@@ -19,13 +19,13 @@ LABEL maintainer=pcrespov
 # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/]
 RUN rm -f /etc/apt/apt.conf.d/docker-clean && \
   echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux && \
   apt-get update && \
   apt-get install -y --no-install-recommends \
   gosu \
   && apt-get clean -y \
+  && rm -rf /var/lib/apt/lists/* \
   # verify that the binary works
   && gosu nobody true
 
@@ -67,8 +67,7 @@ FROM base AS build
 
 ENV SC_BUILD_TARGET=build
 
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux \
   && apt-get update \
   && apt-get install -y --no-install-recommends \

diff --git a/services/autoscaling/Dockerfile b/services/autoscaling/Dockerfile
@@ -23,8 +23,7 @@ ENV DOCKER_APT_VERSION="5:26.1.4-1~debian.12~bookworm"
 # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/]
 RUN rm -f /etc/apt/apt.conf.d/docker-clean && \
   echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux; \
   apt-get update; \
   apt-get install -y --no-install-recommends \
@@ -87,8 +86,7 @@ FROM base AS build
 
 ENV SC_BUILD_TARGET=build
 
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux \
   && apt-get update \
   && apt-get install -y --no-install-recommends \

diff --git a/services/catalog/Dockerfile b/services/catalog/Dockerfile
@@ -20,13 +20,13 @@ LABEL maintainer=pcrespov
 # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/]
 RUN rm -f /etc/apt/apt.conf.d/docker-clean && \
   echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux && \
   apt-get update && \
   apt-get install -y --no-install-recommends \
   gosu \
   && apt-get clean -y \
+  && rm -rf /var/lib/apt/lists/* \
   # verify that the binary works
   && gosu nobody true
 
@@ -68,8 +68,7 @@ FROM base AS build
 
 ENV SC_BUILD_TARGET=build
 
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux \
   && apt-get update \
   && apt-get install -y --no-install-recommends \

diff --git a/services/clusters-keeper/Dockerfile b/services/clusters-keeper/Dockerfile
@@ -23,8 +23,7 @@ ENV DOCKER_APT_VERSION="5:26.1.4-1~debian.12~bookworm"
 # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/]
 RUN rm -f /etc/apt/apt.conf.d/docker-clean && \
   echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux; \
   apt-get update; \
   apt-get install -y --no-install-recommends \
@@ -87,8 +86,7 @@ FROM base AS build
 
 ENV SC_BUILD_TARGET=build
 
-RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \
-  --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=private \
   set -eux \
   && apt-get update \
   && apt-get install -y --no-install-recommends \