From b06ae6a816e297cc23965de209fd026b1965e25f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Djema=C3=AF?= <53857555+SamyDjemai@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:48:32 +0100 Subject: [PATCH 1/8] ci: rework multi-arch build --- .github/workflows/docker-build.yml | 103 +++++++++++++++-------------- 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 199f09d..db0a893 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -1,3 +1,4 @@ +# Based on https://docs.docker.com/build/ci/github-actions/multi-platform/ name: Docker on: @@ -31,8 +32,16 @@ jobs: id-token: write steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -47,36 +56,51 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and cache Docker image - id: push + - name: Build and push by digest + id: build uses: docker/build-push-action@v6 with: - context: . platforms: ${{ matrix.platform }} - cache-from: type=gha - cache-to: type=gha,mode=max - push: false + cache-from: type=gha,scope=${{ matrix.platform }} + cache-to: type=gha,scope=${{ matrix.platform }},mode=max + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - push: - needs: build - runs-on: ubuntu-latest + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" - permissions: - contents: read - packages: write - attestations: write - id-token: write + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + merge: + runs-on: ubuntu-latest + needs: + - build steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} + - name: Log in to the Container registry uses: docker/login-action@v3 with: @@ -84,33 +108,12 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha,format=long - - - name: Build and push Docker image - id: push - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.IMAGE_NAME }}@sha256:%s ' *) - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - with: - subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} From 36d4d08d69d98ef98fc933b0386e91049724e240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Djema=C3=AF?= <53857555+SamyDjemai@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:16:52 +0100 Subject: [PATCH 2/8] ci: append REGISTRY to docker image name --- .github/workflows/docker-build.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index db0a893..0d4fe5d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -15,6 +15,12 @@ env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} +permissions: + contents: read + packages: write + attestations: write + id-token: write + jobs: build: runs-on: ubuntu-latest @@ -25,12 +31,6 @@ jobs: - linux/amd64 - linux/arm64 - permissions: - contents: read - packages: write - attestations: write - id-token: write - steps: - name: Prepare run: | @@ -64,7 +64,7 @@ jobs: cache-from: type=gha,scope=${{ matrix.platform }} cache-to: type=gha,scope=${{ matrix.platform }},mode=max labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - name: Export digest run: | @@ -112,8 +112,8 @@ jobs: working-directory: /tmp/digests run: | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.IMAGE_NAME }}@sha256:%s ' *) + $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *) - name: Inspect image run: | - docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} + docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} From c67e975d28945be79429368f5d59f3903783546e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Djema=C3=AF?= <53857555+SamyDjemai@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:18:29 +0100 Subject: [PATCH 3/8] ci: define docker meta tags manually --- .github/workflows/docker-build.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 0d4fe5d..8bf32b1 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -42,6 +42,13 @@ jobs: uses: docker/metadata-action@v5 with: images: ${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,format=long - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -100,6 +107,13 @@ jobs: uses: docker/metadata-action@v5 with: images: ${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,format=long - name: Log in to the Container registry uses: docker/login-action@v3 From 645a6ba02bd03b07c09d638b99a2bc4a12dbf705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Djema=C3=AF?= <53857555+SamyDjemai@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:49:10 +0100 Subject: [PATCH 4/8] ci: set IMAGE_NAME in lowercase --- .github/workflows/docker-build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 8bf32b1..f7b3939 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -13,7 +13,6 @@ on: env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} permissions: contents: read @@ -35,6 +34,8 @@ jobs: - name: Prepare run: | platform=${{ matrix.platform }} + # Store image name in lowercase and platform pair for Docker push + echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> $GITHUB_ENV echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - name: Docker meta @@ -92,6 +93,10 @@ jobs: needs: - build steps: + - name: Prepare + run: | + echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> $GITHUB_ENV + - name: Download digests uses: actions/download-artifact@v4 with: From 1d341cd4a466829c9266b4eebc3072d87ad6288d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Djema=C3=AF?= <53857555+SamyDjemai@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:59:58 +0100 Subject: [PATCH 5/8] ci: append REGISTRY to docker image name --- .github/workflows/docker-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index f7b3939..fb9cc40 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -42,7 +42,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.IMAGE_NAME }} + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | type=ref,event=branch type=ref,event=tag @@ -111,7 +111,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.IMAGE_NAME }} + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | type=ref,event=branch type=ref,event=tag From d39e5d481c080f1396a6ea8604546032bc10449e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Djema=C3=AF?= <53857555+SamyDjemai@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:08:59 +0100 Subject: [PATCH 6/8] ci: add latest tag to main branch docker builds --- .github/workflows/docker-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index fb9cc40..7eb2f36 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -44,6 +44,7 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | + type=raw,value=latest,enable={{is_default_branch}} type=ref,event=branch type=ref,event=tag type=ref,event=pr @@ -113,6 +114,7 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | + type=raw,value=latest,enable={{is_default_branch}} type=ref,event=branch type=ref,event=tag type=ref,event=pr From 90c34b69c7be2bc624a0838ce49974f9e2449bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Djema=C3=AF?= <53857555+SamyDjemai@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:42:25 +0100 Subject: [PATCH 7/8] ci: push docker images only on main or on forks --- .github/workflows/docker-build.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 7eb2f36..4397b1b 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -13,6 +13,7 @@ on: env: REGISTRY: ghcr.io + ORGANIZATION: singlestore-labs permissions: contents: read @@ -73,15 +74,17 @@ jobs: cache-from: type=gha,scope=${{ matrix.platform }} cache-to: type=gha,scope=${{ matrix.platform }},mode=max labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ github.ref == 'refs/heads/main' || github.repository_owner != env.ORGANIZATION }} - name: Export digest + if: github.ref == 'refs/heads/main' || github.repository_owner != env.ORGANIZATION run: | mkdir -p /tmp/digests digest="${{ steps.build.outputs.digest }}" touch "/tmp/digests/${digest#sha256:}" - name: Upload digest + if: github.ref == 'refs/heads/main' || github.repository_owner != env.ORGANIZATION uses: actions/upload-artifact@v4 with: name: digests-${{ env.PLATFORM_PAIR }} @@ -91,6 +94,7 @@ jobs: merge: runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || github.repository_owner != 'singlestore-labs' needs: - build steps: @@ -117,7 +121,6 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} type=ref,event=branch type=ref,event=tag - type=ref,event=pr type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=sha,format=long From 79902ccabbdd4820e5f960cc1f83c1e6df54f1aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Djema=C3=AF?= <53857555+SamyDjemai@users.noreply.github.com> Date: Tue, 12 Nov 2024 22:45:07 +0100 Subject: [PATCH 8/8] ci: add hidden git content to .dockerignore --- .dockerignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.dockerignore b/.dockerignore index 138f099..695179d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,9 @@ *.log tmp/ +.git +.gitignore + .direnv/ dist-newstyle/