From 739eb39cb1a62d8a8df424708aef31e122c352ed Mon Sep 17 00:00:00 2001 From: Javier Maestro Date: Wed, 13 Nov 2024 13:14:08 +0000 Subject: [PATCH 1/3] feat: Docker test image Simple Docker image to manually test the build-and-push Github Action --- buildkite/docker/testimage/Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 buildkite/docker/testimage/Dockerfile diff --git a/buildkite/docker/testimage/Dockerfile b/buildkite/docker/testimage/Dockerfile new file mode 100644 index 0000000000..221b94e6ba --- /dev/null +++ b/buildkite/docker/testimage/Dockerfile @@ -0,0 +1,11 @@ +FROM busybox:stable AS testimage-base +RUN mkdir /tmp/base + +FROM testimage-base AS testimage-target1 +RUN mkdir /tmp/base/target1 + +FROM testimage-base AS testimage-target2-nojdk +RUN mkdir /tmp/base/target2 + +FROM testimage-target2-nojdk AS testimage-target3 +RUN mkdir /tmp/base/target3 From 0846aca616d83cf68ac2504eee0ed12d23f52e45 Mon Sep 17 00:00:00 2001 From: Javier Maestro Date: Tue, 12 Nov 2024 16:47:29 +0000 Subject: [PATCH 2/3] feat: build CI Docker images with GH Actions Workflow to build and deploy Bazel CI Docker images This workflow builds the images, pushes them to the GHCR registry and links them with this repo. For each `Dockerfile` it builds different types of images, each corresponding with a named build stage (`... AS `) in the `Dockerfile`. The workflow is triggered when a push to the `main`/`master` or `testing` branch contains changes to one or more of the CI `Dockerfile`s (`buildkite/docker/*/Dockerfile`). It can also be triggered manually via the Actions web UI, the GH REST API or the GH CLI tool, e.g.: ```sh gh workflow run build-ci-docker-images ``` When triggered by a `push` event the workflow will: 1. Determine which `Dockerfile`s were changed. 2. For those `Dockerfile`s, determine its build context (the `Dockerfile` directory) and the build targets (the named build stages in it). 3. Filter the build targets: * first with `RE_TARGET_INCLUDE`, which defaults to empty so it will match all the named build stages. * then with `RE_TARGET_EXCLUDE`: set by default to remove some of the build stages, e.g. the deprecated images like `centos7` and some targets that we don't want to build as images like the `nojdk` ones. 4. Then, it will also exclude all `testimage` targets because that image is only used for manually testing the workflow. 5. Finally, it will spawn a `docker/build-push-action` job for each of the build targets. For every image built, it will push to the registry three image tags: * a `sha` tag with the short hash of the commit that triggered the push * a `date` tag with the current date in ISO format (`YYYYMMDD`) * a `latest` tag When triggered manually (`workflow_dispatch` event) the workflow will default to "running in test mode": it will follow the same steps as a `push` run but with different default values (see `workflow_dispatch.inputs`): * `RE_TARGET_INCLUDE` set to `testimage` (a very simple image to exercise the build that doesn't take much compute) and * `RE_TARGET_EXCLUDE` set to the same pattern as in the `push` event. This effectively limits the build targets to only those in `testimage` not excluded by `RE_TARGET_EXCLUDE` (e.g. `nojdk`). The "test run" also limits the `PLATFORMS` to `linux/amd64`, to further reduce the cost and time of a test run. Finally, it will build those `testimage` targets but **it won't tag `latest` or push any of the image tags to the registry**. This "test mode" behavior can be changed by setting the `workflow_dispatch.inputs` variables: `RE_TARGET_EXCLUDE`, `RE_TARGET_INCLUDE`, `PLATFORMS`, `TAG_DATE`, `TAG_LATEST` and `PUSH`, e.g.: ```sh gh workflow run build-ci-docker-images \ -f RE_TARGET_INCLUDE=ubuntu2404 -f TAG_DATE=20241101 ``` --- .github/workflows/build-ci-docker-images.yaml | 294 ++++++++++++++++++ buildkite/README.md | 2 + 2 files changed, 296 insertions(+) create mode 100644 .github/workflows/build-ci-docker-images.yaml diff --git a/.github/workflows/build-ci-docker-images.yaml b/.github/workflows/build-ci-docker-images.yaml new file mode 100644 index 0000000000..78772c3f54 --- /dev/null +++ b/.github/workflows/build-ci-docker-images.yaml @@ -0,0 +1,294 @@ +# Workflow to build and deploy Bazel CI Docker images +# +# This workflow builds the images, pushes them to the GHCR registry and +# links them with this repo. +# +# For each `Dockerfile` it builds different types of images, each +# corresponding with a named build stage (`... AS `) in the +# `Dockerfile`. +# +# The workflow is triggered when a push to the `main`/`master` or +# `testing` branch contains changes to one or more of the CI +# `Dockerfile`s (`buildkite/docker/*/Dockerfile`). It can also be +# triggered manually via the Actions web UI, the GH REST API or the GH +# CLI tool, e.g.: +# ```sh +# gh workflow run build-ci-docker-images +# ``` +# +# When triggered by a `push` event the workflow will: +# +# 1. Determine which `Dockerfile`s were changed. +# +# 2. For those `Dockerfile`s, determine its build context (the +# `Dockerfile` directory) and the build targets (the named build +# stages in it). +# +# 3. Filter the build targets: +# * first with `RE_TARGET_INCLUDE`, which defaults to empty so it will +# match all the named build stages. +# * then with `RE_TARGET_EXCLUDE`: set by default to remove some of +# the build stages, e.g. the deprecated images like `centos7` and +# some targets that we don't want to build as images like the +# `nojdk` ones. +# +# 4. Then, it will also exclude all `testimage` targets because that +# image is only used for manually testing the workflow. +# +# 5. Finally, it will spawn a `docker/build-push-action` job for each of +# the build targets. For every image built, it will push to the +# registry three image tags: +# * a `sha` tag with the short hash of the commit that triggered the +# push +# * a `date` tag with the current date in ISO format (`YYYYMMDD`) +# * a `latest` tag +# +# When triggered manually (`workflow_dispatch` event) the workflow will +# default to "running in test mode": it will follow the same steps as a +# `push` run but with different default values (see +# `workflow_dispatch.inputs`): +# * `RE_TARGET_INCLUDE` set to `testimage` (a very simple image to +# exercise the build that doesn't take much compute) and +# * `RE_TARGET_EXCLUDE` set to the same pattern as in the `push` event. +# +# This effectively limits the build targets to only those in `testimage` +# not excluded by `RE_TARGET_EXCLUDE` (e.g. `nojdk`). +# +# The "test run" also limits the `PLATFORMS` to `linux/amd64`, to further +# reduce the cost and time of a test run. +# +# Finally, it will build those `testimage` targets but **it won't tag +# `latest` or push any of the image tags to the registry**. +# +# This "test mode" behavior can be changed by setting the +# `workflow_dispatch.inputs` variables: `RE_TARGET_EXCLUDE`, +# `RE_TARGET_INCLUDE`, `PLATFORMS`, `TAG_DATE`, `TAG_LATEST` and `PUSH`, +# e.g.: +# ```sh +# gh workflow run build-ci-docker-images \ +# -f RE_TARGET_INCLUDE=ubuntu2404 -f TAG_DATE=20241101 +# ``` + +name: build-ci-docker-images + +on: + push: + branches: + - main + - master + - testing + paths: + # NOTE: keep in-sync with env.SPARSE_PATH + - buildkite/docker/*/Dockerfile + + workflow_dispatch: + inputs: + RE_TARGET_EXCLUDE: + description: |- + Filter out Docker targets matching this extended regex pattern + # NOTE: keep in-sync with env.RE_TARGET_EXCLUDE + default: ^centos7|^ubuntu16|nojdk + RE_TARGET_INCLUDE: + description: |- + Select Docker targets matching this extended regex pattern + # NOTE: keep in-sync with env.TEST_IMAGE + default: testimage + PLATFORMS: + default: linux/amd64 + TAG_DATE: + description: Tag image date in ISO format (YYYYMMDD) + TAG_LATEST: + description: Tag image version as 'latest' + default: false + PUSH: + description: Push images to registry + default: false + +env: + REGISTRY: ghcr.io + SPARSE_PATH: buildkite/docker + TEST_IMAGE: testimage + + GH_EVENT_NAME: ${{ github.event_name }} + GH_REF_NAME: ${{ github.ref_name }} + GH_REPO: ${{ github.repository }} + GH_SHA: ${{ github.sha }} + GH_SHA_BEFORE: ${{ github.event.before }} + + RE_TARGET_EXCLUDE: ${{ inputs.RE_TARGET_EXCLUDE || '^centos7|^ubuntu16|nojdk' }} + RE_TARGET_INCLUDE: ${{ inputs.RE_TARGET_INCLUDE }} + PLATFORMS: ${{ inputs.PLATFORMS || 'linux/amd64,linux/arm64' }} + TAG_DATE: ${{ inputs.TAG_DATE }} + TAG_LATEST: ${{ inputs.TAG_LATEST }} + PUSH: ${{ github.event_name == 'push' || inputs.PUSH }} + +jobs: + setup-targets: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} + + outputs: + targets: ${{ steps.define_targets.outputs.targets }} + targets_length: ${{ steps.define_targets.outputs.targets_length }} + + steps: + - name: Sparse checkout SPARSE_PATH + uses: actions/checkout@v4 + with: + sparse-checkout-cone-mode: false + sparse-checkout: |- + ${{ env.SPARSE_PATH }} + + - name: Get DOCKERFILES + run: |- + # NOTE: + # GH_SHA_BEFORE is empty on pushing the first commit of a new branch + # or when running manually via workflow_dispatch + if [[ -z $GH_SHA_BEFORE ]]; then + DOCKERFILES="$(ls $SPARSE_PATH/*/Dockerfile)" + else + DOCKERFILES="$( + git diff --name-only $GH_SHA_BEFORE $GH_SHA | + grep Dockerfile || { + # NOTE: + # this grep could fail if e.g. we are force-pushing a stack + # of commits where one or more commits do change Dockerfiles + # but there's no change to any Dockerfile between the last + # commit and this forced push. + echo "WARNING: EMPTY grep" >&2 + true + } + )" + fi + + DOCKERFILES_JSON="$(echo -n "$DOCKERFILES" | jq -R '.' | jq -sc '.' )" + + echo "DOCKERFILES_JSON=$DOCKERFILES_JSON" + echo "DOCKERFILES_JSON=$DOCKERFILES_JSON" >> $GITHUB_ENV + + - name: Define targets + id: define_targets + run: |- + if [[ "$GH_EVENT_NAME" == "push" ]]; then + RE_TARGET_EXCLUDE="$RE_TARGET_EXCLUDE|$TEST_IMAGE" + fi + + DOCKERFILES=($(echo "$DOCKERFILES_JSON" | jq -r 'join(" ")')) + + if [[ ${#DOCKERFILES[@]} -gt 0 ]]; then + TARGETS_LS="$( + grep -i '^FROM .* AS ' ${DOCKERFILES[@]} | + awk '{print $NF}' | + { grep -E "$RE_TARGET_INCLUDE" || true; } | + { grep -vE "$RE_TARGET_EXCLUDE" || true; } | + jq -R '.' + )" + else + TARGETS_LS="" + fi + + TARGETS="$(echo "$TARGETS_LS" | jq -sc '.')" + TARGETS_LENGTH="$(echo "$TARGETS_LS" | jq -s 'length')" + + echo "targets=$TARGETS" + echo "targets_length=$TARGETS_LENGTH" + + echo "targets=$TARGETS" >> "$GITHUB_OUTPUT" + echo "targets_length=$TARGETS_LENGTH" >> "$GITHUB_OUTPUT" + + build-and-publish-docker-images: + needs: setup-targets + if: ${{ needs.setup-targets.outputs.targets_length > 0 }} + + runs-on: ubuntu-latest + + strategy: + matrix: + target: ${{ fromJson(needs.setup-targets.outputs.targets) }} + + defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} + + steps: + - name: Sparse checkout SPARSE_PATH + uses: actions/checkout@v4 + with: + sparse-checkout-cone-mode: false + sparse-checkout: |- + ${{ env.SPARSE_PATH }} + + - name: Set up dynamic env + env: + MATRIX_TARGET: ${{ matrix.target }} + run: |- + declare -A TAGS + + TAGS[sha]="${GH_SHA::7}" + + TAGS[date]="$TAG_DATE" + # set default date value if TAG_DATE is not set, is empty + # or is an empty string + if [[ -z "${TAGS[date]+isset}" || -z "${TAGS[date]// }" ]]; then + TAGS[date]="$(date +%Y%m%d)" + fi + + if [[ "$GH_EVENT_NAME" == "push" || "$TAG_LATEST" == "true" ]]; then + TAGS[latest]="latest" + fi + + if [[ "$REGISTRY" == "gcr.io" ]]; then + IMAGE_PREFIX="bazel-public" + elif [[ "$REGISTRY" == "ghcr.io" ]]; then + IMAGE_PREFIX="$GH_REPO" + else + echo "Invalid registry: $REGISTRY" + exit 1 + fi + + if [[ "$GH_REF_NAME" == "testing" ]]; then + IMAGE_PREFIX="$IMAGE_PREFIX/testing" + fi + + IMAGE_NAME="$REGISTRY/$IMAGE_PREFIX/$MATRIX_TARGET" + + DISTRO="${MATRIX_TARGET%%-*}" + CONTEXT="$SPARSE_PATH/$DISTRO" + + { + for tag in ${!TAGS[@]}; do + echo "IMAGE_TAG_${tag^^}=$IMAGE_NAME:${TAGS[$tag]}" + done + + echo "CONTEXT=$CONTEXT" + } >> $GITHUB_ENV + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ${{ env.CONTEXT }} + target: ${{ matrix.target }} + platforms: ${{ env.PLATFORMS }} + tags: |- + ${{ env.IMAGE_TAG_SHA }} + ${{ env.IMAGE_TAG_DATE }} + ${{ env.IMAGE_TAG_LATEST }} + labels: |- + org.opencontainers.image.source=${{ github.repositoryUrl }} + push: ${{ env.PUSH }} diff --git a/buildkite/README.md b/buildkite/README.md index d6781b6f48..f6fc76d07d 100644 --- a/buildkite/README.md +++ b/buildkite/README.md @@ -1,3 +1,5 @@ +[![build-docker-images](https://github.com/bazelbuild/continuous-integration/actions/workflows/build-docker-images.yaml/badge.svg?branch=master&event=push)](https://github.com/bazelbuild/continuous-integration/actions/workflows/build-docker-images.yaml) + # Bazel Continuous Integration tl;dr: From 800cac68f709e84373188386fd0d7ef19264ba56 Mon Sep 17 00:00:00 2001 From: Javier Maestro Date: Thu, 14 Nov 2024 11:18:05 +0000 Subject: [PATCH 3/3] feat: lint build-ci-docker-images.yaml --- .github/workflows/build-ci-docker-images.yaml | 26 ++++++++++++------- .pre-commit-config.yaml | 26 +++++++++++++++++++ 2 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/build-ci-docker-images.yaml b/.github/workflows/build-ci-docker-images.yaml index 78772c3f54..17e8273e46 100644 --- a/.github/workflows/build-ci-docker-images.yaml +++ b/.github/workflows/build-ci-docker-images.yaml @@ -94,6 +94,8 @@ on: # NOTE: keep in-sync with env.TEST_IMAGE default: testimage PLATFORMS: + description: |- + Select platforms to build default: linux/amd64 TAG_DATE: description: Tag image date in ISO format (YYYYMMDD) @@ -147,11 +149,11 @@ jobs: # NOTE: # GH_SHA_BEFORE is empty on pushing the first commit of a new branch # or when running manually via workflow_dispatch - if [[ -z $GH_SHA_BEFORE ]]; then - DOCKERFILES="$(ls $SPARSE_PATH/*/Dockerfile)" + if [[ -z "$GH_SHA_BEFORE" ]]; then + DOCKERFILES="$(ls "$SPARSE_PATH"/*/Dockerfile)" else DOCKERFILES="$( - git diff --name-only $GH_SHA_BEFORE $GH_SHA | + git diff --name-only "$GH_SHA_BEFORE" "$GH_SHA" | grep Dockerfile || { # NOTE: # this grep could fail if e.g. we are force-pushing a stack @@ -167,22 +169,26 @@ jobs: DOCKERFILES_JSON="$(echo -n "$DOCKERFILES" | jq -R '.' | jq -sc '.' )" echo "DOCKERFILES_JSON=$DOCKERFILES_JSON" - echo "DOCKERFILES_JSON=$DOCKERFILES_JSON" >> $GITHUB_ENV + echo "DOCKERFILES_JSON=$DOCKERFILES_JSON" >> "$GITHUB_ENV" - name: Define targets id: define_targets + env: + RE_TARGET_INCLUDE: . run: |- if [[ "$GH_EVENT_NAME" == "push" ]]; then RE_TARGET_EXCLUDE="$RE_TARGET_EXCLUDE|$TEST_IMAGE" fi - DOCKERFILES=($(echo "$DOCKERFILES_JSON" | jq -r 'join(" ")')) + read -ra DOCKERFILES <<< "$( + echo "$DOCKERFILES_JSON" | jq -r 'join(" ")' + )" - if [[ ${#DOCKERFILES[@]} -gt 0 ]]; then + if [[ "${#DOCKERFILES[@]}" -gt 0 ]]; then TARGETS_LS="$( - grep -i '^FROM .* AS ' ${DOCKERFILES[@]} | + grep -i '^FROM .* AS ' "${DOCKERFILES[@]}" | awk '{print $NF}' | - { grep -E "$RE_TARGET_INCLUDE" || true; } | + { grep -E "${RE_TARGET_INCLUDE:-}" || true; } | { grep -vE "$RE_TARGET_EXCLUDE" || true; } | jq -R '.' )" @@ -259,12 +265,12 @@ jobs: CONTEXT="$SPARSE_PATH/$DISTRO" { - for tag in ${!TAGS[@]}; do + for tag in "${!TAGS[@]}"; do echo "IMAGE_TAG_${tag^^}=$IMAGE_NAME:${TAGS[$tag]}" done echo "CONTEXT=$CONTEXT" - } >> $GITHUB_ENV + } >> "$GITHUB_ENV" - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..5dae60092e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +files: >- + (?x)^( + .github/workflows/build-ci-docker-images.yaml + )$ + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-merge-conflict + - id: mixed-line-ending + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + +- repo: https://github.com/mpalmer/action-validator + rev: v0.6.0 + hooks: + - id: action-validator + +- repo: https://github.com/rhysd/actionlint + rev: v1.7.4 + hooks: + - id: actionlint