From 8d5d1ff96461c5886d446233a33069db579a7180 Mon Sep 17 00:00:00 2001 From: Andreas Heinrich Date: Tue, 9 Jul 2024 09:25:00 +0200 Subject: [PATCH 1/4] Refactor docker images in EVerest * Add docker image `ghcr.io/everest/everest-core/build-kit-everest-core` * Refactor workflow `build_and_test.yaml` * Deploy docker image `ghcr.io/everest/everest-core/build-kit-everest-core` * Use deployed image as build kit image Signed-off-by: Andreas Heinrich --- .ci/build-kit/docker/Dockerfile | 10 ++++ .ci/build-kit/prepare_integration_tests.sh | 9 ---- .ci/build-kit/{ => scripts}/compile.sh | 0 .ci/build-kit/{ => scripts}/install.sh | 0 .../scripts/prepare_integration_tests.sh | 12 +++++ .ci/build-kit/{ => scripts}/run_unit_tests.sh | 0 .github/workflows/build_and_test.yaml | 50 +++++++++++++++++-- 7 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 .ci/build-kit/docker/Dockerfile delete mode 100755 .ci/build-kit/prepare_integration_tests.sh rename .ci/build-kit/{ => scripts}/compile.sh (100%) rename .ci/build-kit/{ => scripts}/install.sh (100%) create mode 100755 .ci/build-kit/scripts/prepare_integration_tests.sh rename .ci/build-kit/{ => scripts}/run_unit_tests.sh (100%) diff --git a/.ci/build-kit/docker/Dockerfile b/.ci/build-kit/docker/Dockerfile new file mode 100644 index 000000000..f23853844 --- /dev/null +++ b/.ci/build-kit/docker/Dockerfile @@ -0,0 +1,10 @@ +# syntax=docker/dockerfile:1 +ARG BASE_IMAGE_TAG=latest +FROM ghcr.io/everest/everest-ci/build-kit-base:${BASE_IMAGE_TAG} + +RUN apt update \ + && apt install --no-install-recommends -y \ + python3-build=0.9.0-1 + +RUN apt clean \ + && rm -rf /var/lib/apt/lists/* diff --git a/.ci/build-kit/prepare_integration_tests.sh b/.ci/build-kit/prepare_integration_tests.sh deleted file mode 100755 index c083133c2..000000000 --- a/.ci/build-kit/prepare_integration_tests.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -set -e - -rsync -a "$EXT_MOUNT/source/tests" ./ - -pip install $EXT_MOUNT/wheels/everestpy-*.whl -pip install $EXT_MOUNT/wheels/everest_testing-*.whl -pip install pytest-html diff --git a/.ci/build-kit/compile.sh b/.ci/build-kit/scripts/compile.sh similarity index 100% rename from .ci/build-kit/compile.sh rename to .ci/build-kit/scripts/compile.sh diff --git a/.ci/build-kit/install.sh b/.ci/build-kit/scripts/install.sh similarity index 100% rename from .ci/build-kit/install.sh rename to .ci/build-kit/scripts/install.sh diff --git a/.ci/build-kit/scripts/prepare_integration_tests.sh b/.ci/build-kit/scripts/prepare_integration_tests.sh new file mode 100755 index 000000000..dc3ba3115 --- /dev/null +++ b/.ci/build-kit/scripts/prepare_integration_tests.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e + +rsync -a "$EXT_MOUNT/source/tests" ./ + +pip install --break-system-packages \ + $EXT_MOUNT/wheels/everestpy-*.whl +pip install --break-system-packages \ + $EXT_MOUNT/wheels/everest_testing-*.whl +pip install --break-system-packages \ + pytest-html diff --git a/.ci/build-kit/run_unit_tests.sh b/.ci/build-kit/scripts/run_unit_tests.sh similarity index 100% rename from .ci/build-kit/run_unit_tests.sh rename to .ci/build-kit/scripts/run_unit_tests.sh diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 577f7dd14..4f4b61202 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -15,7 +15,8 @@ on: - cron: '37 13,1 * * *' env: - BUILD_KIT_IMAGE: ghcr.io/everest/build-kit-alpine:v1.2.0 + DOCKER_REGISTRY: ghcr.io + EVEREST_CI_VERSION: refactor-docker-images # LTODO: Update this jobs: lint: @@ -27,14 +28,49 @@ jobs: with: path: source - name: Run clang-format - uses: everest/everest-ci/github-actions/run-clang-format@v1.1.0 + uses: everest/everest-ci/github-actions/run-clang-format@refactor-docker-images # LTODO: Update this with: source-dir: source extensions: hpp,cpp exclude: cache + + # Since env variables can't be passed to reusable workflows, we need to pass them as outputs + setup-env: + name: Setup Environment + runs-on: ${{ inputs.runner || 'ubuntu-22.04' }} + outputs: + docker_registry: ${{ env.DOCKER_REGISTRY }} + everest_ci_version: ${{ env.EVEREST_CI_VERSION }} + steps: + - id: check + run: | + echo "Setting up environment" + build-and-push-build-kit: + name: Build and Push Build Kit + uses: everest/everest-ci/.github/workflows/deploy-single-docker-image.yml@refactor-docker-images # LTODO: Update this + needs: setup-env + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + image_name: ${{ github.event.repository.name }}/build-kit-everest-core + directory: .ci/build-kit/docker + docker_registry: ${{ needs.setup-env.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: linux/amd64 + depends_on_paths: | + .ci/build-kit + .github/workflows/build_and_test.yaml + build_args: | + BASE_IMAGE_TAG=${{ needs.setup-env.outputs.everest_ci_version }} + build: name: Build and Unit Tests + needs: build-and-push-build-kit runs-on: ${{ inputs.runner || 'ubuntu-22.04' }} + env: + BUILD_KIT_IMAGE: ${{ needs.build-and-push-build-kit.outputs.one_image_tag_long }} steps: - name: Format branch name for cache key run: | @@ -55,7 +91,7 @@ jobs: - name: Setup run scripts run: | mkdir scripts - rsync -a source/.ci/build-kit/ scripts + rsync -a source/.ci/build-kit/scripts/ scripts - name: Pull build-kit image run: | docker pull --quiet ${{ env.BUILD_KIT_IMAGE }} @@ -102,7 +138,11 @@ jobs: path: ${{ github.workspace }}/ctest-report integration-tests: name: Integration Tests - needs: build + needs: + - build + - build-and-push-build-kit + env: + BUILD_KIT_IMAGE: ${{ needs.build-and-push-build-kit.outputs.one_image_tag_long }} runs-on: ${{ inputs.runner || 'ubuntu-22.04' }} steps: - name: Download dist dir @@ -124,7 +164,7 @@ jobs: - name: Setup run scripts run: | mkdir scripts - rsync -a source/.ci/build-kit/ scripts + rsync -a source/.ci/build-kit/scripts/ scripts - name: Pull build-kit image run: | docker pull --quiet ${{ env.BUILD_KIT_IMAGE }} From 08f658b3f010b9867782549decd828e650f2cd8b Mon Sep 17 00:00:00 2001 From: Andreas Heinrich Date: Mon, 29 Jul 2024 12:39:31 +0200 Subject: [PATCH 2/4] * Move python3-build requirement to everest-ci/build-env-base Signed-off-by: Andreas Heinrich --- .ci/build-kit/docker/Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.ci/build-kit/docker/Dockerfile b/.ci/build-kit/docker/Dockerfile index f23853844..ca655b11d 100644 --- a/.ci/build-kit/docker/Dockerfile +++ b/.ci/build-kit/docker/Dockerfile @@ -1,10 +1,3 @@ # syntax=docker/dockerfile:1 ARG BASE_IMAGE_TAG=latest FROM ghcr.io/everest/everest-ci/build-kit-base:${BASE_IMAGE_TAG} - -RUN apt update \ - && apt install --no-install-recommends -y \ - python3-build=0.9.0-1 - -RUN apt clean \ - && rm -rf /var/lib/apt/lists/* From 661f36cec57a765dcb6fff7bfbdb94be4922bc7e Mon Sep 17 00:00:00 2001 From: Andreas Heinrich Date: Mon, 29 Jul 2024 13:02:33 +0200 Subject: [PATCH 3/4] Apply suggestions from code review - Update everest-ci version to v1.3.0 Signed-off-by: Andreas Heinrich --- .github/workflows/build_and_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 4f4b61202..ade471c98 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -16,7 +16,7 @@ on: env: DOCKER_REGISTRY: ghcr.io - EVEREST_CI_VERSION: refactor-docker-images # LTODO: Update this + EVEREST_CI_VERSION: v1.3.0 jobs: lint: @@ -28,7 +28,7 @@ jobs: with: path: source - name: Run clang-format - uses: everest/everest-ci/github-actions/run-clang-format@refactor-docker-images # LTODO: Update this + uses: everest/everest-ci/github-actions/run-clang-format@v1.3.0 with: source-dir: source extensions: hpp,cpp @@ -47,7 +47,7 @@ jobs: echo "Setting up environment" build-and-push-build-kit: name: Build and Push Build Kit - uses: everest/everest-ci/.github/workflows/deploy-single-docker-image.yml@refactor-docker-images # LTODO: Update this + uses: everest/everest-ci/.github/workflows/deploy-single-docker-image.yml@v1.3.0 needs: setup-env secrets: SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} From de996aed032163fdfd9c802336da72e6b523d6a7 Mon Sep 17 00:00:00 2001 From: Andreas Heinrich Date: Mon, 29 Jul 2024 13:35:56 +0200 Subject: [PATCH 4/4] Apply suggestions from code review - Update everest-ci to v1.3.1 Signed-off-by: Andreas Heinrich --- .github/workflows/build_and_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index ade471c98..d2b8264d4 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -16,7 +16,7 @@ on: env: DOCKER_REGISTRY: ghcr.io - EVEREST_CI_VERSION: v1.3.0 + EVEREST_CI_VERSION: v1.3.1 jobs: lint: @@ -28,7 +28,7 @@ jobs: with: path: source - name: Run clang-format - uses: everest/everest-ci/github-actions/run-clang-format@v1.3.0 + uses: everest/everest-ci/github-actions/run-clang-format@v1.3.1 with: source-dir: source extensions: hpp,cpp @@ -47,7 +47,7 @@ jobs: echo "Setting up environment" build-and-push-build-kit: name: Build and Push Build Kit - uses: everest/everest-ci/.github/workflows/deploy-single-docker-image.yml@v1.3.0 + uses: everest/everest-ci/.github/workflows/deploy-single-docker-image.yml@v1.3.1 needs: setup-env secrets: SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }}