From 67e5e01f99f955fd1b0c94bfe675c3117e14e191 Mon Sep 17 00:00:00 2001 From: Jerico Pena Date: Thu, 21 Mar 2024 11:52:14 -0400 Subject: [PATCH] Add pack cli project.toml descriptior for collecting build files Add dockerfiles and scripts Add github aworkflows for prs and release --- .github/workflows/pull_request.yml | 16 ++++ .github/workflows/release.yml | 27 ++++++ .gitignore | 4 + Dockerfile-example | 78 +++++++++++++++++ Dockerfile-initial | 27 ++++++ README.md | 36 +++++++- pack-with-buildkit.sh | 135 +++++++++++++++++++++++++++++ project.toml | 38 ++++++++ 8 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pull_request.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 Dockerfile-example create mode 100644 Dockerfile-initial create mode 100755 pack-with-buildkit.sh create mode 100644 project.toml diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 0000000..c93eeab --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,16 @@ +name: Build and push to local registry +on: pull_request +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + with: + platforms: linux/amd64,linux/arm64 + - uses: buildpacks/github-actions/setup-pack@v5.5.4 + - uses: buildpacks/github-actions/setup-tools@v5.5.4 + - id: local-registry-push + run: | + ./pack-with-buildkit.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..7edb91e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,27 @@ +name: Build and push to ghcr.io +on: + push: + branches: + - "main" +env: + PUBLISH_TO_GHCR_IO_IMAGE_URI: ghcr.io/jericop/inline-app:latest +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + with: + platforms: linux/amd64,linux/arm64 + - uses: buildpacks/github-actions/setup-pack@v5.5.4 + - uses: buildpacks/github-actions/setup-tools@v5.5.4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GHCR_TOKEN }} + - id: ghcr-push + run: | + ./pack-with-buildkit.sh + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2da6e5a --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +cnb-build-files +Dockerfile +lifecycle-commands.log +pack-build.log \ No newline at end of file diff --git a/Dockerfile-example b/Dockerfile-example new file mode 100644 index 0000000..bf869e8 --- /dev/null +++ b/Dockerfile-example @@ -0,0 +1,78 @@ +# Pin the lifecycle version so we can be sure of the behavior if we run this script in the future +FROM chainguard/crane:latest as crane +FROM buildpacksio/lifecycle:0.16.3 as lifecycle +FROM ghcr.io/jericop/builder-jammy:latest +USER root + +COPY --from=crane /usr/bin/crane /usr/local/bin/crane +COPY --from=lifecycle /cnb/lifecycle /cnb/lifecycle +COPY ./cnb-build-files/cnb/buildpacks /cnb/buildpacks +COPY ./cnb-build-files/layers /layers + +RUN mkdir -p /cache +RUN chown -R cnb:cnb /workspace /cache +RUN chown -R cnb:cnb /layers /platform +RUN find /layers + +USER cnb + +ENV CNB_PLATFORM_API=0.9 + +COPY ./ /workspace + +WORKDIR /workspace + +# Add RUN command(s) below here to run the lifecycle commands and build and export the app to a registry +# It also fixes the architecture because it always gets set to amd64 when running in buildkit. + +RUN < /workspace/lifecycle-build.sh +cat <> /workspace/lifecycle-build.sh + +set -euo pipefail +set -x + +/cnb/lifecycle/analyzer -log-level debug -stack /layers/stack.toml -run-image ghcr.io/jericop/run-jammy:latest localhost:55000/inline-app:$image_arch +/cnb/lifecycle/detector -app /workspace -log-level debug +/cnb/lifecycle/restorer -cache-dir /cache -log-level debug +/cnb/lifecycle/builder -app /workspace -log-level debug +/cnb/lifecycle/exporter -log-level debug -app /workspace -cache-dir /cache -stack /layers/stack.toml localhost:55000/inline-app:$image_arch + +crane pull $image_uri image.tar + +mkdir -p contents +tar -xvf image.tar -C contents +cd contents +ls +cat manifest.json | jq -r '.[0].Config' +current_config=\$(cat manifest.json | jq -r '.[0].Config') +echo \$current_config +cat \$current_config | jq -c ".architecture = \"\${image_arch}\" | .os = \"linux\"" | tr -d '\n' > newconfig.json +new_config="sha256:$(sha256sum newconfig.json | cut -d' ' -f1)" +rm -f "\${current_config}" +mv newconfig.json "\${new_config}" +cat manifest.json| jq -c ".[0].Config = \"\${new_config}\"" | tr -d '\n' > newmanifest.json +mv -f newmanifest.json manifest.json +tar -cvf ../fixed-arch-image.tar * +cd .. + +crane push fixed-arch-image.tar $image_uri + +rm -rf image.tar fixed-arch-image.tar contents + +IN_BUILDKIT_LIFECYCLE_SCRIPT_EOF + +# Run the lifecycle script and ensure correct architecture is set +chmod +x /workspace/lifecycle-build.sh +/workspace/lifecycle-build.sh + +RUN_EOF + diff --git a/Dockerfile-initial b/Dockerfile-initial new file mode 100644 index 0000000..d061c81 --- /dev/null +++ b/Dockerfile-initial @@ -0,0 +1,27 @@ +# Pin the lifecycle version so we can be sure of the behavior if we run this script in the future +FROM chainguard/crane:latest as crane +FROM buildpacksio/lifecycle:0.16.3 as lifecycle +FROM ghcr.io/jericop/builder-jammy:latest +USER root + +COPY --from=crane /usr/bin/crane /usr/local/bin/crane +COPY --from=lifecycle /cnb/lifecycle /cnb/lifecycle +COPY ./cnb-build-files/cnb/buildpacks /cnb/buildpacks +COPY ./cnb-build-files/layers /layers + +RUN mkdir -p /cache +RUN chown -R cnb:cnb /workspace /cache +RUN chown -R cnb:cnb /layers /platform +RUN find /layers + +USER cnb + +ENV CNB_PLATFORM_API=0.9 + +COPY ./ /workspace + +WORKDIR /workspace + +# Add RUN command(s) below here to run the lifecycle commands and build and export the app to a registry +# It also fixes the architecture because it always gets set to amd64 when running in buildkit. + diff --git a/README.md b/README.md index ec3775d..c5a18e8 100644 --- a/README.md +++ b/README.md @@ -1 +1,35 @@ -# cnb-lfx-buildkit-poc \ No newline at end of file +# cnb-lfx-buildkit-poc + +This repo is an initial proof-of-concept for the following LFX mentorship project. + +[CNCF - Cloud Native Buildpacks: Proof of concept making multiarch images with buildkit (2024 Term 1)](https://mentorship.lfx.linuxfoundation.org/project/2c5ced86-d23b-41f5-aec3-59730e29f092) + +It is heavliy inspired by the following article. While the article is invaluable for understanding how the lifecycle works, it requires a fair amount of manual steps, which must be repeated if any changes are made to buildpacks. For the sake of simplicity, I am using an inline buildpack and running `pack build` once in order to copy out the necessary files. + +* https://medium.com/buildpacks/unpacking-cloud-native-buildpacks-ff51b5a767bf + +## Primary objective + +The primary objective is to create a Dockerfile that has all of the lifecycle commands needed to build and push architecture-specific images to a registry. `Dockerfile-example` is an example of how it works. You can then combine the architecture-specific images into a manifest list (multi-arch) image. + +## `pack-with-buildkit.sh` + +This script will do the following: + +* Sets up a buildkit builder with `host` network access +* Starts a local registry on a random port +* Run `pack build` using the `project.toml` which has an inline buildpack that copies the files we need to /workspace +* Copies the files we need out of the container built with pack so we can use them in a new build (with buildkit) +* Generates a `Dockerfile` from `Dockerfile-initial` that will look like `Dockerfile-example` +* Runs `docker buildx build --platform linux/amd64,linux/arm64` to execute the lifecycle commands through buildkit + * This builds and publishes images to the local registry with `amd64` and `arm64` tags + * There is some hackery that happens to fix the architecture (for now) +* Finally it creates a manifest list with the `amd64` and `arm64` tagged images + +## Requirements + +* Multi-arch builders are required. I'm using ones that I build and publish. + +## Published image(s) + +* ghcr.io/jericop/inline-app:latest \ No newline at end of file diff --git a/pack-with-buildkit.sh b/pack-with-buildkit.sh new file mode 100755 index 0000000..766971d --- /dev/null +++ b/pack-with-buildkit.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_name="mybuildpack" +registry_name="local-registry" +builder_name="${registry_name}" + +cleanup() { + docker buildx stop "${builder_name}" > /dev/null 2>&1 + docker buildx rm "${builder_name}" > /dev/null 2>&1 + + log_message "Stopping local registry" + docker stop "${registry_name}" > /dev/null 2>&1 +} + +log_message() { + echo "$1" +} + +# https://github.com/docker/buildx/issues/166#issuecomment-1804970076 +# First, we need to create our own buildx builder that uses the host nework and docker-container driver so that we can get multiarch support +log_message "Setting up docker buildx builder" +docker buildx use "${builder_name}" > /dev/null 2>&1 || docker buildx create --name "${builder_name}" --driver docker-container --driver-opt network=host --bootstrap --use > /dev/null 2>&1 + +# Next, we need to check if a registry is already running +log_message "Setting up a local registry on random port" +docker container inspect "${registry_name}" > /dev/null 2>&1 || docker run -d -e REGISTRY_STORAGE_DELETE_ENABLED=true -p 0:5000 --rm --name "${registry_name}" registry:2 > /dev/null 2>&1 + +# Get local registry port since we asked docker to assign a random port by choosing port 0 +registry_port=$(docker inspect "${registry_name}" | jq -r '.[0].NetworkSettings.Ports["5000/tcp"][0].HostPort') +log_message "Local registry is listening on port ${registry_port}" + +# Rather than creating the necessary files manually, we let pack generate and the inline buildpack +# copy them to /layers/build-files we publish to a local registry so we can capture +pack build localhost:$registry_port/inline-app \ + --publish --verbose --network host 2>&1 | tee pack-build.log \ + | grep -B3 -A4 "Args: '/cnb/lifecycle" > lifecycle-commands.log + +docker pull localhost:$registry_port/inline-app + +if [ -d cnb-build-files ]; then + rm -rf cnb-build-files +fi +mkdir -p cnb-build-files + +# Copy the files created by pack during the build from the volume and into the local directory +docker run --rm --entrypoint bash --user root \ + --volume $(pwd)/cnb-build-files:/hostmnt \ + localhost:$registry_port/inline-app \ + -c 'cp -R /workspace/build-files/* /hostmnt/' + +# We copy the initial Dockerfile (without lifecycle commands) to Dockerfile +cp Dockerfile-initial Dockerfile + +# Then we add a run command (heredoc) with the lifecycle commands to build and push the architecture-specific images +cat <> Dockerfile +RUN < /workspace/lifecycle-build.sh +cat <> /workspace/lifecycle-build.sh + +set -euo pipefail +set -x + +ADD_RUN_COMMAND_TO_DOCKERFILE_EOF + +# Add the lifecycle commands from the saved log output of `pack build` we ran above +while read cmd; do + if echo $cmd | grep -q "localhost:$registry_port/inline-app\$"; then + echo "$cmd:\$image_arch" >> Dockerfile + else + echo "$cmd" >> Dockerfile + fi +done < <(grep Args: lifecycle-commands.log | cut -d"'" -f2) + +cat <<'CONTINUE_ADD_RUN_COMMAND_TO_DOCKERFILE_EOF' >> Dockerfile + +crane pull $image_uri image.tar + +mkdir -p contents +tar -xvf image.tar -C contents +cd contents +ls +cat manifest.json | jq -r '.[0].Config' +current_config=\$(cat manifest.json | jq -r '.[0].Config') +echo \$current_config +cat \$current_config | jq -c ".architecture = \"\${image_arch}\" | .os = \"linux\"" | tr -d '\n' > newconfig.json +new_config="sha256:$(sha256sum newconfig.json | cut -d' ' -f1)" +rm -f "\${current_config}" +mv newconfig.json "\${new_config}" +cat manifest.json| jq -c ".[0].Config = \"\${new_config}\"" | tr -d '\n' > newmanifest.json +mv -f newmanifest.json manifest.json +tar -cvf ../fixed-arch-image.tar * +cd .. + +crane push fixed-arch-image.tar $image_uri + +rm -rf image.tar fixed-arch-image.tar contents + +IN_BUILDKIT_LIFECYCLE_SCRIPT_EOF + +# Run the lifecycle script and ensure correct architecture is set +chmod +x /workspace/lifecycle-build.sh +/workspace/lifecycle-build.sh + +RUN_EOF + +CONTINUE_ADD_RUN_COMMAND_TO_DOCKERFILE_EOF + +# After all of that, we are finally ready to build the multi-arch images using the lifecycle with buildkit. +# We are not publishing with this command because the lifecyle will publish the images to the local registry for us. +docker buildx build --tag ignored --platform linux/amd64,linux/arm64 . + +# Now we can combine the linux/amd64 and linux/arm64 images into a manifest list and push it to the local registry +docker buildx imagetools create \ + --tag localhost:$registry_port/inline-app:latest \ + localhost:$registry_port/inline-app:amd64 \ + localhost:$registry_port/inline-app:arm64 + +crane manifest localhost:$registry_port/inline-app:latest | jq + +if [[ ! -z "${PUBLISH_TO_GHCR_IO_IMAGE_URI:-}" ]]; then + crane copy localhost:$registry_port/inline-app:latest ghcr.io/jericop/inline-app:latest + log_message "Published image to: $PUBLISH_TO_GHCR_IO_IMAGE_URI" +fi + +# This image can then be copied to another registry diff --git a/project.toml b/project.toml new file mode 100644 index 0000000..6bf14c1 --- /dev/null +++ b/project.toml @@ -0,0 +1,38 @@ +[project] +name = "inline-app" + +# stack images need to be multi-arch for this to work +[build] +builder = "ghcr.io/jericop/builder-jammy:latest" + +[[build.buildpacks]] +id = "inline/shell-buildpack" + +[build.buildpacks.script] +api = "0.8" +inline = """ +set -x + +# sleep 300 + +build_files=/workspace/build-files + +mkdir -p $build_files/cnb +mkdir -p $build_files/layers + +ls /cnb/*.toml +cp -r /cnb/*.toml $build_files/cnb/ +cp -r /cnb/buildpacks $build_files/cnb/ + +cp /cnb/order.toml $build_files/layers/ +cp /layers/*.toml $build_files/layers/ + +find $build_files + +cat < ${1}/launch.toml +[[processes]] +type = 'web' +command = 'bin/bash' +default = true +EOF +"""