From 0bcf528455ca6ff9a3775533caee98abeb1f8bee Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Mon, 9 Oct 2023 14:19:02 +0200 Subject: [PATCH 01/14] Refactor flavors and dockerfiles Fixes #1898 Signed-off-by: Dimitris Karakasilis --- .github/flavors-arm.json | 43 -- .github/flavors.json | 359 +++++++++++++- .github/workflows/image-arm-pr.yaml | 10 +- .github/workflows/image-arm.yaml | 22 +- .github/workflows/image-pr.yaml | 26 +- .github/workflows/image.yaml | 157 ++++-- .github/workflows/release-arm.yaml | 4 +- .github/workflows/release.yaml | 26 +- .github/workflows/reusable-build-flavor.yaml | 75 ++- .../reusable-build-framework-flavor.yaml | 5 +- .../workflows/reusable-build-provider.yaml | 36 +- .../workflows/reusable-docker-arm-build.yaml | 31 +- .../workflows/reusable-encryption-test.yaml | 7 +- .github/workflows/reusable-install-test.yaml | 7 +- .../workflows/reusable-provider-tests.yaml | 9 +- ...reusable-provider-upgrade-latest-test.yaml | 7 +- .../reusable-qemu-acceptance-test.yaml | 7 +- .../workflows/reusable-qemu-bundles-test.yaml | 7 +- .../workflows/reusable-qemu-netboot-test.yaml | 24 +- .../workflows/reusable-qemu-reset-test.yaml | 7 +- .../reusable-upgrade-latest-test.yaml | 7 +- .../reusable-upgrade-with-cli-test.yaml | 9 +- .github/workflows/reusable-zfs-test.yaml | 7 +- .github/workflows/uki.yaml | 14 +- Earthfile | 448 +++++++++++------- images/Dockerfile.alpine | 4 +- images/Dockerfile.debian | 5 +- ...idia-jetson-agx-orin => Dockerfile.nvidia} | 0 images/Dockerfile.opensuse-leap | 3 +- images/Dockerfile.opensuse-tumbleweed | 3 +- images/Dockerfile.ubuntu | 104 ++-- naming.sh | 107 +++++ 32 files changed, 1198 insertions(+), 382 deletions(-) delete mode 100644 .github/flavors-arm.json rename images/{Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin => Dockerfile.nvidia} (100%) create mode 100755 naming.sh diff --git a/.github/flavors-arm.json b/.github/flavors-arm.json deleted file mode 100644 index 210ee0eb8..000000000 --- a/.github/flavors-arm.json +++ /dev/null @@ -1,43 +0,0 @@ -[ - { - "flavor": "opensuse-leap-arm-rpi", - "model": "rpi4", - "worker": "ubuntu-latest", - "standard": "true" - }, - { - "flavor": "opensuse-tumbleweed-arm-rpi", - "model": "rpi4", - "worker": "ubuntu-latest", - "standard": "true" - }, - { - "flavor": "alpine-arm-rpi", - "model": "rpi4", - "worker": "ubuntu-latest", - "standard": "true" - }, - { - "flavor": "ubuntu-arm-rpi", - "model": "rpi4", - "worker": "ubuntu-latest", - "standard": "true" - }, - { - "flavor": "ubuntu-20-lts-arm-rpi", - "model": "rpi4", - "worker": "ubuntu-latest", - "standard": "true" - }, - { - "flavor": "ubuntu-22-lts-arm-rpi", - "model": "rpi4", - "worker": "ubuntu-latest", - "standard": "true" - }, - { - "flavor": "ubuntu-20-lts-arm-nvidia-jetson-agx-orin", - "model": "none", - "worker": "fast" - } -] diff --git a/.github/flavors.json b/.github/flavors.json index 8ba95ebd6..91bab4ca7 100644 --- a/.github/flavors.json +++ b/.github/flavors.json @@ -1,41 +1,372 @@ [ { - "flavor": "opensuse-leap", - "standard": "true" + "family": "opensuse-leap", + "flavor": "opensuse", + "flavorRelease": "leap-15.5", + "variant": "standard", + "model": "generic", + "baseImage": "opensuse/leap:15.5", + "arch": "amd64", + "worker": "self-hosted" }, { - "flavor": "opensuse-tumbleweed", - "standard": "true" + "family": "opensuse-leap", + "flavor": "opensuse", + "flavorRelease": "leap-15.5", + "variant": "core", + "model": "generic", + "baseImage": "opensuse/leap:15.5", + "arch": "amd64", + "worker": "self-hosted" }, { + "family": "opensuse-leap", + "flavor": "opensuse", + "flavorRelease": "leap-15.5", + "variant": "standard", + "model": "rpi4", + "baseImage": "opensuse/leap:15.5", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "opensuse-leap", + "flavor": "opensuse", + "flavorRelease": "leap-15.5", + "variant": "core", + "model": "rpi4", + "baseImage": "opensuse/leap:15.5", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "opensuse-leap", + "flavor": "opensuse", + "flavorRelease": "leap-15.5", + "variant": "core", + "model": "generic", + "baseImage": "opensuse/leap:15.5", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "opensuse-tumbleweed", + "flavor": "opensuse", + "flavorRelease": "tumbleweed", + "variant": "standard", + "model": "generic", + "baseImage": "opensuse/tumbleweed", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "opensuse-tumbleweed", + "flavor": "opensuse", + "flavorRelease": "tumbleweed", + "variant": "core", + "model": "generic", + "baseImage": "opensuse/tumbleweed", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "opensuse-tumbleweed", + "flavor": "opensuse", + "flavorRelease": "tumbleweed", + "variant": "standard", + "model": "rpi4", + "baseImage": "opensuse/tumbleweed", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "opensuse-tumbleweed", + "flavor": "opensuse", + "flavorRelease": "tumbleweed", + "variant": "core", + "model": "rpi4", + "baseImage": "opensuse/tumbleweed", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "20.04", + "variant": "standard", + "model": "generic", + "baseImage": "ubuntu:20.04", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "20.04", + "variant": "core", + "model": "generic", + "baseImage": "ubuntu:20.04", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "20.04", + "variant": "standard", + "model": "rpi4", + "baseImage": "ubuntu:20.04", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "20.04", + "variant": "core", + "model": "rpi4", + "baseImage": "ubuntu:20.04", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "ubuntu", "flavor": "ubuntu", - "standard": "true" + "flavorRelease": "22.04", + "variant": "standard", + "model": "generic", + "baseImage": "ubuntu:22.04", + "arch": "amd64", + "worker": "self-hosted" }, { - "flavor": "ubuntu-20-lts", - "standard": "true" + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "22.04", + "variant": "core", + "model": "generic", + "baseImage": "ubuntu:22.04", + "arch": "amd64", + "worker": "self-hosted" }, { - "flavor": "ubuntu-22-lts", - "standard": "true" + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "22.04", + "variant": "standard", + "model": "rpi4", + "baseImage": "ubuntu:22.04", + "arch": "arm64", + "worker": "ubuntu-latest" }, { + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "22.04", + "variant": "core", + "model": "rpi4", + "baseImage": "ubuntu:22.04", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "23.04", + "variant": "standard", + "model": "generic", + "baseImage": "ubuntu:23.04", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "23.04", + "variant": "core", + "model": "generic", + "baseImage": "ubuntu:23.04", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "23.04", + "variant": "standard", + "model": "rpi4", + "baseImage": "ubuntu:23.04", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "23.04", + "variant": "core", + "model": "rpi4", + "baseImage": "ubuntu:23.04", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "ubuntu", + "flavor": "ubuntu", + "flavorRelease": "23.04", + "variant": "core", + "model": "generic", + "baseImage": "ubuntu:23.04", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "alpine", "flavor": "alpine", - "standard": "true" + "flavorRelease": "3.18", + "variant": "standard", + "model": "generic", + "baseImage": "alpine:3.18", + "arch": "amd64", + "worker": "self-hosted" }, { + "family": "alpine", + "flavor": "alpine", + "flavorRelease": "3.18", + "variant": "core", + "model": "generic", + "baseImage": "alpine:3.18", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "alpine", + "flavor": "alpine", + "flavorRelease": "3.18", + "variant": "standard", + "model": "rpi4", + "baseImage": "alpine:3.18", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "alpine", + "flavor": "alpine", + "flavorRelease": "3.18", + "variant": "core", + "model": "rpi4", + "baseImage": "alpine:3.18", + "arch": "arm64", + "worker": "ubuntu-latest" + }, + { + "family": "fedora", "flavor": "fedora", - "standard": "true" + "flavorRelease": "38", + "variant": "standard", + "model": "generic", + "baseImage": "fedora:38", + "arch": "amd64", + "worker": "self-hosted" }, { + "family": "fedora", + "flavor": "fedora", + "flavorRelease": "38", + "variant": "core", + "model": "generic", + "baseImage": "fedora:38", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "debian", + "flavor": "debian", + "flavorRelease": "testing", + "variant": "standard", + "model": "generic", + "baseImage": "debian:testing", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "debian", + "flavor": "debian", + "flavorRelease": "testing", + "variant": "core", + "model": "generic", + "baseImage": "debian:testing", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "debian", + "flavor": "debian", + "flavorRelease": "bookworm", + "variant": "standard", + "model": "generic", + "baseImage": "debian:bookworm-slim", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "debian", "flavor": "debian", - "standard": "true" + "flavorRelease": "bookworm", + "variant": "core", + "model": "generic", + "baseImage": "debian:bookworm-slim", + "arch": "amd64", + "worker": "self-hosted" }, { + "family": "rockylinux", "flavor": "rockylinux", - "standard": "true" + "flavorRelease": "9", + "variant": "standard", + "model": "generic", + "baseImage": "rockylinux:9", + "arch": "amd64", + "worker": "self-hosted" }, { - "flavor": "almalinux" + "family": "rockylinux", + "flavor": "rockylinux", + "flavorRelease": "9", + "variant": "core", + "model": "generic", + "baseImage": "rockylinux:9", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "almalinux", + "flavor": "almalinux", + "flavorRelease": "9", + "variant": "standard", + "model": "generic", + "baseImage": "almalinux:9", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "almalinux", + "flavor": "almalinux", + "flavorRelease": "9", + "variant": "core", + "model": "generic", + "baseImage": "almalinux:9", + "arch": "amd64", + "worker": "self-hosted" + }, + { + "family": "nvidia", + "flavor": "ubuntu", + "flavorRelease": "20.04", + "variant": "core", + "model": "nvidia-jetson-agx-orin", + "baseImage": "ubuntu:20.04", + "arch": "arm64", + "worker": "fast" } ] diff --git a/.github/workflows/image-arm-pr.yaml b/.github/workflows/image-arm-pr.yaml index acfcef4d3..f35a8227c 100644 --- a/.github/workflows/image-arm-pr.yaml +++ b/.github/workflows/image-arm-pr.yaml @@ -15,12 +15,18 @@ jobs: docker: uses: ./.github/workflows/reusable-docker-arm-build.yaml with: - flavor: opensuse-leap-arm-rpi + flavor: opensuse + flavor_release: leap-15.5 + family: opensuse-leap + base_image: opensuse/leap:15.5 model: rpi4 worker: fast docker-alpine: uses: ./.github/workflows/reusable-docker-arm-build.yaml with: - flavor: alpine-arm-rpi + flavor: alpine + flavor_release: "3.18" + family: alpine + base_image: alpine:3.18 model: rpi4 worker: fast diff --git a/.github/workflows/image-arm.yaml b/.github/workflows/image-arm.yaml index 489df5353..0a637861b 100644 --- a/.github/workflows/image-arm.yaml +++ b/.github/workflows/image-arm.yaml @@ -22,7 +22,7 @@ jobs: git fetch --prune --unshallow - id: set-matrix run: | - content=`cat ./.github/flavors-arm.json` + content=`cat ./.github/flavors.json | jq -r 'map(select(.arch == "arm64" and .variant == "standard" and .model != "generic"))'` # the following lines are only required for multi line json content="${content//'%'/'%25'}" content="${content//$'\n'/'%0A'}" @@ -49,7 +49,7 @@ jobs: # Construct an array like this from the found versions: sudo luet --config framework-profile.yaml search -o json k8s/k3s | jq '.packages | map(.version) | unique' > k3s_versions.json # Create a combination of flavors and k3s versions. - content=$(jq -s '. | [combinations | .[0] + {"k3s_version": .[1]}] | map(select(.frameworkonly != "true")) | map(select(.standard == "true"))' .github/flavors-arm.json k3s_versions.json) + content=$(jq -s '. | [combinations | .[0] + {"k3s_version": .[1]}] | map(select(.arch == "arm64" and .variant == "standard" and .model != "generic"))' .github/flavors.json k3s_versions.json) # the following lines are only required for multi line json content="${content//'%'/'%25'}" content="${content//$'\n'/'%0A'}" @@ -62,7 +62,10 @@ jobs: secrets: inherit with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} + family: ${{ matrix.family }} model: ${{ matrix.model }} + base_image: ${{ matrix.baseImage }} worker: ${{ matrix.worker }} needs: - get-core-matrix @@ -74,14 +77,13 @@ jobs: uses: ./.github/workflows/reusable-image-and-iso-arm-generic.yaml secrets: inherit with: - flavor: ${{ matrix.flavor }} - needs: - - get-core-matrix - strategy: - fail-fast: false - matrix: - flavor: - - "opensuse-leap" + flavor: "opensuse" + flavor_release: "leap-15.5" + family: opensuse + model: generic + arch: arm64 + base_image: opensuse/leap:15.5 + variant: core notify: runs-on: ubuntu-latest diff --git a/.github/workflows/image-pr.yaml b/.github/workflows/image-pr.yaml index 96c14b14e..1775795bd 100644 --- a/.github/workflows/image-pr.yaml +++ b/.github/workflows/image-pr.yaml @@ -1,4 +1,3 @@ -name: Build and test PRs on: pull_request: paths: @@ -14,16 +13,29 @@ jobs: uses: ./.github/workflows/reusable-build-flavor.yaml with: flavor: ubuntu + flavor_release: "23.04" + family: ubuntu + base_image: ubuntu:23.04 + model: generic + variant: core + arch: amd64 core-alpine: uses: ./.github/workflows/reusable-build-flavor.yaml with: flavor: alpine + flavor_release: "3.18" + family: alpine + base_image: alpine:3.18 + model: generic + variant: core + arch: amd64 install: uses: ./.github/workflows/reusable-install-test.yaml with: flavor: ubuntu + flavor_release: "23.04" needs: - core @@ -38,6 +50,7 @@ jobs: uses: ./.github/workflows/reusable-zfs-test.yaml with: flavor: ubuntu + flavor_release: "23.04" needs: - core @@ -45,6 +58,7 @@ jobs: uses: ./.github/workflows/reusable-qemu-acceptance-test.yaml with: flavor: ubuntu + flavor_release: "23.04" needs: - core @@ -59,6 +73,7 @@ jobs: uses: ./.github/workflows/reusable-qemu-bundles-test.yaml with: flavor: ubuntu + flavor_release: "23.04" needs: - core @@ -66,6 +81,7 @@ jobs: uses: ./.github/workflows/reusable-qemu-reset-test.yaml with: flavor: ubuntu + flavor_release: "23.04" needs: - core @@ -80,6 +96,11 @@ jobs: uses: ./.github/workflows/reusable-qemu-netboot-test.yaml with: flavor: ubuntu + flavor_release: "23.04" + family: ubuntu + base_image: ubuntu:23.04 + model: generic + variant: core needs: - core @@ -94,6 +115,7 @@ jobs: uses: ./.github/workflows/reusable-upgrade-with-cli-test.yaml with: flavor: ubuntu + flavor_release: "23.04" needs: - core @@ -108,6 +130,7 @@ jobs: uses: ./.github/workflows/reusable-upgrade-latest-test.yaml with: flavor: ubuntu + flavor_release: "23.04" needs: - core @@ -123,6 +146,7 @@ jobs: uses: ./.github/workflows/reusable-encryption-test.yaml with: flavor: ubuntu + flavor_release: "23.04" label: ${{ matrix.label }} needs: - core diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 82171fa3c..af56e07dc 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -1,5 +1,8 @@ name: Build and test images on: + pull_request: + paths: + - '**' push: branches: - master @@ -12,7 +15,7 @@ concurrency: env: FORCE_COLOR: 1 jobs: - get-matrix: + get-core-matrix: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -23,7 +26,7 @@ jobs: sudo apt update && sudo apt install -y jq - id: set-matrix run: | - content=`cat .github/flavors.json | jq 'map(select(.frameworkonly != "true"))'` + content=`cat .github/flavors.json | jq 'map(select(.variant == "core"))'` # the following lines are only required for multi line json # the following lines are only required for multi line json content="${content//'%'/'%25'}" @@ -37,11 +40,17 @@ jobs: secrets: inherit with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} + family: ${{ matrix.family }} + base_image: ${{ matrix.baseImage }} + model: ${{ matrix.model }} + variant: ${{ matrix.variant }} + arch: ${{ matrix.arch }} needs: - - get-matrix + - get-core-matrix strategy: fail-fast: false - matrix: ${{fromJson(needs.get-matrix.outputs.matrix)}} + matrix: ${{fromJson(needs.get-core-matrix.outputs.matrix)}} framework: uses: ./.github/workflows/reusable-build-framework-flavor.yaml secrets: inherit @@ -57,17 +66,20 @@ jobs: uses: ./.github/workflows/reusable-install-test.yaml with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} needs: - core strategy: fail-fast: true matrix: include: - - flavor: opensuse-leap + - flavor: opensuse + flavorRelease: leap-15.5 zfs: uses: ./.github/workflows/reusable-zfs-test.yaml with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} needs: - core strategy: @@ -75,10 +87,12 @@ jobs: matrix: include: - flavor: "ubuntu" + flavorRelease: "23.04" acceptance: uses: ./.github/workflows/reusable-qemu-acceptance-test.yaml with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} needs: - core strategy: @@ -86,27 +100,37 @@ jobs: matrix: include: - flavor: "alpine" - - flavor: "opensuse-leap" - - flavor: "opensuse-tumbleweed" + flavorRelease: "3.18" + - flavor: "opensuse" + flavorRelease: "leap-15.5" + - flavor: "opensuse" + flavorRelease: "tumbleweed" - flavor: "ubuntu" + flavorRelease: "23.04" - flavor: "debian" - - flavor: "ubuntu-20-lts" + flavorRelease: "testing" + - flavor: "ubuntu" + flavorRelease: "20.04" - flavor: "ubuntu-22-lts" + flavorRelease: "22.04" bundles: uses: ./.github/workflows/reusable-qemu-bundles-test.yaml with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} needs: - core strategy: fail-fast: false matrix: include: - - flavor: opensuse-tumbleweed # Kubo test needs systemd version 252+ which atm is not available in Leap + - flavor: opensuse # Kubo test needs systemd version 252+ which atm is not available in Leap + flavorRelease: opensuse reset: uses: ./.github/workflows/reusable-qemu-reset-test.yaml with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} needs: - core strategy: @@ -114,11 +138,26 @@ jobs: matrix: include: - flavor: alpine - - flavor: opensuse-leap + flavorRelease: "3.18" + family: alpine + base_image: alpine:3.18 + model: generic + variant: core + - flavor: opensuse + flavorRelease: leap-15.5 + family: opensuse-leap + base_image: opensuse/leap:15.5 + model: generic + variant: core netboot: uses: ./.github/workflows/reusable-qemu-netboot-test.yaml with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} + family: ${{ matrix.family }} + model: ${{ matrix.model }} + variant: ${{ matrix.variant }} + base_image: ${{ matrix.baseImage }} needs: - core strategy: @@ -126,13 +165,28 @@ jobs: matrix: include: - flavor: alpine - - flavor: opensuse-leap + flavorRelease: "3.18" + family: alpine + variant: core + model: generic + baseImage: alpine:3.18 + - flavor: opensuse + flavorRelease: leap-15.5 + family: opensuse-leap + variant: core + model: generic + baseImage: opensuse/leap:15.5 - flavor: ubuntu + flavorRelease: "23.04" + family: ubuntu + variant: core + model: generic + baseImage: ubuntu/23.04 upgrade: uses: ./.github/workflows/reusable-upgrade-with-cli-test.yaml with: flavor: ${{ matrix.flavor }} - + flavor_release: ${{ matrix.flavorRelease }} needs: - core strategy: @@ -140,11 +194,14 @@ jobs: matrix: include: - flavor: alpine - - flavor: opensuse-leap + flavorRelease: "3.18" + - flavor: opensuse + flavorRelease: leap-15.5 upgrade-latest: uses: ./.github/workflows/reusable-upgrade-latest-test.yaml with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} needs: - core strategy: @@ -152,14 +209,14 @@ jobs: matrix: include: - flavor: alpine - - flavor: opensuse-leap - # - flavor: "ubuntu" - # - flavor: "ubuntu" - # - flavor: "ubuntu" + flavorRelease: "3.18" + - flavor: opensuse + flavorRelease: leap-15.5 encryption: uses: ./.github/workflows/reusable-encryption-test.yaml with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} label: ${{ matrix.label }} needs: - core @@ -173,21 +230,37 @@ jobs: - "remote-https-pinned" - "remote-https-bad-cert" flavor: - - "opensuse-leap" + - "opensuse" + flavorRelease: + - "leap-15.5" standard: uses: ./.github/workflows/reusable-build-provider.yaml with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} + family: ${{ matrix.family }} + base_image: ${{ matrix.baseImage }} + variant: standard + model: generic + arch: amd64 strategy: fail-fast: false matrix: - flavor: - - "opensuse-leap" - - "alpine-opensuse-leap" + include: + - flavor: opensuse + flavorRelease: leap-15.5 + family: opensuse-leap + baseImage: opensuse/leap:15.5 + - flavor: alpine + flavorRelease: "3.18" + family: alpine + baseImage: alpine:3.18 + various: uses: ./.github/workflows/reusable-provider-tests.yaml with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} label: ${{ matrix.label }} needs: - standard @@ -195,32 +268,42 @@ jobs: fail-fast: false max-parallel: 2 matrix: - label: - - "provider-qrcode-install" - - "provider-upgrade" - - "provider-decentralized-k8s" - - "provider-upgrade-k8s" - flavor: - - "opensuse-leap" - - "alpine-opensuse-leap" - exclude: # looks like only the k8s stuff is tested on both flavors - - label: "provider-qrcode-install" - flavor: "alpine-opensuse-leap" - - label: "provider-upgrade" - flavor: "alpine-opensuse-leap" + include: + - flavor: "opensuse" + flavorRelease: "leap/15.5" + label: "provider-qrcode-install" + - flavor: "opensuse" + flavorRelease: "leap/15.5" + label: "provider-upgrade" + - flavor: "opensuse" + flavorRelease: "leap/15.5" + label: "provider-decentralized-k8s" + - flavor: "opensuse" + flavorRelease: "leap/15.5" + label: "provider-upgrade-k8s" + - flavor: "alpine" + flavorRelease: "3.18" + label: "provider-decentralized-k8s" + - flavor: "alpine" + flavorRelease: "3.18" + label: "provider-upgrade-k8s" standard-upgrade-latest: uses: ./.github/workflows/reusable-provider-upgrade-latest-test.yaml with: flavor: ${{ matrix.flavor }} + flavor_release: ${{ matrix.flavorRelease }} needs: - standard strategy: fail-fast: false max-parallel: 2 matrix: - flavor: - - "opensuse-leap" - - "alpine-opensuse-leap" + include: + - flavor: "opensuse" + flavorRelease: "leap-15.5" + - flavor: "alpine" + flavorRelease: "3.18" + test-uki: runs-on: kvm steps: diff --git a/.github/workflows/release-arm.yaml b/.github/workflows/release-arm.yaml index 86aa174a0..3d668476f 100644 --- a/.github/workflows/release-arm.yaml +++ b/.github/workflows/release-arm.yaml @@ -14,7 +14,7 @@ jobs: fetch-depth: 0 - id: set-matrix run: | - content=`cat ./.github/flavors-arm.json` + content=`cat ./.github/flavors.json | map(select(.arch == "arm64"))` # the following lines are only required for multi line json content="${content//'%'/'%25'}" content="${content//$'\n'/'%0A'}" @@ -41,7 +41,7 @@ jobs: # Construct an array like this from the found versions: sudo luet --config framework-profile.yaml search -o json k8s/k3s | jq '.packages | map(.version) | unique' > k3s_versions.json # Create a combination of flavors and k3s versions. - content=$(jq -s '. | [combinations | .[0] + {"k3s_version": .[1]}] | map(select(.frameworkonly != "true")) | map(select(.standard == "true"))' .github/flavors-arm.json k3s_versions.json) + content=$(jq -s '. | [combinations | .[0] + {"k3s_version": .[1]}] | map(select(.variant == "standard" and .arch == "arm64"))' .github/flavors.json k3s_versions.json) # the following lines are only required for multi line json content="${content//'%'/'%25'}" content="${content//$'\n'/'%0A'}" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index cec58904d..4aadf3041 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,7 +19,7 @@ jobs: sudo apt update && sudo apt install -y jq - id: set-matrix run: | - content=`cat .github/flavors.json | jq 'map(select(.frameworkonly != "true"))'` + content=`cat .github/flavors.json` # the following lines are only required for multi line json content="${content//'%'/'%25'}" content="${content//$'\n'/'%0A'}" @@ -45,7 +45,7 @@ jobs: sudo mv luet /usr/bin/luet # Construct an array like this from the found versions: sudo luet --config framework-profile.yaml search -o json k8s/k3s | jq '.packages | map(.version) | unique' > k3s_versions.json - content=$(jq -s '. | [combinations | .[0] + {"k3s_version": .[1]}] | map(select(.frameworkonly != "true")) | map(select(.standard == "true"))' .github/flavors.json k3s_versions.json) + content=$(jq -s '. | [combinations | .[0] + {"k3s_version": .[1]}] | map(select(.variant == "standard"))' .github/flavors.json k3s_versions.json) # the following lines are only required for multi line json content="${content//'%'/'%25'}" content="${content//$'\n'/'%0A'}" @@ -79,8 +79,6 @@ jobs: - name: Login to Quay Registry run: echo ${{ secrets.QUAY_PASSWORD }} | docker login -u ${{ secrets.QUAY_USERNAME }} --password-stdin quay.io - name: Build 🔧 - env: - FLAVOR: ${{ matrix.flavor }} run: | # Configure earthly to use the docker mirror in CI # https://docs.earthly.dev/ci-integration/pull-through-cache#configuring-earthly-to-use-the-cache @@ -94,7 +92,7 @@ jobs: insecure = true http = true EOF - earthly +build-framework-image --FLAVOR=${FLAVOR} + earthly +build-framework-image --FLAVOR=${{ matrix.flavor }} --FLAVOR_RELEASE=${{ matrix.flavorRelease }} - name: Push to quay env: COSIGN_YES: true @@ -166,8 +164,24 @@ jobs: - name: Build 🔧 env: FLAVOR: ${{ matrix.flavor }} + FLAVOR_RELEASE: ${{ matrix.flavorRelease }} + FAMILY: ${{ matrix.family }} + ARTIFACT: ${{ matrix.artifact }} + VARIANT: ${{ matrix.variant }} + MODEL: ${{ matrix.model }} + BASE_IMAGE: ${{ matrix.baseImage }} + # TODO: Create a reusable script/workflow/something that generates the artifact name + # TODO: Use the naming.sh script run: | - earthly +all --VARIANT=core --FLAVOR=$FLAVOR + earthly +all \ + --VARIANT=${{ matrix.variant }} \ + --FAMILY=${{ matrix.family }} \ + --FLAVOR=${{ matrix.flavor }} \ + --FLAVOR=${{ matrix.flavorRelease }} \ + --MODEL=${{ matrix.model }} \ + --BASE_IMAGE=${{ matrix.baseImage }} \ + --IMAGE=quay.io/kairos/${{ matrix.variant }}-${{ inputs.flavor }}-${{ matrix.model }}:latest + sudo mv build release - name: Push to quay env: diff --git a/.github/workflows/reusable-build-flavor.yaml b/.github/workflows/reusable-build-flavor.yaml index 4d48c9810..ad4078cc5 100644 --- a/.github/workflows/reusable-build-flavor.yaml +++ b/.github/workflows/reusable-build-flavor.yaml @@ -6,6 +6,24 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string + family: + required: true + type: string + base_image: + required: true + type: string + model: + required: true + type: string + variant: + required: true + type: string + arch: + required: true + type: string jobs: build: @@ -72,19 +90,41 @@ jobs: - name: Build PR 🔧 if: ${{ github.event_name == 'pull_request' }} env: - FLAVOR: ${{ inputs.flavor }} - IMAGE: quay.io/kairos/core-${{ inputs.flavor }}:latest + VERSION: latest run: | - earthly +ci --SECURITY_SCANS=false --VARIANT=core --IMAGE=$IMAGE --FLAVOR=$FLAVOR + # Desired artifact names: + # container image: quay.io/kairos/ubuntu:22.04-core-amd64-generic-v2.4.0 + # iso: kairos-ubuntu-20.04-core-amd64-generic-v2.4.1-ipxe.iso + # + # TODO: + # - New branch: + # - Merge alma, rocky, fedora to one family dockerfile + # - Move rest of logic from Earthfile to Dockerfiles + # - Create a reusable worflow that simply calls docker build passing the required build-args + # - Delete the earthly base-image target and force the use of the above workflow anywhere needed + # - Make all remaining earthly targets to use a pre-cooked KAIROS_IMAGE rather than calling +base-image + earthly --platform=linux/${{ inputs.arch }} +ci \ + --SECURITY_SCANS=false \ + --VARIANT=${{ inputs.variant }} \ + --FLAVOR=${{ inputs.flavor }} \ + --FLAVOR_RELEASE=${{ inputs.flavor_release }} \ + --BASE_IMAGE=${{ inputs.base_image }} \ + --MODEL=${{ inputs.model }} \ + --FAMILY=${{ inputs.family }} sudo mv build/* . sudo rm -rf build - name: Build master 🔧 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - env: - FLAVOR: ${{ inputs.flavor }} - IMAGE: quay.io/kairos/core-${{ inputs.flavor }}:latest run: | - earthly +ci --VARIANT=core --IMAGE=$IMAGE --FLAVOR=$FLAVOR + earthly --platform=linux/${{ inputs.arch }} +ci \ + --SECURITY_SCANS=false \ + --VARIANT=${{ inputs.variant }} \ + --FLAVOR=${{ inputs.flavor }} \ + --FLAVOR_RELEASE=${{ inputs.flavor_release }} \ + --BASE_IMAGE=${{ inputs.base_image }} \ + --MODEL=${{ inputs.model }} \ + --FAMILY=${{ inputs.family }} + sudo mv build/* . sudo rm -rf build mkdir sarif @@ -94,10 +134,10 @@ jobs: uses: github/codeql-action/upload-sarif@v2 with: sarif_file: 'sarif' - category: ${{ inputs.flavor }} + category: ${{ inputs.flavor }}-${{ inputs.flavor_release }} - uses: actions/upload-artifact@v3 with: - name: kairos-${{ inputs.flavor }}.iso.zip + name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}.iso.zip path: | *.iso *.sha256 @@ -106,7 +146,7 @@ jobs: - uses: actions/upload-artifact@v3 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} with: - name: kairos-${{ inputs.flavor }}.sbom.zip + name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}.sbom.zip path: | *.syft.json *.spdx.json @@ -114,12 +154,13 @@ jobs: - name: Push to quay if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} env: - IMAGE: "quay.io/kairos/core-${{ inputs.flavor }}" - TAG: "latest" COSIGN_YES: true run: | - docker push "$IMAGE:$TAG" - image_ref=$(docker image inspect --format='{{index .RepoDigests 0}}' "$IMAGE:$TAG") + IMAGE=$(cat IMAGE) + VERSION=$(cat VERSION) + IMAGE="${IMAGE/$VERSION/latest}" + docker push $IMAGE + image_ref=$(docker image inspect --format='{{index .RepoDigests 0}}' "$IMAGE") spdx=$(ls *.spdx.json) cosign attach sbom --sbom $spdx $image_ref cosign sign $image_ref --attachment sbom @@ -127,5 +168,7 @@ jobs: cosign attest --type spdx --predicate $spdx $image_ref - name: Push to testing run: | - docker tag quay.io/kairos/core-${{ inputs.flavor }}:latest ttl.sh/kairos-${{ inputs.flavor }}-${{ github.sha }}:24h - docker push ttl.sh/kairos-${{ inputs.flavor }}-${{ github.sha }}:24h + _IMG=$(cat IMAGE) + TEST_IMAGE="ttl.sh/kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}-${{ github.sha }}:24h" + docker tag $_IMG $TEST_IMAGE + docker push $TEST_IMAGE diff --git a/.github/workflows/reusable-build-framework-flavor.yaml b/.github/workflows/reusable-build-framework-flavor.yaml index 266fcb46e..74eb76c59 100644 --- a/.github/workflows/reusable-build-framework-flavor.yaml +++ b/.github/workflows/reusable-build-framework-flavor.yaml @@ -6,6 +6,9 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string jobs: build: @@ -48,6 +51,6 @@ jobs: insecure = true http = true EOF - earthly +build-framework-image --FLAVOR=${FLAVOR} --VERSION=master + earthly +build-framework-image --FLAVOR=${FLAVOR} --FLAVOR_RELEASE=${FLAVOR_RELEASE} --VERSION=master docker push "$IMAGE:$TAG" # Otherwise .RepoDigests will be empty for some reason cosign sign $(docker image inspect --format='{{index .RepoDigests 0}}' "$IMAGE:$TAG") diff --git a/.github/workflows/reusable-build-provider.yaml b/.github/workflows/reusable-build-provider.yaml index b2515a396..54d29df83 100644 --- a/.github/workflows/reusable-build-provider.yaml +++ b/.github/workflows/reusable-build-provider.yaml @@ -6,6 +6,24 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string + base_image: + required: true + type: string + model: + required: true + type: string + family: + required: true + type: string + variant: + required: true + type: string + arch: + required: true + type: string jobs: build: @@ -63,16 +81,21 @@ jobs: - name: Build PR 🔧 run: | K3S_VERSION=$(sudo luet --config framework-profile.yaml search -o json k8s/k3s | jq '.packages | map(.version) | unique | last') - earthly +ci \ + earthly --platform=linux/${{ inputs.arch }} +ci \ --SECURITY_SCANS=false \ - --IMAGE=ttl.sh/kairos-${{ inputs.flavor }}-${{ github.sha }}-provider:24h \ + --VARIANT=${{ inputs.variant }} \ --FLAVOR=${{ inputs.flavor }} \ - --VARIANT=standard -K3S_VERSION=${K3S_VERSION} + --FLAVOR_RELEASE=${{ inputs.flavor_release }} \ + --BASE_IMAGE=${{ inputs.base_image }} \ + --MODEL=${{ inputs.model }} \ + --FAMILY=${{ inputs.family }} \ + --K3S_VERSION=${K3S_VERSION} + sudo mv build/* . sudo rm -rf build - uses: actions/upload-artifact@v3 with: - name: kairos-${{ inputs.flavor }}-provider.iso.zip + name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}-provider.iso.zip path: | *.iso *.sha256 @@ -80,4 +103,7 @@ jobs: if-no-files-found: error - name: Push to testing run: | - docker push ttl.sh/kairos-${{ inputs.flavor }}-${{ github.sha }}-provider:24h + _IMG=$(cat IMAGE) + TEST_IMAGE="ttl.sh/kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}-${{ github.sha }}-provider:24h" + docker tag $_IMG $TEST_IMAGE + docker push $TEST_IMAGE diff --git a/.github/workflows/reusable-docker-arm-build.yaml b/.github/workflows/reusable-docker-arm-build.yaml index 8949bf2e2..632315137 100644 --- a/.github/workflows/reusable-docker-arm-build.yaml +++ b/.github/workflows/reusable-docker-arm-build.yaml @@ -1,5 +1,9 @@ name: Reusable workflow - docker arm build +# TODO: Merge this workflow with the reusable-build-flavor one +# The same workflow should be able to build any combination of flavor, +# model, architecture, etc. + on: workflow_call: inputs: @@ -9,9 +13,18 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string + family: + required: true + type: string model: required: true type: string + base_image: + required: true + type: string jobs: build: @@ -87,18 +100,16 @@ jobs: run: | sudo iptables -I INPUT -s 169.254.169.254 -j DROP sudo iptables -I OUTPUT -d 169.254.169.254 -j DROP - - name: Shared runner build 🔧 - if: ${{ inputs.worker != 'fast' }} - run: | - ./earthly.sh +all-arm --VARIANT=core --MODEL=${{ inputs.model }} \ - --FLAVOR=${{ inputs.flavor }} \ - --IMG_COMPRESSION=${{env.IMG_COMPRESSION}} - - name: fast runner Build 🔧 - if: ${{ inputs.worker == 'fast' }} + - name: Build 🔧 run: | earthly --allow-privileged +all-arm \ - --VARIANT=core --MODEL=${{ inputs.model }} \ - --FLAVOR=${{ inputs.flavor }} --IMG_COMPRESSION=${{env.IMG_COMPRESSION}} + --FAMILY=${{ inputs.family }} \ + --FLAVOR=${{ inputs.flavor }} \ + --FLAVOR_RELEASE=${{ inputs.flavor_release }} \ + --BASE_IMAGE=${{ inputs.base_image }} \ + --MODEL=${{ inputs.model }} \ + --VARIANT=core \ + --IMG_COMPRESSION=${{env.IMG_COMPRESSION}} - name: Show img sizes run: | ls -ltra build diff --git a/.github/workflows/reusable-encryption-test.yaml b/.github/workflows/reusable-encryption-test.yaml index b955a568e..81fef0446 100644 --- a/.github/workflows/reusable-encryption-test.yaml +++ b/.github/workflows/reusable-encryption-test.yaml @@ -6,6 +6,9 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string label: required: true type: string @@ -74,7 +77,7 @@ jobs: id: iso uses: actions/download-artifact@v3 with: - name: kairos-${{ inputs.flavor }}.iso.zip + name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}.iso.zip - name: Display structure of downloaded files run: ls -las . - name: Block all traffic to metadata ip # For cloud runners, the metadata ip can interact with our test machines @@ -87,5 +90,5 @@ jobs: EMULATE_TPM: true USE_QEMU: true run: | - export ISO=$(ls $PWD/kairos-core-*${{ inputs.flavor }}*.iso) + export ISO=$(ls $PWD/*${{ inputs.flavor }}*.iso) .github/encryption-tests.sh diff --git a/.github/workflows/reusable-install-test.yaml b/.github/workflows/reusable-install-test.yaml index 657df0f32..6894e4677 100644 --- a/.github/workflows/reusable-install-test.yaml +++ b/.github/workflows/reusable-install-test.yaml @@ -6,6 +6,9 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string jobs: test: @@ -18,7 +21,7 @@ jobs: id: iso uses: actions/download-artifact@v3 with: - name: kairos-${{ inputs.flavor }}.iso.zip + name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release}}.iso.zip - name: Display structure of downloaded files run: ls -las . - name: Install deps @@ -31,7 +34,7 @@ jobs: cache-dependency-path: tests/go.sum - name: Ginkgo run: | - export ISO=$(ls $PWD/kairos-core-*${{ inputs.flavor }}*.iso) + export ISO=$(ls $PWD/*${{ inputs.flavor }}*.iso) export GOPATH="/Users/runner/go" export PATH=$PATH:$GOPATH/bin export CREATE_VM=true diff --git a/.github/workflows/reusable-provider-tests.yaml b/.github/workflows/reusable-provider-tests.yaml index 9459eb60b..bbc4a3072 100644 --- a/.github/workflows/reusable-provider-tests.yaml +++ b/.github/workflows/reusable-provider-tests.yaml @@ -6,6 +6,9 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string label: required: true type: string @@ -54,7 +57,7 @@ jobs: - name: Download artifacts uses: actions/download-artifact@v3 with: - name: kairos-${{ inputs.flavor }}-provider.iso.zip + name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}-provider.iso.zip - name: Run tests env: USE_QEMU: true @@ -62,7 +65,7 @@ jobs: MEMORY: 4000 CPUS: 2 DRIVE_SIZE: 30000 - CONTAINER_IMAGE: ttl.sh/kairos-${{ inputs.flavor }}-${{ github.sha }}-provider:24h + CONTAINER_IMAGE: ttl.sh/kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}-${{ github.sha }}-provider:24h run: | ls *.iso export ISO=$PWD/$(ls *.iso) @@ -72,6 +75,6 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: ${{ inputs.flavor }}-${{ inputs.label }}-provider-test.logs.zip + name: ${{ inputs.flavor }}-${{ inputs.flavor_release }}-${{ inputs.label }}-provider-test.logs.zip path: tests/**/logs/* if-no-files-found: warn diff --git a/.github/workflows/reusable-provider-upgrade-latest-test.yaml b/.github/workflows/reusable-provider-upgrade-latest-test.yaml index 7baf23ba2..954a44539 100644 --- a/.github/workflows/reusable-provider-upgrade-latest-test.yaml +++ b/.github/workflows/reusable-provider-upgrade-latest-test.yaml @@ -6,6 +6,9 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string jobs: test: @@ -50,7 +53,7 @@ jobs: with: latest: true repository: "kairos-io/kairos" - fileName: 'kairos-standard-*${{ inputs.flavor }}-*k3s1.iso' + fileName: 'kairos-standard-*${{ inputs.flavor }}-${{ inputs.flavor_release }}.*k3s1.iso' out-file-path: "" - name: Run tests run: | @@ -59,7 +62,7 @@ jobs: export MEMORY=4000 export CPUS=2 export DRIVE_SIZE=30000 - export CONTAINER_IMAGE=ttl.sh/kairos-${{ inputs.flavor }}-${{ github.sha }}-provider:24h + export CONTAINER_IMAGE=ttl.sh/kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}-${{ github.sha }}-provider:24h K3S_VERSION=$(sudo luet --config framework-profile.yaml search -o json k8s/k3s | jq -r '.packages | map(.version) | unique | last' | cut -d+ -f1) diff --git a/.github/workflows/reusable-qemu-acceptance-test.yaml b/.github/workflows/reusable-qemu-acceptance-test.yaml index 7fc8fbcf3..75ecf1780 100644 --- a/.github/workflows/reusable-qemu-acceptance-test.yaml +++ b/.github/workflows/reusable-qemu-acceptance-test.yaml @@ -6,6 +6,9 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string port: required: false type: string @@ -53,7 +56,7 @@ jobs: id: iso uses: actions/download-artifact@v3 with: - name: kairos-${{ inputs.flavor }}.iso.zip + name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}.iso.zip - name: Display structure of downloaded files run: ls -las . - name: Install earthly @@ -66,4 +69,4 @@ jobs: sudo iptables -I INPUT -s 169.254.169.254 -j DROP sudo iptables -I OUTPUT -d 169.254.169.254 -j DROP - run: | - earthly +run-qemu-datasource-tests --PREBUILT_ISO=$(ls kairos-core-*${{ inputs.flavor }}*.iso) --FLAVOR=${{ inputs.flavor }} --SSH_PORT=${{ inputs.port }} + earthly +run-qemu-datasource-tests --PREBUILT_ISO=$(ls *${{ inputs.flavor }}*.iso) --FLAVOR=${{ inputs.flavor }} --SSH_PORT=${{ inputs.port }} diff --git a/.github/workflows/reusable-qemu-bundles-test.yaml b/.github/workflows/reusable-qemu-bundles-test.yaml index e6bd32352..c630772fb 100644 --- a/.github/workflows/reusable-qemu-bundles-test.yaml +++ b/.github/workflows/reusable-qemu-bundles-test.yaml @@ -6,6 +6,9 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string jobs: test: @@ -18,7 +21,7 @@ jobs: id: iso uses: actions/download-artifact@v3 with: - name: kairos-${{ inputs.flavor }}.iso.zip + name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}.iso.zip - name: Display structure of downloaded files run: ls -las . - name: Install earthly @@ -47,4 +50,4 @@ jobs: EOF earthly -P +prepare-bundles-tests - earthly -P +run-qemu-bundles-tests --PREBUILT_ISO=$(ls kairos-core-*${{ inputs.flavor }}*.iso) --FLAVOR=${{ inputs.flavor }} + earthly -P +run-qemu-bundles-tests --PREBUILT_ISO=$(ls *${{ inputs.flavor }}*.iso) --FLAVOR=${{ inputs.flavor }} diff --git a/.github/workflows/reusable-qemu-netboot-test.yaml b/.github/workflows/reusable-qemu-netboot-test.yaml index d4c42755b..22ea8f184 100644 --- a/.github/workflows/reusable-qemu-netboot-test.yaml +++ b/.github/workflows/reusable-qemu-netboot-test.yaml @@ -6,6 +6,21 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string + family: + required: true + type: string + base_image: + required: true + type: string + model: + required: true + type: string + variant: + required: true + type: string jobs: test: @@ -56,4 +71,11 @@ jobs: sudo iptables -I INPUT -s 169.254.169.254 -j DROP sudo iptables -I OUTPUT -d 169.254.169.254 -j DROP - run: | - earthly -P +run-qemu-netboot-test --TEST_SUITE=netboot-test --FLAVOR=${{ inputs.flavor }} + earthly -P +run-qemu-netboot-test \ + --TEST_SUITE=netboot-test \ + --FLAVOR=${{ inputs.flavor }} \ + --FLAVOR_RELEASE=${{ inputs.flavor_release }} \ + --FAMILY=${{ inputs.family }} \ + --BASE_IMAGE=${{ inputs.base_image}} \ + --MODEL=${{ inputs.model }} \ + --VARIANT=${{ inputs.variant }} diff --git a/.github/workflows/reusable-qemu-reset-test.yaml b/.github/workflows/reusable-qemu-reset-test.yaml index fd9f4d5b1..613985e5a 100644 --- a/.github/workflows/reusable-qemu-reset-test.yaml +++ b/.github/workflows/reusable-qemu-reset-test.yaml @@ -6,6 +6,9 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string jobs: test: @@ -18,7 +21,7 @@ jobs: id: iso uses: actions/download-artifact@v3 with: - name: kairos-${{ inputs.flavor }}.iso.zip + name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}.iso.zip - name: Display structure of downloaded files run: ls -las . - name: Install earthly @@ -45,4 +48,4 @@ jobs: insecure = true http = true EOF - earthly -P +run-qemu-datasource-tests --PREBUILT_ISO=$(ls kairos-core-*${{ inputs.flavor }}*.iso) --TEST_SUITE=reset-test --FLAVOR=${{ inputs.flavor }} + earthly -P +run-qemu-datasource-tests --PREBUILT_ISO=$(ls *${{ inputs.flavor }}*.iso) --TEST_SUITE=reset-test --FLAVOR=${{ inputs.flavor }} diff --git a/.github/workflows/reusable-upgrade-latest-test.yaml b/.github/workflows/reusable-upgrade-latest-test.yaml index 667c05fd8..b1b4b158c 100644 --- a/.github/workflows/reusable-upgrade-latest-test.yaml +++ b/.github/workflows/reusable-upgrade-latest-test.yaml @@ -6,6 +6,9 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string jobs: test: @@ -51,6 +54,7 @@ jobs: # A flag to set the download target as latest release # The default value is 'false' latest: true + # TODO: Until we publish with the new names, what should this be? fileName: 'kairos-core-${{ inputs.flavor }}-amd64-generic-v*.iso' out-file-path: "" - name: Display structure of downloaded files @@ -67,10 +71,11 @@ jobs: - run: | # release-downloader globing matches more than one iso. Make sure # we use the right one. + # TODO: This needs to be adapted after we release with the new artifact names ISO=$(ls kairos-core-${{ inputs.flavor }}-amd64-generic-v*.iso | grep -v ipxe) earthly +run-qemu-test --PREBUILT_ISO=$ISO \ --FLAVOR=${{ inputs.flavor }} \ - --CONTAINER_IMAGE=ttl.sh/kairos-${{ inputs.flavor }}-${{ github.sha }}:24h \ + --CONTAINER_IMAGE=ttl.sh/kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}-${{ github.sha }}:24h \ --TEST_SUITE=upgrade-latest-with-cli - uses: actions/upload-artifact@v3 if: failure() diff --git a/.github/workflows/reusable-upgrade-with-cli-test.yaml b/.github/workflows/reusable-upgrade-with-cli-test.yaml index 8c27f1208..66310c1d6 100644 --- a/.github/workflows/reusable-upgrade-with-cli-test.yaml +++ b/.github/workflows/reusable-upgrade-with-cli-test.yaml @@ -6,6 +6,9 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string jobs: test: @@ -55,7 +58,7 @@ jobs: id: iso uses: actions/download-artifact@v3 with: - name: kairos-${{ inputs.flavor }}.iso.zip + name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}.iso.zip - name: Display structure of downloaded files run: ls -las . - name: Block all traffic to metadata ip # For cloud runners, the metadata ip can interact with our test machines @@ -63,9 +66,9 @@ jobs: sudo iptables -I INPUT -s 169.254.169.254 -j DROP sudo iptables -I OUTPUT -d 169.254.169.254 -j DROP - run: | - earthly +run-qemu-test --PREBUILT_ISO=$(ls kairos-core-*${{ inputs.flavor }}*.iso) \ + earthly +run-qemu-test --PREBUILT_ISO=$(ls *${{ inputs.flavor }}*.iso) \ --FLAVOR=${{ inputs.flavor }} \ - --CONTAINER_IMAGE=ttl.sh/kairos-${{ inputs.flavor }}-${{ github.sha }}:24h \ + --CONTAINER_IMAGE=ttl.sh/kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}-${{ github.sha }}:24h \ --TEST_SUITE=upgrade-with-cli - uses: actions/upload-artifact@v3 if: failure() diff --git a/.github/workflows/reusable-zfs-test.yaml b/.github/workflows/reusable-zfs-test.yaml index 7c114b6e8..149c40fd4 100644 --- a/.github/workflows/reusable-zfs-test.yaml +++ b/.github/workflows/reusable-zfs-test.yaml @@ -6,6 +6,9 @@ on: flavor: required: true type: string + flavor_release: + required: true + type: string jobs: test: @@ -16,7 +19,7 @@ jobs: id: iso uses: actions/download-artifact@v3 with: - name: kairos-${{ inputs.flavor }}.iso.zip + name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release }}.iso.zip - name: Display structure of downloaded files run: ls -R - name: Install earthly @@ -29,4 +32,4 @@ jobs: sudo iptables -I INPUT -s 169.254.169.254 -j DROP sudo iptables -I OUTPUT -d 169.254.169.254 -j DROP - run: | - earthly +run-qemu-test --PREBUILT_ISO=$(ls kairos-core-*${{ inputs.flavor }}*.iso) --TEST_SUITE=zfs --FLAVOR=${{ inputs.flavor }} + earthly +run-qemu-test --PREBUILT_ISO=$(ls *${{ inputs.flavor }}*.iso) --TEST_SUITE=zfs --FLAVOR=${{ inputs.flavor }} diff --git a/.github/workflows/uki.yaml b/.github/workflows/uki.yaml index ee3f089d3..59958f0a1 100644 --- a/.github/workflows/uki.yaml +++ b/.github/workflows/uki.yaml @@ -40,7 +40,17 @@ jobs: # Do fedora as its the smaller uki possible # Use immmucore master as it has patches not released for uki # Use kairos-agent main branch as it has patches not released for uki - earthly +iso-uki --FLAVOR=opensuse-leap --KAIROS_AGENT_DEV=true --KAIROS_AGENT_DEV_BRANCH=main --IMMUCORE_DEV=true --IMMUCORE_DEV_BRANCH=master + earthly +iso-uki \ + --FLAVOR=opensuse \ + --FLAVOR_RELEASE=leap-15.5 \ + --FAMILY=opensuse-leap \ + --MODEL=generic \ + --VARIANT=core \ + --BASE_IMAGE=opensuse/leap:15.5 \ + --KAIROS_AGENT_DEV=true \ + --KAIROS_AGENT_DEV_BRANCH=main \ + --IMMUCORE_DEV=true \ + --IMMUCORE_DEV_BRANCH=master - name: Run tests env: USE_QEMU: true @@ -49,7 +59,7 @@ jobs: CPUS: 2 FIRMWARE: /usr/share/OVMF/OVMF_CODE.fd run: | - export ISO=$(ls $PWD/build/kairos-core-*opensuse-leap*.iso) + export ISO=$(ls $PWD/build/kairos-*opensuse-leap*.iso) cp tests/go.* . go run github.com/onsi/ginkgo/v2/ginkgo -v --label-filter "uki" --fail-fast -r ./tests/ - uses: actions/upload-artifact@v3 diff --git a/Earthfile b/Earthfile index b12618bf6..3805a4c11 100644 --- a/Earthfile +++ b/Earthfile @@ -1,15 +1,9 @@ VERSION 0.6 FROM alpine -ARG VARIANT=core # core, lite, framework -ARG FLAVOR=opensuse-leap -ARG BASE_URL=quay.io/kairos +ARG REGISTRY_AND_ORG=quay.io/kairos ARG IMAGE -ARG MODEL=generic ARG SUPPORT=official # not using until this is defined in https://github.com/kairos-io/kairos/issues/1527 ARG GITHUB_REPO=kairos-io/kairos -ARG OS_ID=kairos -ARG OS_REPO=${BASE_URL}/${VARIANT}-${FLAVOR} -ARG OS_NAME=${OS_ID}-${VARIANT}-${FLAVOR} # renovate: datasource=docker depName=quay.io/luet/base ARG LUET_VERSION=0.35.0 # renovate: datasource=docker depName=aquasec/trivy @@ -43,6 +37,15 @@ ARG K3S_VERSION all: ARG SECURITY_SCANS=true + + ARG TARGETARCH + ARG --required FAMILY # The dockerfile to use + ARG --required FLAVOR # The distribution E.g. "ubuntu" + ARG --required FLAVOR_RELEASE # The distribution release/version E.g. "20.04" + ARG --required VARIANT + ARG --required MODEL + ARG --required BASE_IMAGE # BASE_IMAGE is the image to apply the strategy (aka FLAVOR) on. E.g. ubuntu:20.04 + BUILD +base-image IF [ "$SECURITY_SCANS" = "true" ] BUILD +image-sbom @@ -56,6 +59,14 @@ all: # For PR building, only image and iso are needed ci: ARG SECURITY_SCANS=true + + # args for base-image target + ARG --required FLAVOR + ARG --required FLAVOR_RELEASE + ARG --required BASE_IMAGE + ARG --required MODEL + ARG --required VARIANT + BUILD +base-image IF [ "$SECURITY_SCANS" = "true" ] BUILD +image-sbom @@ -66,26 +77,24 @@ ci: all-arm: ARG SECURITY_SCANS=true - ARG MODEL=rpi4 - BUILD --platform=linux/arm64 +base-image --MODEL=$MODEL + + BUILD --platform=linux/arm64 +base-image IF [ "$SECURITY_SCANS" = "true" ] - BUILD --platform=linux/arm64 +image-sbom --MODEL=$MODEL - BUILD --platform=linux/arm64 +trivy-scan --MODEL=$MODEL - BUILD --platform=linux/arm64 +grype-scan --MODEL=$MODEL + BUILD --platform=linux/arm64 +image-sbom + BUILD --platform=linux/arm64 +trivy-scan + BUILD --platform=linux/arm64 +grype-scan END - + IF [[ "$FLAVOR" = "ubuntu-20-lts-arm-nvidia-jetson-agx-orin" ]] - BUILD +prepare-arm-image --MODEL=$MODEL --FLAVOR=${FLAVOR} + BUILD +prepare-arm-image ELSE - BUILD +arm-image --MODEL=$MODEL + BUILD +arm-image END arm-container-image: - ARG MODEL - BUILD --platform=linux/arm64 +base-image --MODEL=$MODEL + BUILD --platform=linux/arm64 +base-image all-arm-generic: - BUILD --platform=linux/arm64 +base-image --MODEL=generic BUILD --platform=linux/arm64 +iso --MODEL=generic build-and-push-golang-testing: @@ -106,42 +115,30 @@ go-deps-test: SAVE ARTIFACT go.mod go.mod AS LOCAL go.mod SAVE ARTIFACT go.sum go.sum AS LOCAL go.sum -CONTAINER_IMAGE_VERSION: - COMMAND - - ARG VERSION - - # quay.io doesn't accept "+" in the repo name - ARG _VERSION=$(echo $VERSION | sed 's/+/-/') - - IF [ "$IMAGE" = "" ] - # TODO: This IF block should be deleted as soon as our repository names - # follow our conventions. - IF [ "$VARIANT" = "standard" ] - RUN echo ${BASE_URL}/kairos-${FLAVOR}:${_VERSION} > IMAGE - ELSE - RUN echo ${BASE_URL}/${VARIANT}-${FLAVOR}:${_VERSION} > IMAGE - END - ELSE - RUN echo $IMAGE > IMAGE - END - - RUN echo "${VERSION}" > VERSION OSRELEASE: COMMAND - ARG OS_ID - ARG OS_NAME - ARG OS_REPO - ARG OS_VERSION - ARG VARIANT - ARG FLAVOR ARG GITHUB_REPO ARG BUG_REPORT_URL ARG HOME_URL - COPY +version/VERSION ./ - ARG OS_LABEL=$(cat VERSION) + ARG OS_ID=kairos + + # For naming.sh + ARG TARGETARCH # Earthly built-in (not passed) + ARG --required FAMILY + ARG --required FLAVOR + ARG --required FLAVOR_RELEASE + ARG --required VARIANT + ARG --required MODEL + ARG --required KAIROS_VERSION + + COPY ./naming.sh . + ARG OS_REPO=$(./naming.sh container_artifact_name) + ARG OS_NAME=$(./naming.sh common_artifact_name) + + ARG OS_VERSION=$KAIROS_VERSION + ARG OS_LABEL=$KAIROS_VERSION # update OS-release file RUN sed -i -n '/KAIROS_/!p' /etc/os-release @@ -217,20 +214,23 @@ syft: image-sbom: ARG TARGETARCH + ARG --required FAMILY # The dockerfile to use + ARG --required FLAVOR # The distribution E.g. "ubuntu" + ARG --required FLAVOR_RELEASE # The distribution release/version E.g. "20.04" + ARG --required VARIANT + ARG --required MODEL + ARG --required BASE_IMAGE # BASE_IMAGE is the image to apply the strategy (aka FLAVOR) on. E.g. ubuntu:20.04 + # Use base-image so it can read original os-release file FROM +base-image WORKDIR /build ARG FLAVOR ARG VARIANT COPY +version/VERSION ./ - ARG VERSION=$(cat VERSION) + ARG KAIROS_VERSION=$(cat VERSION) - IF [ "$TARGETARCH" = "arm64" ] - ARG DISTRO=$(echo $FLAVOR | sed 's/-arm-.*//') - ARG ISO_NAME=${OS_ID}-${VARIANT}-${DISTRO}-${TARGETARCH}-${MODEL}-${VERSION} - ELSE - ARG ISO_NAME=${OS_ID}-${VARIANT}-${FLAVOR}-${TARGETARCH}-${MODEL}-${VERSION} - END + COPY ./naming.sh . + ARG ISO_NAME=$(./naming.sh bootable_artifact_name) COPY +syft/syft /usr/bin/syft RUN syft / -o json=sbom.syft.json -o spdx-json=sbom.spdx.json @@ -252,6 +252,8 @@ framework: FROM golang:alpine ARG FLAVOR + ARG FLAVOR_RELEASE + WORKDIR /build COPY ./profile-build /build @@ -259,7 +261,7 @@ framework: RUN go mod download COPY framework-profile.yaml /build - RUN go run main.go ${FLAVOR} framework-profile.yaml /framework + RUN go run main.go ${FLAVOR}-${FLAVOR_RELEASE} framework-profile.yaml /framework RUN mkdir -p /framework/etc/kairos/ RUN luet database --system-target /framework get-all-installed --output /framework/etc/kairos/versions.yaml @@ -278,7 +280,9 @@ framework: build-framework-image: FROM alpine - ARG FLAVOR + + ARG --required FLAVOR + ARG --required FLAVOR_RELEASE # Just in case, make sure this is valid IF [ "$FLAVOR" = "" ] @@ -295,45 +299,33 @@ build-framework-image: FROM scratch - COPY (+framework/framework --FLAVOR=$FLAVOR) / + COPY (+framework/framework --FLAVOR=$FLAVOR --FLAVOR_RELEASE=$FLAVOR_RELEASE) / SAVE IMAGE --push $IMAGE_REPOSITORY_ORG/framework:${VERSION}_${FLAVOR} base-image: - ARG MODEL - ARG FLAVOR - ARG VARIANT - ARG KAIROS_VERSION + ARG TARGETARCH # Earthly built-in (not passed) + ARG --required FAMILY # The dockerfile to use + ARG --required FLAVOR # The distribution E.g. "ubuntu" + ARG --required FLAVOR_RELEASE # The distribution release/version E.g. "20.04" + ARG --required VARIANT + ARG --required MODEL + ARG --required BASE_IMAGE # BASE_IMAGE is the image to apply the strategy (aka FLAVOR) on. E.g. ubuntu:20.04 + ARG BUILD_INITRD="true" - ARG TARGETARCH # HWE is used to determine if the HWE kernel should be installed on Ubuntu LTS. # The default value is empty, which means the HWE kernel WILL be installed # if you want to disable the HWE kernel, set HWE to "-non-hwe" ARG HWE - IF [ "$BASE_IMAGE" = "" ] - # DISTRO is used to match the Linux distribution in the Dockerfile e.g. Dockerfile.ubuntu - # This is a bit messy at the moment, but it will be sorted out when we stop including the model and the arch in - # the flavor name e.g. ubuntu-20-lts-arm-rpi - IF [[ "$FLAVOR" =~ ^alpine* ]] # TODO: only needed while we don't have a pure alpine - ARG DISTRO=alpine - ELSE IF [[ "$FLAVOR" = "ubuntu-20-lts-arm-nvidia-jetson-agx-orin" ]] # TODO: needs to still be merged on Dockerfile.ubuntu (or not?) - ARG DISTRO=ubuntu-20-lts-arm-nvidia-jetson-agx-orin - ELSE IF [[ "$FLAVOR" =~ "ubuntu" ]] # TODO: need to find a better way to match the flavor and the distro in the dockerfile for Ubuntu - ARG DISTRO=ubuntu - ELSE - ARG DISTRO=$(echo $FLAVOR | sed 's/-arm-.*//') - END - - # SIMPLE_FLAVOR is used to distinguish the flavor inside the Dockerfile, where it's important to make a distinction - # between e.g. ubuntu and ubuntu-20-lts, but we don't really need to know the model and the arch since this is - # defined using MODEL and TARGETARCH. - ARG SIMPLE_FLAVOR=$(echo $FLAVOR | sed 's/-arm-.*//') - - FROM DOCKERFILE --build-arg MODEL=$MODEL --build-arg FLAVOR=$SIMPLE_FLAVOR --build-arg HWE=$HWE -f images/Dockerfile.$DISTRO images/ - ELSE - FROM $BASE_IMAGE - END + # TODO: Pass the VARIANT here and let Dockerfiles handle it + FROM DOCKERFILE \ + --build-arg BASE_IMAGE=$BASE_IMAGE \ + --build-arg MODEL=$MODEL \ + --build-arg FLAVOR=$FLAVOR \ + --build-arg FLAVOR_RELEASE=$FLAVOR_RELEASE \ + --build-arg HWE=$HWE \ + -f images/Dockerfile.$FAMILY images/ # Includes overlay/files # We only support non-fips for now, they are built separatedly and pushed for other to consume, not used in our workflows @@ -344,15 +336,23 @@ base-image: # Set proper os-release file with all the info IF [ "$KAIROS_VERSION" = "" ] COPY +version/VERSION ./ - ARG VERSION=$(cat VERSION) - RUN echo "version ${VERSION}" - ARG OS_VERSION=${VERSION} - RUN rm VERSION + ARG KAIROS_VERSION=$(cat VERSION) + ARG OS_VERSION=${KAIROS_VERSION} ELSE ARG OS_VERSION=${KAIROS_VERSION} + RUN echo $OS_VERSION > ./VERSION END - DO +OSRELEASE --HOME_URL=https://github.com/kairos-io/kairos --BUG_REPORT_URL=https://github.com/kairos-io/kairos/issues --GITHUB_REPO=kairos-io/kairos --VARIANT=${VARIANT} --FLAVOR=${FLAVOR} --OS_ID=${OS_ID} --OS_LABEL=${OS_LABEL} --OS_NAME=${OS_NAME} --OS_REPO=${OS_REPO} --OS_VERSION=${OS_VERSION} + DO +OSRELEASE \ + --HOME_URL=https://github.com/kairos-io/kairos \ + --BUG_REPORT_URL=https://github.com/kairos-io/kairos/issues \ + --GITHUB_REPO=kairos-io/kairos \ + --KAIROS_VERSION=${OS_VERSION} \ + --FAMILY=${FAMILY} \ + --FLAVOR=${FLAVOR} \ + --FLAVOR_RELEASE=${FLAVOR_RELEASE} \ + --VARIANT=${VARIANT} \ + --MODEL=${MODEL} # Fully remove machine-id, it will be generated on first boot RUN rm -rf /etc/machine-id @@ -398,16 +398,19 @@ base-image: # https://docs.earthly.dev/docs/earthfile#arg # Overrides GITHUB_REPO, VARIANT, OS_REPO and OS_NAME to add the kairos name in there # which points to the provider-kairos repo - DO +OSRELEASE --HOME_URL=https://github.com/kairos-io/kairos \ - --BUG_REPORT_URL=https://github.com/kairos-io/kairos/issues \ - --GITHUB_REPO=kairos-io/provider-kairos --VARIANT=kairos \ - --OS_REPO=quay.io/kairos/kairos-${FLAVOR} \ - --FLAVOR=${FLAVOR} --OS_ID=${OS_ID} --OS_LABEL=${OS_LABEL} \ - --OS_NAME=${OS_ID}-kairos-${FLAVOR} \ - --OS_VERSION=${OS_VERSION} + DO +OSRELEASE \ + --HOME_URL=https://github.com/kairos-io/kairos \ + --BUG_REPORT_URL=https://github.com/kairos-io/kairos/issues \ + --GITHUB_REPO=kairos-io/provider-kairos \ + --KAIROS_VERSION=${OS_VERSION} \ + --FAMILY=${FAMILY} \ + --FLAVOR=${FLAVOR} \ + --FLAVOR_RELEASE=${FLAVOR_RELEASE} \ + --VARIANT=${VARIANT} \ + --MODEL=${MODEL} END - IF [[ "$FLAVOR" =~ ^ubuntu* ]] + IF expr "$FLAVOR" : '^ubuntu' > /dev/null # compress firmware RUN find /usr/lib/firmware -type f -execdir zstd --rm -9 {} \+ # compress modules @@ -419,7 +422,6 @@ base-image: RUN rm -rf /boot/initrd.img-* END - RUN --no-cache kernel=$(ls /lib/modules | head -n1) && depmod -a "${kernel}" IF [ -f "/usr/bin/dracut" ] @@ -467,23 +469,39 @@ base-image: RUN rm -rf /tmp/* - DO +CONTAINER_IMAGE_VERSION -VERSION=${OS_VERSION} - ARG _CIMG=$(cat IMAGE) + COPY ./naming.sh . + RUN ./naming.sh container_artifact_name + RUN ./naming.sh container_artifact_name > ./IMAGE # luet cleanup RUN luet cleanup RUN rm -rf /var/luet + ARG _CIMG=$(cat ./IMAGE) SAVE IMAGE $_CIMG SAVE ARTIFACT IMAGE AS LOCAL build/IMAGE SAVE ARTIFACT VERSION AS LOCAL build/VERSION SAVE ARTIFACT /etc/kairos/versions.yaml versions.yaml AS LOCAL build/versions.yaml image-rootfs: + ARG --required FAMILY + ARG --required FLAVOR + ARG --required BASE_IMAGE + ARG --required MODEL + ARG --required VARIANT + FROM +base-image + SAVE ARTIFACT --keep-own /. rootfs uki-artifacts: + ARG --required FAMILY # The dockerfile to use + ARG --required FLAVOR # The distribution E.g. "ubuntu" + ARG --required FLAVOR_RELEASE # The distribution release/version E.g. "20.04" + ARG --required VARIANT + ARG --required MODEL + ARG --required BASE_IMAGE + FROM +base-image --BUILD_INITRD=false RUN /usr/bin/immucore version RUN ln -s /usr/bin/immucore /init @@ -522,11 +540,22 @@ uki-tools-image: # This is for easy testing SecureBoot locally for development purposes # Installing this keys in other place than a VM for testing SecureBoot is irresponsible uki: + FROM ubuntu + ARG TARGETARCH COPY +version/VERSION ./ RUN echo "version ${VERSION}" - ARG VERSION=$(cat VERSION) - ARG ISO_NAME=${OS_ID}-${VARIANT}-${FLAVOR}-${TARGETARCH}-${MODEL}-${VERSION} + + ARG --required FAMILY # The dockerfile to use + ARG --required FLAVOR # The distribution E.g. "ubuntu" + ARG --required FLAVOR_RELEASE # The distribution release/version E.g. "20.04" + ARG --required VARIANT + ARG --required MODEL + ARG --required BASE_IMAGE + + ARG KAIROS_VERSION=$(cat VERSION) + COPY ./naming.sh . + ARG ISO_NAME=$(./naming.sh bootable_artifact_name) FROM +uki-tools-image WORKDIR build COPY +uki-artifacts/Kernel Kernel @@ -589,10 +618,23 @@ uki-local-artifacts: ### iso: + FROM ubuntu + COPY +version/VERSION ./ - ARG VERSION=$(cat VERSION) + ARG KAIROS_VERSION=$(cat VERSION) ARG TARGETARCH - ARG ISO_NAME=${OS_ID}-${VARIANT}-${FLAVOR}-${TARGETARCH}-${MODEL}-${VERSION} + + # args for base-image target + ARG --required FAMILY + ARG --required FLAVOR + ARG --required FLAVOR_RELEASE + ARG --required BASE_IMAGE + ARG --required MODEL + ARG --required VARIANT + + COPY ./naming.sh . + ARG ISO_NAME=$(./naming.sh bootable_artifact_name) + ARG OSBUILDER_IMAGE FROM $OSBUILDER_IMAGE WORKDIR /build @@ -604,10 +646,21 @@ iso: iso-uki: + FROM ubuntu + COPY +version/VERSION ./ - ARG VERSION=$(cat VERSION) + ARG KAIROS_VERSION=$(cat VERSION) ARG TARGETARCH - ARG ISO_NAME=${OS_ID}-${VARIANT}-${FLAVOR}-${TARGETARCH}-${MODEL}-${VERSION} + + ARG --required FAMILY # The dockerfile to use + ARG --required FLAVOR # The distribution E.g. "ubuntu" + ARG --required FLAVOR_RELEASE # The distribution release/version E.g. "20.04" + ARG --required VARIANT + ARG --required MODEL + ARG --required BASE_IMAGE + + COPY ./naming.sh . + ARG ISO_NAME=$(./naming.sh bootable_artifact_name) ARG OSBUILDER_IMAGE FROM $OSBUILDER_IMAGE WORKDIR /build @@ -652,12 +705,23 @@ iso-uki: # you can override either the full thing by setting --REMOTE_IMG=docker:REPO/IMAGE:TAG # or by --REMOTE_IMG=REPO/IMAGE:TAG iso-remote: + FROM ubuntu + ARG TARGETARCH ARG REMOTE_IMG COPY +version/VERSION ./ - ARG VERSION=$(cat VERSION) - ARG ISO_NAME=${OS_ID}-${VARIANT}-${FLAVOR}-${TARGETARCH}-${MODEL}-${VERSION} + ARG KAIROS_VERSION=$(cat VERSION) + + ARG --required FAMILY # The dockerfile to use + ARG --required FLAVOR # The distribution E.g. "ubuntu" + ARG --required FLAVOR_RELEASE # The distribution release/version E.g. "20.04" + ARG --required VARIANT + ARG --required MODEL + ARG --required BASE_IMAGE + + COPY ./naming.sh . + ARG ISO_NAME=$(./naming.sh bootable_artifact_name) ARG OSBUILDER_IMAGE FROM $OSBUILDER_IMAGE WORKDIR /build @@ -667,12 +731,25 @@ iso-remote: SAVE ARTIFACT /build/$ISO_NAME.iso.sha256 kairos.iso.sha256 AS LOCAL build/$ISO_NAME.iso.sha256 netboot: - ARG TARGETARCH + FROM ubuntu + COPY +version/VERSION ./ RUN echo "version ${VERSION}" - ARG VERSION=$(cat VERSION) - ARG ISO_NAME=${OS_ID}-${VARIANT}-${FLAVOR}-${TARGETARCH}-${MODEL}-${VERSION} + ARG KAIROS_VERSION=$(cat VERSION) + + ARG TARGETARCH # Earthly built-in (not passed) + ARG --required FAMILY # The dockerfile to use + ARG --required FLAVOR # The distribution E.g. "ubuntu" + ARG --required FLAVOR_RELEASE # The distribution release/version E.g. "20.04" + ARG --required VARIANT + ARG --required MODEL + ARG --required BASE_IMAGE # BASE_IMAGE is the image to apply the strategy (aka FLAVOR) on. E.g. ubuntu:20.04 + + COPY ./naming.sh . + ARG ISO_NAME=$(./naming.sh bootable_artifact_name) ARG OSBUILDER_IMAGE + + # Used here: https://github.com/kairos-io/osbuilder/blob/66e9e7a9403a413e310f462136b70d715605ab09/tools-image/ipxe.tmpl#L5 ARG RELEASE_URL=https://github.com/kairos-io/kairos/releases/download FROM $OSBUILDER_IMAGE @@ -695,14 +772,21 @@ arm-image: ARG COMPRESS_IMG=true ARG IMG_COMPRESSION=xz FROM $OSBUILDER_IMAGE - ARG MODEL=rpi4 + COPY +version/VERSION ./ RUN echo "version ${VERSION}" - ARG VERSION=$(cat VERSION) - ARG DISTRO=$(echo $FLAVOR | sed 's/-arm-.*//') - # TARGETARCH is not used here because OSBUILDER_IMAGE is not available in arm64. When this changes, then the caller - # of this target can simply pass the desired TARGETARCH. - ARG IMAGE_NAME=${OS_ID}-${VARIANT}-${DISTRO}-arm64-${MODEL}-${VERSION}.img + ARG KAIROS_VERSION=$(cat VERSION) + + ARG TARGETARCH + ARG --required FAMILY + ARG --required FLAVOR + ARG --required FLAVOR_RELEASE + ARG --required VARIANT + ARG --required MODEL + ARG --required BASE_IMAGE + + COPY ./naming.sh . + ARG IMAGE_NAME=$(./naming.sh bootable_artifact_name).img RUN echo $IMAGE_NAME WORKDIR /build # These sizes are in MB @@ -743,11 +827,24 @@ prepare-arm-image: ARG OSBUILDER_IMAGE ARG COMPRESS_IMG=true FROM $OSBUILDER_IMAGE + + COPY +version/VERSION ./ + RUN echo "version ${VERSION}" + ARG KAIROS_VERSION=$(cat VERSION) + + ARG TARGETARCH + ARG --required FAMILY + ARG --required FLAVOR + ARG --required FLAVOR_RELEASE + ARG --required VARIANT + ARG --required BASE_IMAGE ARG MODEL=rpi4 - ARG IMAGE_NAME=${FLAVOR}.img + + COPY ./naming.sh . + ARG IMAGE_NAME=$(./naming.sh bootable_artifact_name).img WORKDIR /build # These sizes are in MB - + ENV SIZE="15200" IF [[ "$FLAVOR" = "ubuntu-20-lts-arm-nvidia-jetson-agx-orin" ]] @@ -787,18 +884,28 @@ ipxe-iso: mtools syslinux isolinux gcc-arm-none-eabi git make gcc liblzma-dev mkisofs xorriso # jq docker WORKDIR /build + COPY +version/VERSION ./ - RUN echo "version ${VERSION}" - ARG VERSION=$(cat VERSION) - ARG ISO_NAME=${OS_ID}-${VARIANT}-${FLAVOR}-${TARGETARCH}-${MODEL}-${VERSION} - COPY +version/VERSION ./ - ARG VERSION=$(cat VERSION) + ARG KAIROS_VERSION=$(cat VERSION) + ARG TARGETARCH + + # args for base-image target + ARG --required FAMILY + ARG --required FLAVOR + ARG --required FLAVOR_RELEASE + ARG --required BASE_IMAGE + ARG --required MODEL + ARG --required VARIANT + + COPY ./naming.sh . + ARG ISO_NAME=$(./naming.sh bootable_artifact_name) + + # Used here: https://github.com/kairos-io/osbuilder/blob/66e9e7a9403a413e310f462136b70d715605ab09/tools-image/ipxe.tmpl#L5 ARG RELEASE_URL - RUN echo "version ${VERSION}" RUN git clone https://github.com/ipxe/ipxe IF [ "$ipxe_script" = "" ] - COPY (+netboot/ipxe --VERSION=$VERSION --RELEASE_URL=$RELEASE_URL) /build/ipxe/script.ipxe + COPY (+netboot/ipxe --VERSION=$KAIROS_VERSION --RELEASE_URL=$RELEASE_URL) /build/ipxe/script.ipxe ELSE COPY $ipxe_script /build/ipxe/script.ipxe END @@ -813,11 +920,13 @@ ipxe-iso: # is to boot from them and do a reset to get the latest system installed # This allows us to build a raw disk image locally to test the cloud workflow easily raw-image: + FROM ubuntu ARG TARGETARCH COPY +version/VERSION ./ RUN echo "version ${VERSION}" ARG VERSION=$(cat VERSION) - ARG IMG_NAME=${OS_ID}-${VARIANT}-${FLAVOR}-${TARGETARCH}-${MODEL}-${VERSION}.raw + COPY ./naming.sh . + ARG IMG_NAME=$(./naming.sh bootable_artifact_name).raw ARG OSBUILDER_IMAGE FROM $OSBUILDER_IMAGE WORKDIR /build @@ -851,6 +960,7 @@ trivy: trivy-scan: ARG TARGETARCH + # Use base-image so it can read original os-release file FROM +base-image COPY +trivy/trivy /trivy @@ -859,12 +969,8 @@ trivy-scan: ARG VERSION=$(cat VERSION) ARG FLAVOR ARG VARIANT - IF [ "$TARGETARCH" = "arm64" ] - ARG DISTRO=$(echo $FLAVOR | sed 's/-arm-.*//') - ARG ISO_NAME=${OS_ID}-${VARIANT}-${DISTRO}-${TARGETARCH}-${MODEL}-${VERSION} - ELSE - ARG ISO_NAME=${OS_ID}-${VARIANT}-${FLAVOR}-${TARGETARCH}-${MODEL}-${VERSION} - END + ARG ISO_NAME=$(./naming.sh bootable_artifact_name) + WORKDIR /build RUN /trivy filesystem --skip-dirs /tmp --timeout 30m --format sarif -o report.sarif --no-progress / RUN /trivy filesystem --skip-dirs /tmp --timeout 30m --format template --template "@/contrib/html.tpl" -o report.html --no-progress / @@ -879,19 +985,14 @@ grype: grype-scan: ARG TARGETARCH + # Use base-image so it can read original os-release file FROM +base-image COPY +grype/grype /grype COPY +version/VERSION ./ - ARG VERSION=$(cat VERSION) - ARG FLAVOR - ARG VARIANT - IF [ "$TARGETARCH" = "arm64" ] - ARG DISTRO=$(echo $FLAVOR | sed 's/-arm-.*//') - ARG ISO_NAME=${OS_ID}-${VARIANT}-${DISTRO}-${TARGETARCH}-${MODEL}-${VERSION} - ELSE - ARG ISO_NAME=${OS_ID}-${VARIANT}-${FLAVOR}-${TARGETARCH}-${MODEL}-${VERSION} - END + ARG KAIROS_VERSION=$(cat VERSION) + ARG ISO_NAME=$(./naming.sh bootable_artifact_name) + WORKDIR /build RUN /grype dir:/ --output sarif --add-cpes-if-none --file report.sarif RUN /grype dir:/ --output json --add-cpes-if-none --file report.json @@ -929,7 +1030,7 @@ run-qemu-datasource-tests: IF [ ! -e /test/build/datasource.iso ] COPY ( +datasource-iso/iso.iso --CLOUD_CONFIG=$CLOUD_CONFIG) datasource.iso ENV DATASOURCE=/test/datasource.iso - ELSE + ELSE ENV DATASOURCE=/test/build/datasource.iso END ENV CLOUD_INIT=/tests/tests/$CLOUD_CONFIG @@ -939,26 +1040,32 @@ run-qemu-datasource-tests: run-qemu-netboot-test: - ARG TARGETARCH FROM +go-deps-test COPY . /test WORKDIR /test COPY +version/VERSION ./ - RUN echo "version ${VERSION}" - ARG VERSION=$(cat VERSION) - ARG ISO_NAME=${OS_ID}-${VARIANT}-${FLAVOR}-${TARGETARCH}-${MODEL}-${VERSION} - COPY +version/VERSION ./ - ARG VERSION=$(cat VERSION) + ARG KAIROS_VERSION=$(cat VERSION) + + ARG TARGETARCH # Earthly built-in (not passed) + ARG --required FAMILY # The dockerfile to use + ARG --required FLAVOR # The distribution E.g. "ubuntu" + ARG --required FLAVOR_RELEASE # The distribution release/version E.g. "20.04" + ARG --required VARIANT + ARG --required MODEL + ARG --required BASE_IMAGE # BASE_IMAGE is the image to apply the strategy (aka FLAVOR) on. E.g. ubuntu:20.04 + + COPY ./naming.sh . + ARG ISO_NAME=$(./naming.sh bootable_artifact_name) # This is the IP at which qemu vm can see the host ARG IP="10.0.2.2" - COPY (+netboot/squashfs --VERSION=$VERSION --RELEASE_URL=http://$IP) ./build/$VERSION/$ISO_NAME.squashfs - COPY (+netboot/kernel --VERSION=$VERSION --RELEASE_URL=http://$IP) ./build/$VERSION/$ISO_NAME-kernel - COPY (+netboot/initrd --VERSION=$VERSION --RELEASE_URL=http://$IP) ./build/$VERSION/$ISO_NAME-initrd - COPY (+netboot/ipxe --VERSION=$VERSION --RELEASE_URL=http://$IP) ./build/$VERSION/$ISO_NAME.ipxe - COPY (+ipxe-iso/iso --VERSION=$VERSION --RELEASE_URL=http://$IP) ./build/${ISO_NAME}-ipxe.iso + COPY (+netboot/squashfs --VERSION=$KAIROS_VERSION --RELEASE_URL=http://$IP) ./build/$VERSION/$ISO_NAME.squashfs + COPY (+netboot/kernel --VERSION=$KAIROS_VERSION --RELEASE_URL=http://$IP) ./build/$VERSION/$ISO_NAME-kernel + COPY (+netboot/initrd --VERSION=$KAIROS_VERSION --RELEASE_URL=http://$IP) ./build/$VERSION/$ISO_NAME-initrd + COPY (+netboot/ipxe --VERSION=$KAIROS_VERSION --RELEASE_URL=http://$IP) ./build/$VERSION/$ISO_NAME.ipxe + COPY (+ipxe-iso/iso --VERSION=$KAIROS_VERSION --RELEASE_URL=http://$IP) ./build/${ISO_NAME}-ipxe.iso ENV ISO=/test/build/$ISO_NAME-ipxe.iso @@ -978,12 +1085,9 @@ run-qemu-netboot-test: run-qemu-test: FROM +go-deps-test WORKDIR /test - ARG FLAVOR ARG TEST_SUITE=upgrade-with-cli ARG PREBUILT_ISO ARG CONTAINER_IMAGE - ENV CONTAINER_IMAGE=$CONTAINER_IMAGE - ENV FLAVOR=$FLAVOR ENV SSH_PORT=60022 ENV CREATE_VM=true ENV USE_QEMU=true @@ -1042,7 +1146,7 @@ push-build-artifacts: RUN cd build && tar cvf ../build.tar ./ RUN luet util pack $BUNDLE_IMAGE build.tar image.tar WITH DOCKER - RUN docker load -i image.tar && docker push $BUNDLE_IMAGE + RUN docker load -i image.tar && docker push $BUNDLE_IMAGE END # bundles tests needs to run in sequence: @@ -1060,7 +1164,7 @@ prepare-bundles-tests: ARG VERSION=$(cat VERSION) RUN echo "version ${VERSION}" WITH DOCKER --load $IMG=(+examples-bundle --BUNDLE_IMAGE=$BUNDLE_IMAGE --VERSION=$VERSION) - RUN docker push $BUNDLE_IMAGE + RUN docker push $BUNDLE_IMAGE END BUILD +examples-bundle-config --BUNDLE_IMAGE=$BUNDLE_IMAGE @@ -1079,7 +1183,7 @@ examples-bundle: FROM DOCKERFILE --build-arg VERSION=$VERSION -f examples/bundle/Dockerfile . SAVE IMAGE $BUNDLE_IMAGE -## ./earthly.sh +examples-bundle-config --BUNDLE_IMAGE=ttl.sh/testfoobar:8h +## ./earthly.sh +examples-bundle-config --BUNDLE_IMAGE=ttl.sh/testfoobar:8h ## cat bundles-config.yaml examples-bundle-config: ARG BUNDLE_IMAGE @@ -1105,7 +1209,7 @@ docs: COPY . . WORKDIR ./docs - + RUN npm install postcss-cli RUN npm run prepare @@ -1113,14 +1217,14 @@ docs: SAVE ARTIFACT public /public AS LOCAL docs/public ## ./earthly.sh --push +temp-image --FLAVOR=ubuntu -## all same flags than the `docker` target plus +## all same flags than the `docker` target plus ## - the EXPIRATION time, defaults to 24h ## - the NAME of the image in ttl.sh, defaults to the branch name + short sha ## the push flag is optional -## +## ## you will have access to an image in ttl.sh e.g. ttl.sh/add-earthly-target-to-build-temp-images-339dfc7:24h temp-image: - FROM alpine + FROM alpine RUN apk add git COPY . ./ @@ -1136,6 +1240,12 @@ temp-image: ARG TTL_IMAGE = "ttl.sh/${NAME}:${EXPIRATION}" + # args for base-image target + ARG --required FLAVOR + ARG --required BASE_IMAGE + ARG --required MODEL + ARG --required VARIANT + FROM +base-image SAVE IMAGE --push $TTL_IMAGE @@ -1150,7 +1260,7 @@ generate-schema: ARG RELEASE_VERSION=$(cat VERSION) RUN mkdir "docs/static/$RELEASE_VERSION" ARG SCHEMA_FILE="docs/static/$RELEASE_VERSION/cloud-config.json" - RUN kairos-agent print-schema > $SCHEMA_FILE + RUN kairos-agent print-schema > $SCHEMA_FILE SAVE ARTIFACT ./docs/static/* AS LOCAL docs/static/ last-commit-packages: @@ -1175,6 +1285,12 @@ bump-repositories: SAVE ARTIFACT framework-profile.yaml AS LOCAL framework-profile.yaml luet-versions: + # args for base-image target + ARG --required FLAVOR + ARG --required BASE_IMAGE + ARG --required MODEL + ARG --required VARIANT + FROM +base-image SAVE ARTIFACT /framework/etc/kairos/versions.yaml versions.yaml AS LOCAL build/versions.yaml diff --git a/images/Dockerfile.alpine b/images/Dockerfile.alpine index 99215df26..2bf9e0324 100644 --- a/images/Dockerfile.alpine +++ b/images/Dockerfile.alpine @@ -2,12 +2,12 @@ #### ARGS #### ############################################################### ARG MODEL=generic +ARG BASE_IMAGE=alpine ############################################################### #### Common #### ############################################################### -FROM alpine AS common - +FROM $BASE_IMAGE AS common RUN apk --no-cache add \ bash \ bash-completion \ diff --git a/images/Dockerfile.debian b/images/Dockerfile.debian index 8e17f53b3..f2af6586a 100644 --- a/images/Dockerfile.debian +++ b/images/Dockerfile.debian @@ -2,18 +2,19 @@ #### ARGS #### ############################################################### ARG MODEL=generic +ARG BASE_IMAGE=debian:testing ############################################################### #### Upstream Images #### ############################################################### # we use testing for amd64 because zfsutils-linux is not available in bookworm -FROM debian:testing AS amd64-upstream +FROM $BASE_IMAGE AS amd64-upstream RUN <> /etc/apt/sources.list.d/kairos.list EOF -FROM debian:bookworm-slim AS arm64-upstream +FROM $BASE_IMAGE AS arm64-upstream ############################################################### #### Common #### diff --git a/images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin b/images/Dockerfile.nvidia similarity index 100% rename from images/Dockerfile.ubuntu-20-lts-arm-nvidia-jetson-agx-orin rename to images/Dockerfile.nvidia diff --git a/images/Dockerfile.opensuse-leap b/images/Dockerfile.opensuse-leap index 1f8a57f56..806750e8c 100644 --- a/images/Dockerfile.opensuse-leap +++ b/images/Dockerfile.opensuse-leap @@ -2,11 +2,12 @@ #### ARGS #### ############################################################### ARG MODEL=generic +ARG BASE_IMAGE=opensuse/leap:15.5 ############################################################### #### Common #### ############################################################### -FROM opensuse/leap:15.5 AS common +FROM $BASE_IMAGE AS common RUN zypper ar -G https://download.opensuse.org/repositories/utilities/15.4/utilities.repo && \ zypper ref diff --git a/images/Dockerfile.opensuse-tumbleweed b/images/Dockerfile.opensuse-tumbleweed index ff387aa4d..b073e4a21 100644 --- a/images/Dockerfile.opensuse-tumbleweed +++ b/images/Dockerfile.opensuse-tumbleweed @@ -2,11 +2,12 @@ #### ARGS #### ############################################################### ARG MODEL=generic +ARG BASE_IMAGE=opensuse/tumbleweed ############################################################### #### Upstream #### ############################################################### -FROM opensuse/tumbleweed AS upstream +FROM $BASE_IMAGE AS upstream ############################################################### #### Arch Config #### diff --git a/images/Dockerfile.ubuntu b/images/Dockerfile.ubuntu index 97a9025e6..a6c426231 100644 --- a/images/Dockerfile.ubuntu +++ b/images/Dockerfile.ubuntu @@ -3,9 +3,11 @@ ############################################################### # Currently supported flavors are: # - ubuntu -# - ubuntu-20-lts -# - ubuntu-22-lts ARG FLAVOR +# - 20.04 +# - 22.04 +# - 23.04 +ARG FLAVOR_RELEASE # Currently supported models are: # - generic # - rpi4 @@ -15,23 +17,26 @@ ARG MODEL=generic # the default value is empty, which means the HWE kernel WILL be installed # if you want to disable the HWE kernel, set HWE to "-non-hwe" ARG HWE="" +ARG BASE_IMAGE=ubuntu:23.04 # TARGETARCH is used to determine the architecture of the image # it is already set by Docker so it doesn't need to be defined here ############################################################### #### Upstream Images #### ############################################################### -FROM ubuntu:rolling AS ubuntu-upstream +FROM ${BASE_IMAGE} AS ubuntu-23.04-upstream -FROM ubuntu:20.04 AS ubuntu-20-lts-upstream +FROM ${BASE_IMAGE} AS ubuntu-20.04-upstream + +FROM ${BASE_IMAGE} AS ubuntu-22.04-upstream -FROM ubuntu:22.04 AS ubuntu-22-lts-upstream ############################################################### #### Pre-Process Common to All #### ############################################################### -FROM ${FLAVOR}-upstream AS common +FROM ${FLAVOR}-${FLAVOR_RELEASE}-upstream AS common ENV DEBIAN_FRONTEND=noninteractive + RUN apt-get update \ && apt-get install -y --no-install-recommends \ gnupg \ @@ -126,7 +131,7 @@ RUN apt-get update \ ############################################################### #### Common to an Arch and Flavor #### ############################################################### -FROM ${TARGETARCH} AS base-ubuntu-20-lts +FROM ${TARGETARCH} AS base-ubuntu-20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ file \ fuse \ @@ -134,7 +139,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ policykit-1 \ && apt-get clean && rm -rf /var/lib/apt/lists/* -FROM ${TARGETARCH} AS base-ubuntu-22-lts +FROM ${TARGETARCH} AS base-ubuntu-22.04 RUN apt-get update && apt-get install -y --no-install-recommends \ dracut-live \ firmware-sof-signed \ @@ -143,73 +148,73 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ polkitd \ && apt-get clean && rm -rf /var/lib/apt/lists/* -FROM base-ubuntu-22-lts AS hwe-22-lts +FROM base-ubuntu-22.04 AS hwe-22.04 RUN apt-get update && apt-get install -y --no-install-recommends \ linux-image-generic-hwe-22.04 \ && apt-get clean && rm -rf /var/lib/apt/lists/* -FROM base-ubuntu-20-lts AS hwe-20-lts +FROM base-ubuntu-20.04 AS hwe-20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ linux-image-generic-hwe-20.04 \ && apt-get clean && rm -rf /var/lib/apt/lists/* -FROM hwe-22-lts AS amd64-ubuntu -FROM hwe-22-lts AS amd64-ubuntu-22-lts -FROM hwe-20-lts AS amd64-ubuntu-20-lts +FROM hwe-22.04 AS amd64-ubuntu.23.04 +FROM hwe-22.04 AS amd64-ubuntu-22.04 +FROM hwe-20.04 AS amd64-ubuntu-20.04 -FROM base-${FLAVOR} AS amd64-non-hwe +FROM base-${FLAVOR}-${FLAVOR_RELEASE} AS amd64-non-hwe RUN apt-get update && apt-get install -y --no-install-recommends \ linux-image-generic \ && apt-get clean && rm -rf /var/lib/apt/lists/* -FROM amd64-non-hwe AS amd64-ubuntu-22-lts-non-hwe -FROM amd64-non-hwe AS amd64-ubuntu-20-lts-non-hwe +FROM amd64-non-hwe AS amd64-ubuntu-22.04-non-hwe +FROM amd64-non-hwe AS amd64-ubuntu-20.04-non-hwe -FROM base-ubuntu-22-lts AS arm64-ubuntu -FROM base-ubuntu-22-lts AS arm64-ubuntu-22-lts -FROM base-ubuntu-20-lts AS arm64-ubuntu-20-lts +FROM base-ubuntu-22.04 AS arm64-ubuntu-23.04 +FROM base-ubuntu-22.04 AS arm64-ubuntu-22.04 +FROM base-ubuntu-20.04 AS arm64-ubuntu-20.04 ############################################################### #### Common to a Single Model #### ############################################################### -FROM ${TARGETARCH}-${FLAVOR}${HWE} AS generic +FROM ${TARGETARCH}-${FLAVOR}-${FLAVOR_RELEASE}${HWE} AS generic RUN apt-get update && apt-get install -y --no-install-recommends \ linux-base \ qemu-guest-agent \ && apt-get clean && rm -rf /var/lib/apt/lists/* -FROM ${TARGETARCH}-${FLAVOR} AS rpicommon +FROM ${TARGETARCH}-${FLAVOR}-${FLAVOR_RELEASE} AS rpicommon RUN apt-get update && apt-get install -y --no-install-recommends \ linux-raspi \ && apt-get clean && rm -rf /var/lib/apt/lists/* COPY rpi/config.txt /boot/config.txt -FROM rpicommon AS ubuntu-20-lts-rpi +FROM rpicommon AS ubuntu-20.04-rpi RUN apt-get update && apt-get install -y --no-install-recommends \ linux-firmware-raspi2 \ && apt-get clean && rm -rf /var/lib/apt/lists/* -FROM rpicommon AS ubuntu-rpi +FROM rpicommon AS ubuntu-23.04-rpi RUN apt-get update && apt-get install -y --no-install-recommends \ linux-firmware-raspi \ linux-modules-extra-raspi \ && apt-get clean && rm -rf /var/lib/apt/lists/* -FROM generic AS amd64-ubuntu-generic -FROM generic AS amd64-ubuntu-22-lts-generic -FROM generic AS amd64-ubuntu-20-lts-generic -FROM generic AS arm64-ubuntu-generic -FROM ubuntu-rpi AS arm64-ubuntu-rpi3 -FROM ubuntu-rpi AS arm64-ubuntu-rpi4 -FROM ubuntu-rpi AS arm64-ubuntu-22-lts-rpi3 -FROM ubuntu-rpi AS arm64-ubuntu-22-lts-rpi4 -FROM ubuntu-20-lts-rpi AS arm64-ubuntu-20-lts-rpi3 -FROM ubuntu-20-lts-rpi AS arm64-ubuntu-20-lts-rpi4 +FROM generic AS amd64-ubuntu-23.04-generic +FROM generic AS amd64-ubuntu-22.04-generic +FROM generic AS amd64-ubuntu-20.04-generic +FROM generic AS arm64-ubuntu-23.04-generic +FROM ubuntu-23.04-rpi AS arm64-ubuntu-23.04-rpi3 +FROM ubuntu-23.04-rpi AS arm64-ubuntu-23.04-rpi4 +FROM ubuntu-23.04-rpi AS arm64-ubuntu-22.04-rpi3 +FROM ubuntu-23.04-rpi AS arm64-ubuntu-22.04-rpi4 +FROM ubuntu-20.04-rpi AS arm64-ubuntu-20.04-rpi3 +FROM ubuntu-20.04-rpi AS arm64-ubuntu-20.04-rpi4 ############################################################### #### Common to a Single Flavor #### ############################################################### -FROM ${TARGETARCH}-${FLAVOR}-${MODEL} AS ubuntu +FROM ${TARGETARCH}-${FLAVOR}-${FLAVOR_RELEASE}-${MODEL} AS ubuntu-23.04 RUN apt-get update \ && apt-get install -y --no-install-recommends \ dbus-user-session \ @@ -218,17 +223,17 @@ RUN apt-get update \ systemd-resolved \ && apt-get clean && rm -rf /var/lib/apt/lists/* -FROM ${TARGETARCH}-${FLAVOR}-${MODEL} AS ubuntu-20-lts -FROM ${TARGETARCH}-${FLAVOR}-${MODEL} AS ubuntu-20-lts-non-hwe -FROM ${TARGETARCH}-${FLAVOR}-${MODEL} AS ubuntu-22-lts-non-hwe +FROM ${TARGETARCH}-${FLAVOR}-${FLAVOR_RELEASE}-${MODEL} AS ubuntu-20.04 +FROM ${TARGETARCH}-${FLAVOR}-${FLAVOR_RELEASE}-${MODEL} AS ubuntu-20.04-non-hwe +FROM ${TARGETARCH}-${FLAVOR}-${FLAVOR_RELEASE}-${MODEL} AS ubuntu-22.04-non-hwe -FROM ubuntu-22-lts-non-hwe AS ubuntu-22-lts +FROM ubuntu-22.04-non-hwe AS ubuntu-22.04 RUN apt-get update \ && apt-get install -y --no-install-recommends \ systemd-hwe-hwdb \ && apt-get clean && rm -rf /var/lib/apt/lists/* -FROM ${FLAVOR}${HWE} AS all +FROM ${FLAVOR}-${FLAVOR_RELEASE}${HWE} AS all ############################################################### #### Post-Process Common to All #### @@ -244,3 +249,24 @@ RUN systemctl enable ssh RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo RUN journalctl --vacuum-size=1K && rm /etc/machine-id && rm /var/lib/dbus/machine-id && rm /etc/hostname + +# TODO: Consider installing luet packages here and skipping framework image altogether +# +# # Setup luet +# COPY --from=quay.io/luet/base:latest /usr/bin/luet /usr/bin/luet +# COPY framework-profile.yaml /etc/luet/luet.yaml + +# # Install luet packages +# RUN luet install -y \ +# static/grub-config \ +# dracut/immucore \ +# system/suc-upgrade \ +# system/grub2-efi \ +# static/kairos-overlay-files \ +# system/kcrypt \ +# system/kcrypt-challenger \ +# system/immucore \ +# system/kairos-agent \ +# init-svc/systemd \ +# dracut/network-legacy-compat + diff --git a/naming.sh b/naming.sh new file mode 100755 index 000000000..2ee98a90f --- /dev/null +++ b/naming.sh @@ -0,0 +1,107 @@ +#!/bin/bash + + +# This script accepts values as defined in .github/flavors.json +# and returns a proper artifact name for that set of values. +# It's meant to be the single point of truth for artifacts names. + +setEnvVarsFromJSON() { + export FLAVOR + export FLAVOR_RELEASE + export VARIANT + export TARGETARCH + export MODEL + FLAVOR=$(echo "$ARTIFACT_JSON" | jq -r '.flavor | select (.!=null)') + FLAVOR_RELEASE=$(echo "$ARTIFACT_JSON" | jq -r '.flavorRelease | select (.!=null)') + VARIANT=$(echo "$ARTIFACT_JSON" | jq -r '.variant | select (.!=null)') + TARGETARCH=$(echo "$ARTIFACT_JSON" | jq -r '.arch | select (.!=null)') + MODEL=$(echo "$ARTIFACT_JSON" | jq -r '.model | select (.!=null)') +} + +common_artifact_name() { + if [ -z "$FLAVOR_RELEASE" ]; then + echo 'FLAVOR_RELEASE must be defined' + exit 1 + fi + if [ -z "$VARIANT" ]; then + echo 'VARIANT must be defined' + exit 1 + fi + if [ -z "$TARGETARCH" ]; then + echo 'TARGETARCH must be defined' + exit 1 + fi + if [ -z "$MODEL" ]; then + echo 'MODEL must be defined' + exit 1 + fi + if [ -z "$KAIROS_VERSION" ]; then + echo 'KAIROS_VERSION must be defined' + exit 1 + fi + + echo "$FLAVOR_RELEASE-$VARIANT-$TARGETARCH-$MODEL-$KAIROS_VERSION" +} + + +bootable_artifact_name() { + if [ -z "$FLAVOR" ]; then + echo 'FLAVOR must be defined' + exit 1 + fi + local common + common=$(common_artifact_name) + + echo "kairos-$FLAVOR-$common" +} + +container_artifact_name() { + if [ -z "$KAIROS_VERSION" ]; then + echo 'KAIROS_VERSION must be defined' + exit 1 + fi + + if [ -z "$FLAVOR" ]; then + echo 'FLAVOR must be defined' + exit 1 + fi + + if [ -z "$REGISTRY_AND_ORG" ]; then + echo 'REGISTRY_AND_ORG must be defined' + exit 1 + fi + + # quay.io doesn't accept "+" in the repo name + export VERSION="${KAIROS_VERSION/+/-}" + local tag + tag=$(common_artifact_name) + + echo "$REGISTRY_AND_ORG/$FLAVOR:$tag" +} + + +if [ -n "$ARTIFACT_JSON" ]; then + setEnvVarsFromJSON +fi + +case "$1" in + "container_artifact_name") + container_artifact_name + ;; + "bootable_artifact_name") + bootable_artifact_name + ;; + "common_artifact_name") + common_artifact_name + ;; + *) + echo "Function not found: $1" + exit 1 + ;; +esac + +# ARTIFACT_JSON='{"flavor":"opensuse-leap","flavorRelease":"15.5","variant":"standard","model":"generic","arch":"amd64"}' +# KAIROS_VERSION=v2.4.1 +# REGISTRY_AND_ORG=quay.io/kairos +# container_artifact_name +# bootable_artifact_name From e757edd257b3e30728e9b78ecdff7ec486278ec5 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Mon, 23 Oct 2023 15:16:42 +0300 Subject: [PATCH 02/14] Use FRAMEWORK_FLAVOR variable to avoid confusion with FLAVOR Signed-off-by: Dimitris Karakasilis --- .github/workflows/release.yaml | 2 +- .../reusable-build-framework-flavor.yaml | 2 +- Earthfile | 20 +++++++++---------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4aadf3041..fadf3fee5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -92,7 +92,7 @@ jobs: insecure = true http = true EOF - earthly +build-framework-image --FLAVOR=${{ matrix.flavor }} --FLAVOR_RELEASE=${{ matrix.flavorRelease }} + earthly +build-framework-image --FRAMEWORK_FLAVOR=${{ matrix.flavor }} - name: Push to quay env: COSIGN_YES: true diff --git a/.github/workflows/reusable-build-framework-flavor.yaml b/.github/workflows/reusable-build-framework-flavor.yaml index 74eb76c59..d5545ace2 100644 --- a/.github/workflows/reusable-build-framework-flavor.yaml +++ b/.github/workflows/reusable-build-framework-flavor.yaml @@ -51,6 +51,6 @@ jobs: insecure = true http = true EOF - earthly +build-framework-image --FLAVOR=${FLAVOR} --FLAVOR_RELEASE=${FLAVOR_RELEASE} --VERSION=master + earthly +build-framework-image --FRAMEWORK_FLAVOR=${FLAVOR} --VERSION=master docker push "$IMAGE:$TAG" # Otherwise .RepoDigests will be empty for some reason cosign sign $(docker image inspect --format='{{index .RepoDigests 0}}' "$IMAGE:$TAG") diff --git a/Earthfile b/Earthfile index 3805a4c11..d06ba1704 100644 --- a/Earthfile +++ b/Earthfile @@ -251,8 +251,7 @@ luet: framework: FROM golang:alpine - ARG FLAVOR - ARG FLAVOR_RELEASE + ARG FRAMEWORK_FLAVOR WORKDIR /build @@ -261,7 +260,7 @@ framework: RUN go mod download COPY framework-profile.yaml /build - RUN go run main.go ${FLAVOR}-${FLAVOR_RELEASE} framework-profile.yaml /framework + RUN go run main.go ${FRAMEWORK_FLAVOR} framework-profile.yaml /framework RUN mkdir -p /framework/etc/kairos/ RUN luet database --system-target /framework get-all-installed --output /framework/etc/kairos/versions.yaml @@ -281,27 +280,26 @@ framework: build-framework-image: FROM alpine - ARG --required FLAVOR - ARG --required FLAVOR_RELEASE + ARG --required FRAMEWORK_FLAVOR # Just in case, make sure this is valid - IF [ "$FLAVOR" = "" ] - ARG FLAVOR=generic + IF [ "$FRAMEWORK_FLAVOR" = "" ] + ARG FRAMEWORK_FLAVOR=generic END COPY +version/VERSION ./ ARG VERSION=$(cat VERSION) - ARG _IMG="$IMAGE_REPOSITORY_ORG/framework:${VERSION}_${FLAVOR}" + ARG _IMG="$IMAGE_REPOSITORY_ORG/framework:${VERSION}_${FRAMEWORK_FLAVOR}" RUN echo $_IMG > FRAMEWORK_IMAGE SAVE ARTIFACT FRAMEWORK_IMAGE AS LOCAL build/FRAMEWORK_IMAGE FROM scratch - COPY (+framework/framework --FLAVOR=$FLAVOR --FLAVOR_RELEASE=$FLAVOR_RELEASE) / + COPY (+framework/framework --FRAMEWORK_FLAVOR=$FRAMEWORK_FLAVOR) / - SAVE IMAGE --push $IMAGE_REPOSITORY_ORG/framework:${VERSION}_${FLAVOR} + SAVE IMAGE --push $IMAGE_REPOSITORY_ORG/framework:${VERSION}_${FRAMEWORK_FLAVOR} base-image: ARG TARGETARCH # Earthly built-in (not passed) @@ -329,7 +327,7 @@ base-image: # Includes overlay/files # We only support non-fips for now, they are built separatedly and pushed for other to consume, not used in our workflows - COPY (+framework/framework --FLAVOR=generic) / + COPY (+framework/framework --FRAMEWORK_FLAVOR=generic) / # Avoid to accidentally push keys generated by package managers RUN rm -rf /etc/ssh/ssh_host_* From 9fd07df2fdbfc6c75854c5353cd4a1d87bbb3796 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Mon, 23 Oct 2023 18:19:44 +0300 Subject: [PATCH 03/14] Fix pipeline by adding mandatory value Signed-off-by: Dimitris Karakasilis --- .github/workflows/image-pr.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/image-pr.yaml b/.github/workflows/image-pr.yaml index 1775795bd..1674f34f4 100644 --- a/.github/workflows/image-pr.yaml +++ b/.github/workflows/image-pr.yaml @@ -43,6 +43,7 @@ jobs: uses: ./.github/workflows/reusable-install-test.yaml with: flavor: alpine + flavor_release: "3.18" needs: - core-alpine @@ -66,6 +67,7 @@ jobs: uses: ./.github/workflows/reusable-qemu-acceptance-test.yaml with: flavor: alpine + flavor_release: "3.18" needs: - core-alpine @@ -89,6 +91,7 @@ jobs: uses: ./.github/workflows/reusable-qemu-reset-test.yaml with: flavor: alpine + flavor_release: "3.18" needs: - core-alpine @@ -108,6 +111,11 @@ jobs: uses: ./.github/workflows/reusable-qemu-netboot-test.yaml with: flavor: alpine + flavor_release: "3.18" + family: alpine + base_image: alpine:3.18 + model: generic + variant: core needs: - core-alpine @@ -123,6 +131,7 @@ jobs: uses: ./.github/workflows/reusable-upgrade-with-cli-test.yaml with: flavor: alpine + flavor_release: "3.18" needs: - core-alpine @@ -139,6 +148,7 @@ jobs: # uses: ./.github/workflows/reusable-upgrade-latest-test.yaml # with: # flavor: alpine + # flavor_release: "3.18" # needs: # - core-alpine @@ -164,6 +174,7 @@ jobs: uses: ./.github/workflows/reusable-encryption-test.yaml with: flavor: alpine + flavor_release: "3.18" label: ${{ matrix.label }} needs: - core-alpine From 503e150a371e80d6c381f6a95b589146f19434ea Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Mon, 23 Oct 2023 20:47:02 +0300 Subject: [PATCH 04/14] Fix ubuntu Dockerfile Signed-off-by: Dimitris Karakasilis --- images/Dockerfile.ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/Dockerfile.ubuntu b/images/Dockerfile.ubuntu index a6c426231..33f267a19 100644 --- a/images/Dockerfile.ubuntu +++ b/images/Dockerfile.ubuntu @@ -158,7 +158,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ linux-image-generic-hwe-20.04 \ && apt-get clean && rm -rf /var/lib/apt/lists/* -FROM hwe-22.04 AS amd64-ubuntu.23.04 +FROM hwe-22.04 AS amd64-ubuntu-23.04 FROM hwe-22.04 AS amd64-ubuntu-22.04 FROM hwe-20.04 AS amd64-ubuntu-20.04 From 6f9c9e13c4536bd54b92bedf8553c50b14b2b489 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Tue, 24 Oct 2023 09:49:49 +0300 Subject: [PATCH 05/14] Apply suggestions from code review Signed-off-by: Dimitris Karakasilis --- .github/workflows/image-arm.yaml | 2 +- .../reusable-upgrade-latest-test.yaml | 2 +- images/Dockerfile.ubuntu | 21 ------------------- 3 files changed, 2 insertions(+), 23 deletions(-) diff --git a/.github/workflows/image-arm.yaml b/.github/workflows/image-arm.yaml index 0a637861b..8821c18e0 100644 --- a/.github/workflows/image-arm.yaml +++ b/.github/workflows/image-arm.yaml @@ -22,7 +22,7 @@ jobs: git fetch --prune --unshallow - id: set-matrix run: | - content=`cat ./.github/flavors.json | jq -r 'map(select(.arch == "arm64" and .variant == "standard" and .model != "generic"))'` + content=`cat ./.github/flavors.json | jq -r 'map(select(.arch == "arm64" and .variant == "core" and .model != "generic"))'` # the following lines are only required for multi line json content="${content//'%'/'%25'}" content="${content//$'\n'/'%0A'}" diff --git a/.github/workflows/reusable-upgrade-latest-test.yaml b/.github/workflows/reusable-upgrade-latest-test.yaml index b1b4b158c..4d1e122cf 100644 --- a/.github/workflows/reusable-upgrade-latest-test.yaml +++ b/.github/workflows/reusable-upgrade-latest-test.yaml @@ -54,7 +54,7 @@ jobs: # A flag to set the download target as latest release # The default value is 'false' latest: true - # TODO: Until we publish with the new names, what should this be? + # TODO: This should change after we release with the new artifact names otherwise it won't work fileName: 'kairos-core-${{ inputs.flavor }}-amd64-generic-v*.iso' out-file-path: "" - name: Display structure of downloaded files diff --git a/images/Dockerfile.ubuntu b/images/Dockerfile.ubuntu index 33f267a19..0e53d2584 100644 --- a/images/Dockerfile.ubuntu +++ b/images/Dockerfile.ubuntu @@ -249,24 +249,3 @@ RUN systemctl enable ssh RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo RUN journalctl --vacuum-size=1K && rm /etc/machine-id && rm /var/lib/dbus/machine-id && rm /etc/hostname - -# TODO: Consider installing luet packages here and skipping framework image altogether -# -# # Setup luet -# COPY --from=quay.io/luet/base:latest /usr/bin/luet /usr/bin/luet -# COPY framework-profile.yaml /etc/luet/luet.yaml - -# # Install luet packages -# RUN luet install -y \ -# static/grub-config \ -# dracut/immucore \ -# system/suc-upgrade \ -# system/grub2-efi \ -# static/kairos-overlay-files \ -# system/kcrypt \ -# system/kcrypt-challenger \ -# system/immucore \ -# system/kairos-agent \ -# init-svc/systemd \ -# dracut/network-legacy-compat - From 5d5e4e139a7ef75ed3acd59012605f570b9b624e Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Tue, 24 Oct 2023 09:36:56 +0300 Subject: [PATCH 06/14] Remove TODO (already implemented) and non-used ARG "IMAGE" Signed-off-by: Dimitris Karakasilis --- .github/workflows/release.yaml | 5 +---- Earthfile | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fadf3fee5..2d022f792 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -170,8 +170,6 @@ jobs: VARIANT: ${{ matrix.variant }} MODEL: ${{ matrix.model }} BASE_IMAGE: ${{ matrix.baseImage }} - # TODO: Create a reusable script/workflow/something that generates the artifact name - # TODO: Use the naming.sh script run: | earthly +all \ --VARIANT=${{ matrix.variant }} \ @@ -179,8 +177,7 @@ jobs: --FLAVOR=${{ matrix.flavor }} \ --FLAVOR=${{ matrix.flavorRelease }} \ --MODEL=${{ matrix.model }} \ - --BASE_IMAGE=${{ matrix.baseImage }} \ - --IMAGE=quay.io/kairos/${{ matrix.variant }}-${{ inputs.flavor }}-${{ matrix.model }}:latest + --BASE_IMAGE=${{ matrix.baseImage }} sudo mv build release - name: Push to quay diff --git a/Earthfile b/Earthfile index d06ba1704..45eb4b2d3 100644 --- a/Earthfile +++ b/Earthfile @@ -699,7 +699,7 @@ iso-uki: # This target builds an iso using a remote docker image as rootfs instead of building the whole rootfs # This should be really fast as it uses an existing image. This requires a pushed image from the +image target -# defaults to use the $IMAGE name (so ttl.sh/core-opensuse-leap:latest) +# defaults to use the $REMOTE_IMG name (so ttl.sh/core-opensuse-leap:latest) # you can override either the full thing by setting --REMOTE_IMG=docker:REPO/IMAGE:TAG # or by --REMOTE_IMG=REPO/IMAGE:TAG iso-remote: From 2b9c9430c987cc3b6d5ed92e08aafc67c17b7145 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Tue, 24 Oct 2023 09:40:03 +0300 Subject: [PATCH 07/14] Remove TODOs moved here: https://github.com/kairos-io/kairos/issues/1897#issue-1932574939 Signed-off-by: Dimitris Karakasilis --- .github/workflows/reusable-build-flavor.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/reusable-build-flavor.yaml b/.github/workflows/reusable-build-flavor.yaml index ad4078cc5..1ce76b671 100644 --- a/.github/workflows/reusable-build-flavor.yaml +++ b/.github/workflows/reusable-build-flavor.yaml @@ -92,17 +92,6 @@ jobs: env: VERSION: latest run: | - # Desired artifact names: - # container image: quay.io/kairos/ubuntu:22.04-core-amd64-generic-v2.4.0 - # iso: kairos-ubuntu-20.04-core-amd64-generic-v2.4.1-ipxe.iso - # - # TODO: - # - New branch: - # - Merge alma, rocky, fedora to one family dockerfile - # - Move rest of logic from Earthfile to Dockerfiles - # - Create a reusable worflow that simply calls docker build passing the required build-args - # - Delete the earthly base-image target and force the use of the above workflow anywhere needed - # - Make all remaining earthly targets to use a pre-cooked KAIROS_IMAGE rather than calling +base-image earthly --platform=linux/${{ inputs.arch }} +ci \ --SECURITY_SCANS=false \ --VARIANT=${{ inputs.variant }} \ From d21f2e8c084b56644e8ba7ef9614b314f4d6891b Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Tue, 24 Oct 2023 10:26:14 +0300 Subject: [PATCH 08/14] Remove unecessary env variables and fix VARIANT_RELEASE missing Signed-off-by: Dimitris Karakasilis --- .github/workflows/release.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2d022f792..2210c599d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -162,20 +162,12 @@ jobs: repository: quay.io/kairos/packages packages: utils/earthly - name: Build 🔧 - env: - FLAVOR: ${{ matrix.flavor }} - FLAVOR_RELEASE: ${{ matrix.flavorRelease }} - FAMILY: ${{ matrix.family }} - ARTIFACT: ${{ matrix.artifact }} - VARIANT: ${{ matrix.variant }} - MODEL: ${{ matrix.model }} - BASE_IMAGE: ${{ matrix.baseImage }} run: | earthly +all \ --VARIANT=${{ matrix.variant }} \ --FAMILY=${{ matrix.family }} \ --FLAVOR=${{ matrix.flavor }} \ - --FLAVOR=${{ matrix.flavorRelease }} \ + --FLAVOR_RELEASE=${{ matrix.flavorRelease }} \ --MODEL=${{ matrix.model }} \ --BASE_IMAGE=${{ matrix.baseImage }} From 20252ee7577eea9576898ecd92d0ef64d7a8c5fa Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Tue, 24 Oct 2023 14:17:37 +0300 Subject: [PATCH 09/14] PR comments Signed-off-by: Dimitris Karakasilis --- .github/workflows/reusable-build-framework-flavor.yaml | 3 --- images/Dockerfile.ubuntu | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/reusable-build-framework-flavor.yaml b/.github/workflows/reusable-build-framework-flavor.yaml index d5545ace2..376ae4a89 100644 --- a/.github/workflows/reusable-build-framework-flavor.yaml +++ b/.github/workflows/reusable-build-framework-flavor.yaml @@ -6,9 +6,6 @@ on: flavor: required: true type: string - flavor_release: - required: true - type: string jobs: build: diff --git a/images/Dockerfile.ubuntu b/images/Dockerfile.ubuntu index 0e53d2584..ff8528a70 100644 --- a/images/Dockerfile.ubuntu +++ b/images/Dockerfile.ubuntu @@ -17,7 +17,7 @@ ARG MODEL=generic # the default value is empty, which means the HWE kernel WILL be installed # if you want to disable the HWE kernel, set HWE to "-non-hwe" ARG HWE="" -ARG BASE_IMAGE=ubuntu:23.04 +ARG BASE_IMAGE # TARGETARCH is used to determine the architecture of the image # it is already set by Docker so it doesn't need to be defined here From 912624430e4c8ef33f991f500bd3ca0b8521e395 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Tue, 24 Oct 2023 15:42:59 +0300 Subject: [PATCH 10/14] Remove naming.sh script from base image Signed-off-by: Dimitris Karakasilis --- Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index 45eb4b2d3..5b82e6e81 100644 --- a/Earthfile +++ b/Earthfile @@ -468,8 +468,8 @@ base-image: RUN rm -rf /tmp/* COPY ./naming.sh . - RUN ./naming.sh container_artifact_name RUN ./naming.sh container_artifact_name > ./IMAGE + RUN rm naming.sh # luet cleanup RUN luet cleanup From d0f3efd5127299f5fb9f3f3748d48b9758f0af33 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Tue, 24 Oct 2023 16:08:06 +0300 Subject: [PATCH 11/14] Make sure we use the "fixed" kairos version with the remove "+" symbols when creating a tag for container images. Signed-off-by: Dimitris Karakasilis --- naming.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/naming.sh b/naming.sh index 2ee98a90f..7a82d6792 100755 --- a/naming.sh +++ b/naming.sh @@ -72,7 +72,7 @@ container_artifact_name() { fi # quay.io doesn't accept "+" in the repo name - export VERSION="${KAIROS_VERSION/+/-}" + export KAIROS_VERSION="${KAIROS_VERSION/+/-}" local tag tag=$(common_artifact_name) From a5875678cd7caf430bb9b1349fda53295aaaf113 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Tue, 24 Oct 2023 16:38:27 +0300 Subject: [PATCH 12/14] Setup qemu and docker buildx in reusable-build-flavor workflow because otherwise it fails to build rpi images Signed-off-by: Dimitris Karakasilis --- .github/workflows/reusable-build-flavor.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/reusable-build-flavor.yaml b/.github/workflows/reusable-build-flavor.yaml index 1ce76b671..9d5b9e704 100644 --- a/.github/workflows/reusable-build-flavor.yaml +++ b/.github/workflows/reusable-build-flavor.yaml @@ -73,6 +73,13 @@ jobs: run: | sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET + - name: Set up QEMU + uses: docker/setup-qemu-action@master + with: + platforms: all + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@master - name: Install Cosign uses: sigstore/cosign-installer@main - name: Login to Quay Registry From c6ba2dc00fd822145742783953845e8bbd856386 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Tue, 24 Oct 2023 16:47:27 +0300 Subject: [PATCH 13/14] Remove not needed env variables Signed-off-by: Dimitris Karakasilis --- .github/workflows/image.yaml | 2 +- .github/workflows/reusable-docker-arm-build.yaml | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index af56e07dc..66104968b 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -26,7 +26,7 @@ jobs: sudo apt update && sudo apt install -y jq - id: set-matrix run: | - content=`cat .github/flavors.json | jq 'map(select(.variant == "core"))'` + content=`cat .github/flavors.json | jq 'map(select(.variant == "core" and .arch == "amd64"))'` # the following lines are only required for multi line json # the following lines are only required for multi line json content="${content//'%'/'%25'}" diff --git a/.github/workflows/reusable-docker-arm-build.yaml b/.github/workflows/reusable-docker-arm-build.yaml index 632315137..8765a2052 100644 --- a/.github/workflows/reusable-docker-arm-build.yaml +++ b/.github/workflows/reusable-docker-arm-build.yaml @@ -116,10 +116,6 @@ jobs: ls -ltrh build - name: Push 🔧 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - env: - FLAVOR: ${{ inputs.flavor }} - IMAGE: quay.io/kairos/core-$FLAVOR:latest - MODEL: ${{ inputs.model }} run: | export _IMG=$(cat build/IMAGE) export _NEW_IMG=$(echo $_IMG | cut -f1 -d:):latest @@ -138,10 +134,6 @@ jobs: category: ${{ inputs.flavor }} - name: Sign image if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - env: - IMAGE: "quay.io/kairos/core-${{ inputs.flavor }}" - TAG: "latest" - COSIGN_YES: true run: | # Avoid pushing a new image for every commit (re-use latest) export _IMG=$(cat build/IMAGE) From 4fb17dc716d82ee3595c695598cb9bbff5464e66 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Tue, 24 Oct 2023 18:11:00 +0300 Subject: [PATCH 14/14] Remove temporary change Signed-off-by: Dimitris Karakasilis --- .github/workflows/image.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 66104968b..e586a0237 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -1,8 +1,5 @@ name: Build and test images on: - pull_request: - paths: - - '**' push: branches: - master