release-arm.yaml

name: Build arm images
on:
  push:
    tags:
      - 'v*'
env:
  FORCE_COLOR: 1
  EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
permissions: read-all
jobs:
  get-core-matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
        with:
          fetch-depth: 0
      - id: set-matrix
        run: |
          content=`cat ./.github/flavors.json | jq -r 'map(select(.arch == "arm64" and .variant == "core" and .model != "generic"))'`
          # the following lines are only required for multi line json
          content="${content//'%'/'%25'}"
          content="${content//$'\n'/'%0A'}"
          content="${content//$'\r'/'%0D'}"
          # end of optional handling for multi line json
          echo "::set-output name=matrix::{\"include\": $content }"

  # The matrix for standard (provider) images
  get-standard-matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
      with:
        fetch-depth: 0
    - run: |
        sudo apt update && sudo apt install -y jq
    - name: Install earthly
      uses: Luet-lab/luet-install-action@cec77490c3f2416d7d07a47cfab04d448641d7ce # v1.1
      with:
        repository: quay.io/kairos/packages
        packages: utils/earthly
    - name: Set up QEMU
      uses: docker/setup-qemu-action@master
      with:
        platforms: all
    - name: Set up Docker Buildx
      id: buildx
      uses: docker/setup-buildx-action@master
    - id: set-matrix
      run: |
          docker run --name luet quay.io/luet/base && docker cp luet:/usr/bin/luet ./
          chmod +x luet
          sudo mv luet /usr/bin/luet
          # Construct an array like this from the found versions:
          earthly --platform=linux/arm64 +extract-framework-profile
          # fetch "k3s-openrc" versions
          sudo luet --config framework-profile.yaml search -o json k8s/k3s  | jq '.packages | map(select(.name == "k3s-openrc")) | map(.version) | unique' >  k3s_openrc.json
          # fetch alpine flavors
          jq 'map(select(.arch == "arm64" and .variant == "standard" and .model != "generic" and .flavor == "alpine"))' .github/flavors.json >  flavors_openrc.json
          # generate combinations
          jq -s '. | [combinations | .[0] + {"k3s_version": .[1]}]' flavors_openrc.json k3s_openrc.json > combinations_openrc.json
          # fetch "k3s-systemd" versions
          sudo luet --config framework-profile.yaml search -o json k8s/k3s  | jq '.packages | map(select(.name == "k3s-systemd")) | map(.version) | unique' >  k3s_systemd.json
          # fetch non-alpine flavors
          jq 'map(select(.arch == "arm64" and .variant == "standard" and .model != "generic" and .flavor != "alpine"))' .github/flavors.json >  flavors_systemd.json
          # generate combinations
          jq -s '. | [combinations | .[0] + {"k3s_version": .[1]}]' flavors_systemd.json k3s_systemd.json > combinations_systemd.json
          # merge the two combinations
          content=$(jq -s 'add' combinations_openrc.json combinations_systemd.json)
          # the following lines are only required for multi line json
          content="${content//'%'/'%25'}"
          content="${content//$'\n'/'%0A'}"
          content="${content//$'\r'/'%0D'}"
          # end of optional handling for multi line json
          echo "::set-output name=matrix::{\"include\": $content }"

  build-nvidia-base:
    runs-on: ARM64
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
        with:
          fetch-depth: 0
      - name: Install kairos-agent (for versioneer)
        uses: Luet-lab/luet-install-action@cec77490c3f2416d7d07a47cfab04d448641d7ce # v1.1
        with:
          repository: quay.io/kairos/packages
          packages: system/kairos-agent
      - name: Release space from worker
        run: |
          echo "Listing top largest packages"
          pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr)
          head -n 30 <<< "${pkgs}"
          echo
          df -h
          echo
          sudo apt-get remove -y '^llvm-.*|^libllvm.*' || true
          sudo apt-get remove --auto-remove android-sdk-platform-tools || true
          sudo apt-get purge --auto-remove android-sdk-platform-tools || true
          sudo rm -rf /usr/local/lib/android
          sudo apt-get remove -y '^dotnet-.*|^aspnetcore-.*' || true
          sudo rm -rf /usr/share/dotnet
          sudo apt-get remove -y '^mono-.*' || true
          sudo apt-get remove -y '^ghc-.*' || true
          sudo apt-get remove -y '.*jdk.*|.*jre.*' || true
          sudo apt-get remove -y 'php.*' || true
          sudo apt-get remove -y hhvm || true
          sudo apt-get remove -y powershell || true
          sudo apt-get remove -y firefox || true
          sudo apt-get remove -y monodoc-manual || true
          sudo apt-get remove -y msbuild || true
          sudo apt-get remove -y microsoft-edge-stable || true
          sudo apt-get remove -y '^google-.*' || true
          sudo apt-get remove -y azure-cli || true
          sudo apt-get remove -y '^mongo.*-.*|^postgresql-.*|^mysql-.*|^mssql-.*' || true
          sudo apt-get remove -y '^gfortran-.*' || true
          sudo apt-get remove -y '^gcc-*' || true
          sudo apt-get remove -y '^g++-*' || true
          sudo apt-get remove -y '^cpp-*' || true
          sudo apt-get autoremove -y
          sudo apt-get clean
          echo
          echo "Listing top largest packages"
          pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr)
          head -n 30 <<< "${pkgs}"
          echo
          sudo rm -rfv build || true
          df -h
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@master
      - name: Block all traffic to metadata ip  # For cloud runners, the metadata ip can interact with our test machines
        run: |
          sudo iptables -I INPUT -s 169.254.169.254 -j DROP
          sudo iptables -I OUTPUT -d 169.254.169.254 -j DROP
      - name: Login to Quay Registry
        run: echo ${{ secrets.QUAY_PASSWORD }} | docker login -u ${{ secrets.QUAY_USERNAME }} --password-stdin quay.io
      - name: Build  🔧 & Push 🚀
        run: |
          export IMAGE=quay.io/kairos/cache:nvidia-base
          docker build --platform=linux/arm64 -t $IMAGE -f ./images/Dockerfile.nvidia ./images
          docker push $IMAGE

  nvidia-arm-core:
    uses: ./.github/workflows/reusable-docker-arm-build.yaml
    permissions:
      id-token: write  # OIDC support
      contents: write
      security-events: write
      actions: read
      attestations: read
      checks: read
      deployments: read
      discussions: read
      issues: read
      packages: read
      pages: read
      pull-requests: read
      repository-projects: read
      statuses: read
    needs: build-nvidia-base
    secrets: inherit
    with:
      flavor: ubuntu
      flavor_release: "20.04"
      family: ubuntu
      base_image: quay.io/kairos/cache:nvidia-base
      model: nvidia-jetson-agx-orin
      worker: ARM64

  build-arm-core:
    runs-on: ${{ matrix.worker }}
    needs:
      - get-core-matrix
    permissions:
      id-token: write  # OIDC support
      contents: write
      actions: read
      security-events: write
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.get-core-matrix.outputs.matrix)}}
    steps:
      - name: Release space from worker
        if: ${{ matrix.worker != 'kvm' }}
        run: |
          echo "Listing top largest packages"
          pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr)
          head -n 30 <<< "${pkgs}"
          echo
          df -h
          echo
          sudo apt-get remove -y '^llvm-.*|^libllvm.*' || true
          sudo apt-get remove --auto-remove android-sdk-platform-tools || true
          sudo apt-get purge --auto-remove android-sdk-platform-tools || true
          sudo rm -rf /usr/local/lib/android
          sudo apt-get remove -y '^dotnet-.*|^aspnetcore-.*' || true
          sudo rm -rf /usr/share/dotnet
          sudo apt-get remove -y '^mono-.*' || true
          sudo apt-get remove -y '^ghc-.*' || true
          sudo apt-get remove -y '.*jdk.*|.*jre.*' || true
          sudo apt-get remove -y 'php.*' || true
          sudo apt-get remove -y hhvm || true
          sudo apt-get remove -y powershell || true
          sudo apt-get remove -y firefox || true
          sudo apt-get remove -y monodoc-manual || true
          sudo apt-get remove -y msbuild || true
          sudo apt-get remove -y microsoft-edge-stable || true
          sudo apt-get remove -y '^google-.*' || true
          sudo apt-get remove -y azure-cli || true
          sudo apt-get remove -y '^mongo.*-.*|^postgresql-.*|^mysql-.*|^mssql-.*' || true
          sudo apt-get remove -y '^gfortran-.*' || true
          sudo apt-get remove -y '^gcc-*' || true
          sudo apt-get remove -y '^g++-*' || true
          sudo apt-get remove -y '^cpp-*' || true
          sudo apt-get autoremove -y
          sudo apt-get clean
          echo
          echo "Listing top largest packages"
          pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr)
          head -n 30 <<< "${pkgs}"
          echo
          sudo rm -rfv build || true
          df -h
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
        with:
          fetch-depth: 0
      - name: Set up QEMU
        uses: docker/setup-qemu-action@master
        with:
          platforms: all
      - name: Install Cosign
        uses: sigstore/cosign-installer@main
      - name: Install earthly
        uses: Luet-lab/luet-install-action@cec77490c3f2416d7d07a47cfab04d448641d7ce # v1.1
        with:
          repository: quay.io/kairos/packages
          packages: utils/earthly
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@master
      - name: Login to DockerHub
        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3
        with:
          registry: quay.io
          username: ${{ secrets.QUAY_USERNAME }}
          password: ${{ secrets.QUAY_PASSWORD }}
      - name: Cache trivy DB
        uses: nick-fields/retry@v3
        with:
          max_attempts: 3
          retry_on: error
          timeout_minutes: 5
          command: earthly +trivy
      - name: Build  🔧
        run: |
          earthly -P +all-arm \
            -VARIANT=core \
            -MODEL=${{ matrix.model }} \
            -FLAVOR=${{ matrix.flavor }} \
            -FLAVOR_RELEASE=${{ matrix.flavorRelease }} \
            -FAMILY=${{ matrix.family }} \
            -BASE_IMAGE=${{ matrix.baseImage }}
      - name: Convert all json files into a reports.tar.gz file
        run: |
          cd build
          filename=$(ls *-grype.json | head -n 1) && filename=${filename%%-grype.json}
          sudo tar cvf "${filename}-sbom-scan-reports.tar.gz" *.json
      - name: Push  🔧
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          docker push $(cat build/IMAGE)
      - name: Sign image
        env:
          COSIGN_YES: true
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          export IMAGE=$(cat build/IMAGE)
          docker push "$IMAGE" # Otherwise .RepoDigests will be empty for some reason
          cosign sign $(docker image inspect --format='{{index .RepoDigests 0}}' "$IMAGE")
      - name: Upload Image
        if: startsWith(github.ref, 'refs/tags/')
        env:
          COSIGN_YES: true
        run: |
          curl https://luet.io/install.sh | sudo sh
          IMAGE=$(cat build/IMAGE | sed 's/$/-img/')
          sudo tar cvf build.tar build
          sudo luet util pack $IMAGE build.tar image.tar
          sudo -E docker load -i image.tar
          sudo -E docker push "$IMAGE"

          # Sign the -img image
          cosign sign $(docker image inspect --format='{{index .RepoDigests 0}}' "$IMAGE")

          sudo rm -rf build/IMAGE
      - name: Release
        uses: softprops/action-gh-release@c062e08bd532815e2082a85e87e3ef29c3e6d191 # v2.0.8
        if: startsWith(github.ref, 'refs/tags/')
        with:
          files: |
            build/*scan-reports.tar.gz
      - name: Prepare sarif files  🔧
        run: |
          mkdir trivy-sarif grype-sarif
          sudo mv build/*trivy.sarif trivy-sarif/
          sudo mv build/*grype.sarif grype-sarif/
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3
        if: startsWith(github.ref, 'refs/tags/')
        with:
          sarif_file: 'trivy-sarif'
          category: ${{ matrix.flavor }}-trivy
      - name: Upload Grype scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3
        if: startsWith(github.ref, 'refs/tags/')
        with:
          sarif_file: 'grype-sarif'
          category: ${{ matrix.flavor }}-grype
  build-arm-standard:
    runs-on: ARM64
    needs:
      - get-standard-matrix
    permissions:
      id-token: write  # OIDC support
      contents: write
      actions: read
      security-events: write
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.get-standard-matrix.outputs.matrix)}}
    steps:
      - name: Install required packages
        run: |
          sudo apt-get update
          sudo apt-get install -y \
            curl
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
        with:
          fetch-depth: 0
      - name: Set up QEMU
        uses: docker/setup-qemu-action@master
        with:
          platforms: all
      - name: Install Cosign
        uses: sigstore/cosign-installer@main
      - name: Install earthly
        uses: Luet-lab/luet-install-action@cec77490c3f2416d7d07a47cfab04d448641d7ce # v1.1
        with:
          repository: quay.io/kairos/packages
          packages: utils/earthly
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@master
      - name: size of docker images
        run: |
          docker images --format "{{.Size}} - {{.Repository}}:{{.Tag}}"
      - name: Available space
        run: |
          df -h
      - name: Login to DockerHub
        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3
        with:
          registry: quay.io
          username: ${{ secrets.QUAY_USERNAME }}
          password: ${{ secrets.QUAY_PASSWORD }}
      - name: Cache trivy DB
        uses: nick-fields/retry@v3
        with:
          max_attempts: 3
          retry_on: error
          timeout_minutes: 5
          command: earthly +trivy
      - name: Build  🔧
        run: |
          earthly -P +all-arm \
            -VARIANT=standard \
            -MODEL=${{ matrix.model }} \
            -K3S_VERSION=${{ matrix.k3s_version }} \
            -FLAVOR=${{ matrix.flavor }} \
            -FLAVOR_RELEASE=${{ matrix.flavorRelease }} \
            -FAMILY=${{ matrix.family }} \
            -BASE_IMAGE=${{ matrix.baseImage }}
      - name: Convert all json files into a reports.tar.gz file
        run: |
          cd build
          filename=$(ls *-grype.json | head -n 1) && filename=${filename%%-grype.json}
          sudo tar cvf "${filename}-sbom-scan-reports.tar.gz" *.json
      - name: Push  🔧
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          docker push $(cat build/IMAGE)
      - name: Sign image
        env:
          COSIGN_YES: true
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          export IMAGE=$(cat build/IMAGE)
          docker push "$IMAGE" # Otherwise .RepoDigests will be empty for some reason
          cosign sign $(docker image inspect --format='{{index .RepoDigests 0}}' "$IMAGE")
      - name: Upload Image
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          curl https://luet.io/install.sh | sudo sh
          IMAGE=$(cat build/IMAGE | sed 's/$/-img/')
          sudo tar cvf build.tar build
          sudo luet util pack $IMAGE build.tar image.tar
          sudo -E docker load -i image.tar
          sudo -E docker push "$IMAGE"
          sudo rm -rf build/IMAGE
      - name: Release
        uses: softprops/action-gh-release@c062e08bd532815e2082a85e87e3ef29c3e6d191 # v2.0.8
        if: startsWith(github.ref, 'refs/tags/')
        with:
          files: |
            build/*scan-reports.tar.gz
      - name: Prepare sarif files  🔧
        run: |
          mkdir trivy-sarif grype-sarif
          sudo mv build/*trivy.sarif trivy-sarif/
          sudo mv build/*grype.sarif grype-sarif/
      - name: Upload Trivy scan results to GitHub Security tab
        if: startsWith(github.ref, 'refs/tags/')
        uses: github/codeql-action/upload-sarif@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3
        with:
          sarif_file: 'trivy-sarif'
          category: ${{ matrix.flavor }}-trivy
      - name: Upload Grype scan results to GitHub Security tab
        if: startsWith(github.ref, 'refs/tags/')
        uses: github/codeql-action/upload-sarif@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3
        with:
          sarif_file: 'grype-sarif'
          category: ${{ matrix.flavor }}-grype
      - name: Space stats
        if: always()
        run: |
          df -h
      - name: Image stats
        if: always()
        run: |
          docker images --format "{{.Size}} - {{.Repository}}:{{.Tag}}"

  build-arm-generic:
    runs-on: ARM64
    permissions:
      id-token: write  # OIDC support
      contents: write
      security-events: write
      actions: read
      attestations: read
      checks: read
      deployments: read
      discussions: read
      issues: read
      packages: read
      pages: read
      pull-requests: read
      repository-projects: read
      statuses: read
    strategy:
      fail-fast: false
      matrix:
        include:
          - flavor: opensuse
            flavor_release: leap-15.6
            family: opensuse
            base_image: opensuse/leap:15.6
            model: generic
            variant: standard
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
        with:
          fetch-depth: 0
      - name: Install earthly
        uses: Luet-lab/luet-install-action@cec77490c3f2416d7d07a47cfab04d448641d7ce # v1.1
        with:
          repository: quay.io/kairos/packages
          packages: utils/earthly
      - name: Set up QEMU
        uses: docker/setup-qemu-action@master
        with:
          platforms: all
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3
      - name: Login to Quay Registry
        run: echo ${{ secrets.QUAY_PASSWORD }} | docker login -u ${{ secrets.QUAY_USERNAME }} --password-stdin quay.io
      - name: Cache trivy DB
        uses: nick-fields/retry@v3
        with:
          max_attempts: 3
          retry_on: error
          timeout_minutes: 5
          command: earthly +trivy
      - name: Build iso  🔧
        run: |
          INIT=$([[ "${{ matrix.flavor }}" == "alpine" ]] && echo "openrc" || echo "systemd")
          earthly --platform=linux/arm64 +extract-framework-profile
          K3S_VERSION=$(sudo luet --config framework-profile.yaml search -o json k8s/k3s  | jq --arg INIT "$INIT" '.packages | map(select(.name == "k3s-" + $INIT)) | map(.version) | unique | last' | tr -d '"')
          earthly -P +all-arm-generic \
            --FLAVOR=${{ matrix.flavor }} \
            --FLAVOR_RELEASE=${{ matrix.flavor_release }} \
            --FAMILY=${{ matrix.family }} \
            --BASE_IMAGE=${{ matrix.base_image}} \
            --MODEL=${{ matrix.model }} \
            --VARIANT=${{ matrix.variant }} \
            --K3S_VERSION=${K3S_VERSION}

          sudo mv build release
      - name: Push to quay
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          docker push $(cat release/IMAGE)
      - name: Release
        uses: softprops/action-gh-release@c062e08bd532815e2082a85e87e3ef29c3e6d191 # v2.0.8
        if: startsWith(github.ref, 'refs/tags/')
        with:
          files: |
            release/*iso*