diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 9ccadf4..176d665 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -61,7 +61,7 @@ jobs: with: builder: ${{ steps.buildx.outputs.name }} build-args: | - BINARY_VERSION=${{ steps.prep.outputs.binary_version }} + VERSION=${{ steps.prep.outputs.binary_version }} context: ./ file: ./Dockerfile platforms: linux/amd64,linux/arm64 diff --git a/.gitignore b/.gitignore index 86d369e..30621da 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ examples/airgap/build examples/airgap/data dist/ +build/ diff --git a/Dockerfile b/Dockerfile index 299eaab..58a415a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,24 @@ -ARG VERSION=v0.400.3 ARG LUET_VERSION=0.35.5 +ARG LEAP_VERSION=15.5 FROM quay.io/luet/base:$LUET_VERSION AS luet FROM golang AS builder -ARG BINARY_VERSION=v0.0.0 +ARG VERSION=v0.0.0 WORKDIR /work ADD go.mod . ADD go.sum . RUN go mod download ADD . . -RUN CGO_ENABLED=0 go build -ldflags "-X main.version=${BINARY_VERSION}" -o auroraboot - -FROM quay.io/kairos/osbuilder-tools:$VERSION +ENV CGO_ENABLED=0 +ENV VERSION=$VERSION +RUN go build -ldflags "-X main.version=${VERSION}" -o auroraboot +FROM opensuse/leap:$LEAP_VERSION AS default +RUN zypper ref && zypper dup -y +## ISO+ Arm image + Netboot + cloud images Build depedencies +RUN zypper ref && zypper in -y bc qemu qemu-tools jq cdrtools docker git curl gptfdisk kpartx sudo xfsprogs parted binutils \ + util-linux-systemd e2fsprogs curl util-linux udev rsync grub2 dosfstools grub2-x86_64-efi squashfs mtools xorriso lvm2 zstd COPY --from=luet /usr/bin/luet /usr/bin/luet ENV LUET_NOLOCK=true ENV TMPDIR=/tmp @@ -27,6 +32,80 @@ RUN cp /tmp/luet-${TARGETARCH}.yaml /etc/luet/luet.yaml ## Uki artifacts, will be set under the /usr/kairos directory RUN luet install -y system/systemd-boot -RUN zypper in -y qemu binutils +## Live CD artifacts +RUN luet install -y livecd/grub2 --system-target /grub2 +RUN luet install -y livecd/grub2-efi-image --system-target /efi + +## RPI64 +RUN luet install -y firmware/u-boot-rpi64 firmware/raspberrypi-firmware firmware/raspberrypi-firmware-config firmware/raspberrypi-firmware-dt --system-target /rpi/ + +## PineBook64 Pro +RUN luet install -y arm-vendor-blob/u-boot-rockchip --system-target /pinebookpro/u-boot + +## Odroid fw +RUN luet install -y firmware/odroid-c2 --system-target /firmware/odroid-c2 + +## RAW images for current arch +RUN luet install -y static/grub-efi --system-target /raw/grub +RUN luet install -y static/grub-config --system-target /raw/grubconfig +RUN luet install -y static/grub-artifacts --system-target /raw/grubartifacts + +## RAW images for arm64 +# Luet will install this artifacts from the current arch repo, so in x86 it will +# get them from the x86 repo and we want it to do it from the arm64 repo, even on x86 +# so we use the arm64 luet config and use that to install those on x86 +# This is being used by the prepare_arm_images.sh and build-arch-image.sh scripts +RUN luet install --config /tmp/luet-arm64.yaml -y static/grub-efi --system-target /arm/raw/grubefi +RUN luet install --config /tmp/luet-arm64.yaml -y static/grub-config --system-target /arm/raw/grubconfig +RUN luet install --config /tmp/luet-arm64.yaml -y static/grub-artifacts --system-target /arm/raw/grubartifacts + +# kairos-agent so we can use the pull-image +RUN luet install -y system/kairos-agent + +# remove luet tmp files. Side effect of setting the system-target is that it treats it as a root fs +# so temporal files are stored in each dir +RUN rm -Rf /grub2/var/tmp +RUN rm -Rf /grub2/var/cache +RUN rm -Rf /efi/var/tmp +RUN rm -Rf /efi/var/cache +RUN rm -Rf /rpi/var/tmp +RUN rm -Rf /rpi/var/cache +RUN rm -Rf /pinebookpro/u-boot/var/tmp +RUN rm -Rf /pinebookpro/u-boot/var/cache +RUN rm -Rf /firmware/odroid-c2/var/tmp +RUN rm -Rf /firmware/odroid-c2/var/cache +RUN rm -Rf /raw/grub/var/tmp +RUN rm -Rf /raw/grub/var/cache +RUN rm -Rf /raw/grubconfig/var/tmp +RUN rm -Rf /raw/grubconfig/var/cache +RUN rm -Rf /raw/grubartifacts/var/tmp +RUN rm -Rf /raw/grubartifacts/var/cache +RUN rm -Rf /arm/raw/grubefi/var/tmp +RUN rm -Rf /arm/raw/grubefi/var/cache +RUN rm -Rf /arm/raw/grubconfig/var/tmp +RUN rm -Rf /arm/raw/grubconfig/var/cache +RUN rm -Rf /arm/raw/grubartifacts/var/tmp +RUN rm -Rf /arm/raw/grubartifacts/var/cache + +# ISO build config +COPY ./image-assets/add-cloud-init.sh /add-cloud-init.sh +COPY ./image-assets/kairos-release.tmpl /kairos-release.tmpl +COPY ./image-assets/ipxe.tmpl /ipxe.tmpl +COPY ./image-assets/update-os-release.sh /update-os-release.sh + +# ARM helpers +COPY ./image-assets/build-arm-image.sh /build-arm-image.sh +COPY ./image-assets/arm /arm +COPY ./image-assets/prepare_arm_images.sh /prepare_arm_images.sh + +# RAW images helpers +COPY ./image-assets/gce.sh /gce.sh +COPY ./image-assets/raw-images.sh /raw-images.sh +COPY ./image-assets/azure.sh /azure.sh +COPY ./image-assets/netboot.sh /netboot.sh + +COPY ./image-assets/defaults.yaml /defaults.yaml + COPY --from=builder /work/auroraboot /usr/bin/auroraboot + ENTRYPOINT ["/usr/bin/auroraboot"] diff --git a/Earthfile b/Earthfile index ae6d538..2c81a16 100644 --- a/Earthfile +++ b/Earthfile @@ -1,14 +1,25 @@ VERSION 0.7 -ARG --global OSBUILDER_VERSION=v0.9.0 ARG --global GO_VERSION=1.23-bookworm # renovate: datasource=github-releases depName=kairos-io/kairos ARG IMAGE_VERSION=v3.2.1 ARG --global BASE_IMAGE=quay.io/kairos/ubuntu:24.04-core-amd64-generic-${IMAGE_VERSION}-uki +version: + FROM alpine + RUN apk update && apk add git + + COPY . . + RUN --no-cache git describe --always --tags --dirty > VERSION + SAVE ARTIFACT VERSION VERSION + image: - FROM DOCKERFILE --build-arg VERSION=$OSBUILDER_VERSION -f Dockerfile . - RUN zypper in -y qemu + FROM +version + ARG VERSION=$(cat VERSION) + + FROM DOCKERFILE --build-arg VERSION=$VERSION -f Dockerfile . + + SAVE IMAGE quay.io/kairos/auroraboot:$VERSION test-label: FROM alpine @@ -71,3 +82,27 @@ test-bootable: ARG CREATE_VM=true RUN date RUN go run github.com/onsi/ginkgo/v2/ginkgo run --label-filter "bootable" -v --fail-fast -r ./e2e + +last-commit-packages: + FROM quay.io/skopeo/stable + RUN dnf install -y jq + WORKDIR build + ENV jqQuery='.Tags | map(select(. | contains("-repository.yaml"))) | sort_by(. | sub("v";"") | sub("-repository.yaml";"") | sub("-git.*";"") | .[0:12] | tonumber) | .[-1]' + RUN skopeo list-tags docker://quay.io/kairos/packages | jq -rc "${jqQuery}" > REPO_AMD64 + RUN skopeo list-tags docker://quay.io/kairos/packages-arm64 | jq -rc "${jqQuery}" > REPO_ARM64 + SAVE ARTIFACT REPO_AMD64 REPO_AMD64 + SAVE ARTIFACT REPO_ARM64 REPO_ARM64 + +bump-repositories: + FROM mikefarah/yq + WORKDIR build + COPY +last-commit-packages/REPO_AMD64 REPO_AMD64 + COPY +last-commit-packages/REPO_ARM64 REPO_ARM64 + ARG REPO_AMD64=$(cat REPO_AMD64) + ARG REPO_ARM64=$(cat REPO_ARM64) + COPY image-assets/luet-amd64.yaml luet-amd64.yaml + COPY image-assets/luet-arm64.yaml luet-arm64.yaml + RUN yq eval ".repositories[0] |= . * { \"reference\": \"${REPO_AMD64}\" }" -i luet-amd64.yaml + RUN yq eval ".repositories[0] |= . * { \"reference\": \"${REPO_ARM64}\" }" -i luet-arm64.yaml + SAVE ARTIFACT luet-arm64.yaml AS LOCAL image-assets/luet-arm64.yaml + SAVE ARTIFACT luet-amd64.yaml AS LOCAL image-assets/luet-amd64.yaml diff --git a/deployer/register.go b/deployer/register.go index 7f40f3b..776a386 100644 --- a/deployer/register.go +++ b/deployer/register.go @@ -19,7 +19,7 @@ const ( opPrepareNetboot = "prepare-netboot" opStartNetboot = "start-netboot" - opContainerPull = "container-pull" + opDumpSource = "dump-source" opGenISO = "gen-iso" opPreparetmproot = "prepare-temp" opExtractNetboot = "extract-netboot" @@ -43,7 +43,7 @@ func RegisterAll(d *Deployer) error { d.StepPrepNetbootDir, d.StepPrepISODir, d.StepCopyCloudConfig, - d.StepPullContainer, + d.StepDumpSource, d.StepGenISO, d.StepExtractNetboot, //TODO: add Validate step diff --git a/deployer/steps.go b/deployer/steps.go index 345e64f..c063488 100644 --- a/deployer/steps.go +++ b/deployer/steps.go @@ -4,7 +4,6 @@ import ( "context" "os" "path/filepath" - "strings" "github.com/kairos-io/AuroraBoot/pkg/ops" "github.com/spectrocloud-labs/herd" @@ -21,7 +20,7 @@ func (d *Deployer) StepPrepNetbootDir() error { func (d *Deployer) StepPrepTmpRootDir() error { return d.Add(opPreparetmproot, herd.WithCallback( func(ctx context.Context) error { - return os.MkdirAll(d.dstNetboot(), 0700) + return os.MkdirAll(d.tmpRootFs(), 0700) }, )) } @@ -40,17 +39,17 @@ func (d *Deployer) StepCopyCloudConfig() error { })) } -func (d *Deployer) StepPullContainer() error { +func (d *Deployer) StepDumpSource() error { // Ops to generate from container image - return d.Add(opContainerPull, + return d.Add(opDumpSource, herd.EnableIf(d.fromImage), - herd.WithDeps(opPreparetmproot), herd.WithCallback(ops.PullContainerImage(d.containerImage(), d.tmpRootFs()))) + herd.WithDeps(opPreparetmproot), herd.WithCallback(ops.DumpSource(d.Artifact.ContainerImage, d.tmpRootFs()))) } func (d *Deployer) StepGenISO() error { return d.Add(opGenISO, herd.EnableIf(func() bool { return d.fromImage() && !d.rawDiskIsSet() && d.Config.Disk.ARM == nil }), - herd.WithDeps(opContainerPull, opCopyCloudConfig), herd.WithCallback(ops.GenISO(d.tmpRootFs(), d.destination(), d.Config.ISO))) + herd.WithDeps(opDumpSource, opCopyCloudConfig), herd.WithCallback(ops.GenISO(d.tmpRootFs(), d.destination(), d.Config.ISO))) } func (d *Deployer) StepExtractNetboot() error { @@ -178,16 +177,6 @@ func (d *Deployer) StepStartNetboot() error { ) } -func (d *Deployer) containerImage() string { - // Pull local docker daemon if container image starts with docker:// - containerImage := d.Artifact.ContainerImage - if strings.HasPrefix(containerImage, "docker://") { - containerImage = strings.ReplaceAll(containerImage, "docker://", "") - } - - return containerImage -} - func (d *Deployer) fromImage() bool { return d.Artifact.ContainerImage != "" } @@ -233,7 +222,7 @@ func (d *Deployer) isoOption() bool { } func (d *Deployer) imageOrSquashFS() herd.OpOption { - return herd.IfElse(d.fromImage(), herd.WithDeps(opContainerPull), herd.WithDeps(opExtractSquashFS)) + return herd.IfElse(d.fromImage(), herd.WithDeps(opDumpSource), herd.WithDeps(opExtractSquashFS)) } func (d *Deployer) cloudConfigPath() string { diff --git a/go.mod b/go.mod index 74fe849..8b74fc0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ godebug x509negativeserial=1 require ( github.com/cavaliergopher/grab/v3 v3.0.1 github.com/containerd/containerd v1.7.23 - github.com/distribution/reference v0.6.0 github.com/foxboron/go-uefi v0.0.0-20241017190036-fab4fdf2f2f3 github.com/foxboron/sbctl v0.0.0-20240526163235-64e649b31c8e github.com/gofrs/uuid v4.4.0+incompatible @@ -71,6 +70,7 @@ require ( github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/diskfs/go-diskfs v1.4.2 // indirect + github.com/distribution/reference v0.6.0 // indirect github.com/djherbis/times v1.6.0 // indirect github.com/docker/cli v27.1.1+incompatible // indirect github.com/docker/distribution v2.8.2+incompatible // indirect diff --git a/image-assets/add-cloud-init.sh b/image-assets/add-cloud-init.sh new file mode 100755 index 0000000..f414a4f --- /dev/null +++ b/image-assets/add-cloud-init.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# docker run --entrypoint /add-cloud-init.sh -v $PWD:/work -ti --rm test https://github.com/kairos-io/kairos/releases/download/v1.1.2/kairos-alpine-v1.1.2.iso /work/test.iso /work/config.yaml + +set -ex + +ISO=$1 +OUT=$2 +CONFIG=$3 + +case ${ISO} in +http*) + curl -L "${ISO}" -o in.iso + ISO=in.iso + ;; +esac + +# Needs xorriso >=1.5.4 +xorriso -indev $ISO -outdev $OUT -map $CONFIG /config.yaml -boot_image any replay \ No newline at end of file diff --git a/image-assets/arm/boards/odroid_c2.sh b/image-assets/arm/boards/odroid_c2.sh new file mode 100755 index 0000000..c83c020 --- /dev/null +++ b/image-assets/arm/boards/odroid_c2.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +image=$1 + +if [ -z "$image" ]; then + echo "No image specified" + exit 1 +fi + +# conv=notrunc ? +dd if=/firmware/odroid-c2/bl1.bin.hardkernel of=$image conv=fsync bs=1 count=442 +dd if=/firmware/odroid-c2/bl1.bin.hardkernel of=$image conv=fsync bs=512 skip=1 seek=1 +dd if=/firmware/odroid-c2/u-boot.odroidc2 of=$image conv=fsync bs=512 seek=97 \ No newline at end of file diff --git a/image-assets/arm/boards/pinebookpro.sh b/image-assets/arm/boards/pinebookpro.sh new file mode 100755 index 0000000..e1a3acd --- /dev/null +++ b/image-assets/arm/boards/pinebookpro.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +image=$1 + +if [ -z "$image" ]; then + echo "No image specified" + exit 1 +fi + +LOADER_OFFSET=${LOADER_OFFSET:-"64"} +LOADER_IMAGE=${LOADER_IMAGE:-"idbloader.img"} +UBOOT_IMAGE=${UBOOT_IMAGE:-"u-boot.itb"} +UBOOT_OFFSET=${UBOOT_OFFSET:-"16384"} + +echo "Writing idbloader" +dd conv=notrunc if=/pinebookpro/u-boot/usr/lib/u-boot/pinebook-pro-rk3399/${LOADER_IMAGE} of="$image" conv=fsync seek=${LOADER_OFFSET} +echo "Writing u-boot image" +dd conv=notrunc if=/pinebookpro/u-boot/usr/lib/u-boot/pinebook-pro-rk3399/${UBOOT_IMAGE} of="$image" conv=fsync seek=${UBOOT_OFFSET} +sync $image \ No newline at end of file diff --git a/image-assets/arm/boards/rpi3.sh b/image-assets/arm/boards/rpi3.sh new file mode 100755 index 0000000..86fe0aa --- /dev/null +++ b/image-assets/arm/boards/rpi3.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +partprobe + +kpartx -va $DRIVE + +image=$1 + +if [ -z "$image" ]; then + echo "No image specified" + exit 1 +fi + +set -ax +TEMPDIR="$(mktemp -d)" +echo $TEMPDIR +mount "${device}p1" "${TEMPDIR}" + +# Copy all rpi files +cp -rfv /rpi/* $TEMPDIR + +umount "${TEMPDIR}" diff --git a/image-assets/arm/boards/rpi4.sh b/image-assets/arm/boards/rpi4.sh new file mode 100755 index 0000000..16479b8 --- /dev/null +++ b/image-assets/arm/boards/rpi4.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +partprobe + +image=$1 + +if [ -z "$image" ]; then + echo "No image specified" + exit 1 +fi + +set -ax +TEMPDIR="$(mktemp -d)" +echo $TEMPDIR +mount "${device}p1" "${TEMPDIR}" + +# Copy all rpi files +cp -rfv /rpi/* $TEMPDIR + +umount "${TEMPDIR}" diff --git a/image-assets/azure.sh b/image-assets/azure.sh new file mode 100755 index 0000000..819f6f8 --- /dev/null +++ b/image-assets/azure.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Transform a raw image disk to azure vhd +RAWIMAGE="$1" +VHDDISK="${2:-disk.vhd}" +cp -rf $RAWIMAGE $VHDDISK.work + +MB=$((1024*1024)) +size=$(qemu-img info -f raw --output json "$RAWIMAGE" | gawk 'match($0, /"virtual-size": ([0-9]+),/, val) {print val[1]}') +# shellcheck disable=SC2004 +ROUNDED_SIZE=$(((($size+$MB-1)/$MB)*$MB)) +echo "Resizing raw image to $ROUNDED_SIZE" +qemu-img resize -f raw "$VHDDISK.work" $ROUNDED_SIZE +echo "Converting $RAWIMAGE to $VHDDISK" +qemu-img convert -f raw -o subformat=fixed,force_size -O vpc "$VHDDISK.work" "$VHDDISK" +echo "Done" +rm -rf "$VHDDISK.work" \ No newline at end of file diff --git a/image-assets/bios-raw-image.sh b/image-assets/bios-raw-image.sh new file mode 100755 index 0000000..a5f4c93 --- /dev/null +++ b/image-assets/bios-raw-image.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Generates raw bootable images with qemu +set -ex +CLOUD_INIT=${1:-cloud_init.yaml} +QEMU=${QEMU:-qemu-system-x86_64} +ISO=${2:-iso.iso} + +mkdir -p build +pushd build +touch meta-data +cp -rfv $CLOUD_INIT user-data + +mkisofs -output ci.iso -volid cidata -joliet -rock user-data meta-data +truncate -s "+$((20000*1024*1024))" disk.raw + +${QEMU} -m 8096 -smp cores=2 \ + -nographic -cpu host \ + -serial mon:stdio \ + -rtc base=utc,clock=rt \ + -chardev socket,path=qga.sock,server,nowait,id=qga0 \ + -device virtio-serial \ + -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \ + -drive if=virtio,media=disk,file=disk.raw \ + -drive format=raw,media=cdrom,readonly=on,file=$ISO \ + -drive format=raw,media=cdrom,readonly=on,file=ci.iso \ + -boot d \ + -enable-kvm \ No newline at end of file diff --git a/image-assets/build-arm-image.sh b/image-assets/build-arm-image.sh new file mode 100755 index 0000000..a61137d --- /dev/null +++ b/image-assets/build-arm-image.sh @@ -0,0 +1,507 @@ +#!/bin/bash + +## This is a re-adaptation of https://github.com/rancher/elemental-toolkit/blob/main/images/arm-img-builder.sh + +set -ex + +load_vars() { + + model=${MODEL:-odroid_c2} + directory=${DIRECTORY:-} + output_image="${OUTPUT_IMAGE:-arm.img}" + # Img creation options. Size is in MB for all of the vars below + size="${SIZE:-7608}" + state_size="${STATE_SIZE:-4992}" + oem_size="${OEM_SIZE:-64}" + recovery_size="${RECOVERY_SIZE:-2192}" + default_active_size="${DEFAULT_ACTIVE_SIZE:-2400}" + menu_entry="${DEFAULT_MENU_ENTRY:-Kairos}" + + ## Repositories + final_repo="${FINAL_REPO:-quay.io/costoolkit/releases-teal-arm64}" + repo_type="${REPO_TYPE:-docker}" + + # Warning: these default values must be aligned with the values provided + # in 'packages/cos-config/cos-config', provide an environment file using the + # --cos-config flag if different values are needed. + : "${OEM_LABEL:=COS_OEM}" + : "${RECOVERY_LABEL:=COS_RECOVERY}" + : "${ACTIVE_LABEL:=COS_ACTIVE}" + : "${PASSIVE_LABEL:=COS_PASSIVE}" + : "${PERSISTENT_LABEL:=COS_PERSISTENT}" + : "${SYSTEM_LABEL:=COS_SYSTEM}" + : "${STATE_LABEL:=COS_STATE}" +} + +cleanup() { + sync + sync + sleep 5 + sync + if [ -n "$EFI" ]; then + rm -rf $EFI + fi + if [ -n "$RECOVERY" ]; then + rm -rf $RECOVERY + fi + if [ -n "$STATEDIR" ]; then + rm -rf $STATEDIR + fi + if [ -n "$TARGET" ]; then + umount $TARGET || true + umount $LOOP || true + rm -rf $TARGET + fi + if [ -n "$WORKDIR" ]; then + rm -rf $WORKDIR + fi + if [ -n "$DRIVE" ]; then + umount $DRIVE || true + fi + if [ -n "$recovery" ]; then + umount $recovery || true + fi + if [ -n "$state" ]; then + umount $state || true + fi + if [ -n "$efi" ]; then + umount $efi || true + fi + if [ -n "$oem" ]; then + umount $oem || true + fi + + losetup -D "${LOOP}" || true; + losetup -D "${DRIVE}" || true; + + if [ "$model" == "rpi3" ]; then + dmsetup remove KairosVG-oem || true; + dmsetup remove KairosVG-recovery || true; + fi +} + +ensure_dir_structure() { + local target=$1 + for mnt in /sys /proc /dev /tmp /boot /usr/local /oem + do + if [ ! -d "${target}${mnt}" ]; then + mkdir -p ${target}${mnt} + fi + done +} + +usage() +{ + echo "Usage: $0 [options] image.img" + echo "" + echo "Example: $0 --cos-config cos-config --model odroid-c2 --docker-image output.img" + echo "" + echo "Flags:" + echo " --cos-config: (optional) Specifies a cos-config file for required environment variables" + echo " --config: (optional) Specify a cloud-init config file to embed into the final image" + echo " --manifest: (optional) Specify a manifest file to customize efi/grub packages installed into the image" + echo " --size: (optional) Image size (MB)" + echo " --state-partition-size: (optional) Size of the state partition (MB)" + echo " --recovery-partition-size: (optional) Size of the recovery partition (MB)" + echo " --images-size: (optional) Size of the active/passive/recovery images (MB)" + echo " --docker-image: (optional) A container image which will be used for active/passive/recovery system" + echo " --directory: (optional) A directory which will be used for active/passive/recovery system" + echo " --model: (optional) The board model" + echo " --efi-dir: (optional) A directory with files which will be added to the efi partition" + exit 1 +} + +get_url() +{ + FROM=$1 + TO=$2 + case $FROM in + ftp*|http*|tftp*) + n=0 + attempts=5 + until [ "$n" -ge "$attempts" ] + do + curl -o $TO -fL ${FROM} && break + n=$((n+1)) + echo "Failed to download, retry attempt ${n} out of ${attempts}" + sleep 2 + done + ;; + *) + cp -f $FROM $TO + ;; + esac +} + +trap "cleanup" 1 2 3 6 14 15 EXIT + +load_vars + +while [ "$#" -gt 0 ]; do + case $1 in + --cos-config) + shift 1 + cos_config=$1 + ;; + --config) + shift 1 + config=$1 + ;; + --manifest) + shift 1 + manifest=$1 + ;; + --size) + shift 1 + size=$1 + ;; + --state-partition-size) + shift 1 + state_size=$1 + ;; + --recovery-partition-size) + shift 1 + recovery_size=$1 + ;; + --images-size) + shift 1 + default_active_size=$1 + ;; + --docker-image) + shift 1 + CONTAINER_IMAGE=$1 + ;; + --directory) + shift 1 + directory=$1 + ;; + --model) + shift 1 + model=$1 + ;; + --efi-dir) + shift 1 + efi_dir=$1 + ;; + --final-repo) + shift 1 + final_repo=$1 + ;; + --repo-type) + shift 1 + repo_type=$1 + ;; + -h) + usage + ;; + --help) + usage + ;; + *) + if [ "$#" -gt 2 ]; then + usage + fi + output_image=$1 + break + ;; + esac + shift 1 +done + +if [ "$model" == "rpi64" ];then + echo "rpi64 model not supported anymore, please select either rpi3 or rpi4" + exit 1 +fi + +if [ "$model" == "rpi3" ] || [ "$model" == "rpi4" ]; then + container_image=${CONTAINER_IMAGE:-quay.io/costoolkit/examples:rpi-latest} +else + # Odroid C2 image contains kernel-default-extra, might have broader support + container_image=${CONTAINER_IMAGE:-quay.io/costoolkit/examples:odroid-c2-latest} +fi + +if [ -n "$cos_config" ] && [ -e "$cos_config" ]; then + # shellcheck source=/dev/null + source "$cos_config" +fi + +if [ -z "$output_image" ]; then + echo "No image file specified" + exit 1 +fi + +if [ -n "$manifest" ]; then + YQ_PACKAGES_COMMAND=(yq e -o=json "$manifest") + final_repo=${final_repo:-$(yq e ".raw_disk.$model.repo" "${manifest}")} +fi + +echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" +echo "Image Size: $size MB." +echo "Recovery Partition: $recovery_size." +echo "State Partition: $state_size MB." +echo "Images size (active/passive/recovery.img): $default_active_size MB." +echo "Model: $model" +if [ -n "$container_image" ] && [ -z "$directory" ]; then + echo "Container image: $container_image" +fi +if [ -n "$directory" ]; then + echo "Root from directory: $directory" +fi +echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + +# Temp dir used during build +WORKDIR=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX) +#ROOT_DIR=$(git rev-parse --show-toplevel) +TARGET=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX) +STATEDIR=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX) + + +export WORKDIR + +# Prepare active.img + +echo ">> Preparing active.img" +mkdir -p ${STATEDIR}/cOS + +dd if=/dev/zero of=${STATEDIR}/cOS/active.img bs=1M count=$default_active_size + +mkfs.ext2 ${STATEDIR}/cOS/active.img -L ${ACTIVE_LABEL} + +sync + +LOOP=$(losetup --show -f ${STATEDIR}/cOS/active.img) +if [ -z "$LOOP" ]; then +echo "No device" +exit 1 +fi + +mount -t ext2 $LOOP $TARGET + +ensure_dir_structure $TARGET + +# Download the container image +if [ -z "$directory" ]; then + echo ">>> Downloading container image" + kairos-agent pull-image $container_image $TARGET +else + echo ">>> Copying files from $directory" + rsync -axq --exclude='host' --exclude='mnt' --exclude='proc' --exclude='sys' --exclude='dev' --exclude='tmp' ${directory}/ $TARGET +fi + +# We copy the grubmenu.cfg to a temporary location to be copied later in the state partition +# https://github.com/kairos-io/kairos/blob/62c67e3e61d49435c362014522e5c6696335376f/overlay/files/system/oem/08_grub.yaml#L105 +# This is a hack and we need a better way: https://github.com/kairos-io/kairos/issues/1427 +tmpgrubconfig=$(mktemp /tmp/grubmeny.cfg.XXXXXX) +cp -rfv $TARGET/etc/kairos/branding/grubmenu.cfg "${tmpgrubconfig}" + +umount $TARGET +sync + +if [ -n "$LOOP" ]; then + losetup -d $LOOP +fi + +echo ">> Preparing passive.img" +cp -rfv ${STATEDIR}/cOS/active.img ${STATEDIR}/cOS/passive.img +tune2fs -L ${PASSIVE_LABEL} ${STATEDIR}/cOS/passive.img + +# Preparing recovery +echo ">> Preparing recovery.img" +RECOVERY=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX) +if [ -z "$RECOVERY" ]; then + echo "No recovery directory" + exit 1 +fi + +mkdir -p ${RECOVERY}/cOS +cp -rfv ${STATEDIR}/cOS/active.img ${RECOVERY}/cOS/recovery.img + +tune2fs -L ${SYSTEM_LABEL} ${RECOVERY}/cOS/recovery.img + +# Install real grub config to recovery +cp -rfv /arm/raw/grubconfig/* $RECOVERY +mkdir -p $RECOVERY/grub2/fonts +cp -rfv /arm/raw/grubartifacts/* $RECOVERY/grub2 +mv $RECOVERY/grub2/*pf2 $RECOVERY/grub2/fonts + +sync + +# Prepare efi files +echo ">> Preparing EFI partition" +EFI=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX) +if [ -z "$EFI" ]; then + echo "No EFI directory" + exit 1 +fi + +cp -rfv /arm/raw/grubefi/* $EFI +if [ -n "$EFI" ] && [ -n "$efi_dir" ]; then + echo "Copy $efi_dir to EFI directory" + cp -rfv $efi_dir/* $EFI +fi + +partprobe + +echo ">> Writing image and partition table" +dd if=/dev/zero of="${output_image}" bs=1024000 count="${size}" || exit 1 + +# Image partitions +# only rpi4 supports gpt +if [ "$model" == "rpi3" ]; then + sgdisk -n 1:8192:+96M -c 1:EFI -t 1:0c00 ${output_image} + sgdisk -n 2:0:+${state_size}M -c 2:state -t 2:8300 ${output_image} + sgdisk -n 3:0:+$(( recovery_size + oem_size ))M -c 3:lvm -t 3:8e00 ${output_image} + sgdisk -n 4:0:+64M -c 4:persistent -t 4:8300 ${output_image} + sgdisk -m 1:2:3:4 ${output_image} + sfdisk --part-type ${output_image} 1 c +elif [ "$model" == "rpi4" ]; then + sgdisk -n 1:0:+96M -c 1:EFI -t 1:ef00 ${output_image} + partprobe + sgdisk -n 2:0:+${state_size}M -c 2:state -t 2:8300 ${output_image} + partprobe + sgdisk -n 3:0:+${recovery_size}M -c 3:recovery -t 3:8300 ${output_image} + partprobe + sgdisk -n 4:0:+${oem_size}M -c 4:oem -t 4:8300 ${output_image} + partprobe + sgdisk -n 5:0:+64M -c 5:persistent -t 5:8300 ${output_image} + partprobe + # Move backup header to end of disk + sgdisk -e ${output_image} + sgdisk -v ${output_image} +else + sgdisk -n 1:8192:+16M -c 1:EFI -t 1:0700 ${output_image} + sgdisk -n 2:0:+${state_size}M -c 2:state -t 2:8300 ${output_image} + sgdisk -n 3:0:+$(( recovery_size + oem_size ))M -c 3:lvm -t 3:8e00 ${output_image} + sgdisk -n 4:0:+64M -c 4:persistent -t 4:8300 ${output_image} + sgdisk -m 1:2:3:4 ${output_image} +fi + +# Prepare the image and copy over the files + +DRIVE=$(losetup -f "${output_image}" --show) +export DRIVE +if [ -z "${DRIVE}" ]; then + echo "Cannot execute losetup for $output_image" + exit 1 +fi + +device=${DRIVE/\/dev\//} + +if [ -z "$device" ]; then + echo "No device" + exit 1 +fi + +export device="/dev/mapper/${device}" + +partprobe + +if [ "$model" == 'rpi4' ]; then + kpartx -vag $DRIVE +else + kpartx -va $DRIVE +fi + +echo ">> Populating partitions" +efi=${device}p1 +state=${device}p2 +recovery=${device}p3 + +if [ "$model" == 'rpi4' ]; then + oem=${device}p4 + persistent=${device}p5 +else + persistent=${device}p4 + oem_lv=/dev/mapper/KairosVG-oem + recovery_lv=/dev/mapper/KairosVG-recovery +fi + +# Create partitions (RECOVERY, STATE, COS_PERSISTENT) +mkfs.vfat -F 32 ${efi} +fatlabel ${efi} COS_GRUB +mkfs.ext3 -F -L ${STATE_LABEL} $state +mkfs.ext3 -F -L ${PERSISTENT_LABEL} $persistent + +if [ "$model" == 'rpi4' ]; then + mkfs.ext3 -F -L ${RECOVERY_LABEL} $recovery + mkfs.ext3 -F -L ${OEM_LABEL} $oem +else + pvcreate $recovery + vgcreate KairosVG $recovery + lvcreate -Z n -n oem -L ${oem_size} KairosVG + lvcreate -Z n -n recovery -l 100%FREE KairosVG + vgchange -ay + vgmknodes + mkfs.ext4 -F -L ${OEM_LABEL} $oem_lv + mkfs.ext4 -F -L ${RECOVERY_LABEL} $recovery_lv +fi + +mkdir $WORKDIR/state +mkdir $WORKDIR/recovery +mkdir $WORKDIR/efi +mkdir $WORKDIR/oem + +mount $state $WORKDIR/state +mount $efi $WORKDIR/efi + +if [ "$model" == 'rpi4' ]; then + mount $recovery $WORKDIR/recovery + mount $oem $WORKDIR/oem +else + mount $recovery_lv $WORKDIR/recovery + mount $oem_lv $WORKDIR/oem +fi + +cp -rfv /defaults.yaml $WORKDIR/oem/01_defaults.yaml + +# Set a OEM config file if specified +if [ -n "$config" ]; then + echo ">> Copying $config OEM config file" + get_url $config $WORKDIR/oem/99_custom.yaml +fi + +grub2-editenv $WORKDIR/state/grub_oem_env set "default_menu_entry=$menu_entry" + +# We copy the file we saved earier to the STATE partition +cp -rfv "${tmpgrubconfig}" $WORKDIR/state/grubmenu + +# Copy over content +cp -arf $EFI/* $WORKDIR/efi +cp -arf $RECOVERY/* $WORKDIR/recovery +cp -arf $STATEDIR/* $WORKDIR/state + +umount $WORKDIR/recovery +umount $WORKDIR/state +umount $WORKDIR/efi +umount $WORKDIR/oem + + +if [ "$model" != 'rpi4' ]; then + vgchange -an +fi + +sync + +# Flash uboot and vendor-specific bits +echo ">> Performing $model specific bits.." +/arm/boards/$model.sh ${DRIVE} + +sync +sleep 5 +sync + +if [ "$model" == 'rpi4' ]; then + kpartx -dvg $DRIVE +else + kpartx -dv $DRIVE || true +fi + +umount $DRIVE || true + +echo ">> Done writing $output_image" + +echo ">> Creating SHA256 sum" + +sha256sum $output_image > $output_image.sha256 + +cleanup diff --git a/image-assets/defaults.yaml b/image-assets/defaults.yaml new file mode 100644 index 0000000..a7fe034 --- /dev/null +++ b/image-assets/defaults.yaml @@ -0,0 +1,8 @@ +#cloud-config +name: "Default user" +stages: + initramfs: + - name: "Set default user/pass" + users: + kairos: + passwd: "kairos" diff --git a/image-assets/gce.sh b/image-assets/gce.sh new file mode 100755 index 0000000..d3b1a53 --- /dev/null +++ b/image-assets/gce.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Transform a raw image disk to gce compatible +RAWIMAGE="$1" +OUT="${2:-$RAWIMAGE.gce.raw}" +cp -rf $RAWIMAGE $OUT + +GB=$((1024*1024*1024)) +size=$(qemu-img info -f raw --output json "$OUT" | gawk 'match($0, /"virtual-size": ([0-9]+),/, val) {print val[1]}') +# shellcheck disable=SC2004 +ROUNDED_SIZE=$(echo "$size/$GB+1"|bc) +echo "Resizing raw image from \"$size\"MB to \"$ROUNDED_SIZE\"GB" +qemu-img resize -f raw "$OUT" "$ROUNDED_SIZE"G +echo "Compressing raw image $OUT to $OUT.tar.gz" +tar -c -z --format=oldgnu -f "$OUT".tar.gz $OUT diff --git a/image-assets/ipxe.tmpl b/image-assets/ipxe.tmpl new file mode 100644 index 0000000..9549fa3 --- /dev/null +++ b/image-assets/ipxe.tmpl @@ -0,0 +1,7 @@ +#!ipxe + +set dns 8.8.8.8 +ifconf +kernel ${RELEASE_URL}/${VERSION}/${ISO_NAME}-kernel root=live:${RELEASE_URL}/${VERSION}/${ISO_NAME}.squashfs initrd=${ISO_NAME}-initrd rd.neednet=1 ip=dhcp rd.cos.disable netboot install-mode config_url=${config} console=tty1 console=ttyS0 rd.live.overlay.overlayfs ${cmdline} +initrd ${RELEASE_URL}/${VERSION}/${ISO_NAME}-initrd +boot \ No newline at end of file diff --git a/image-assets/kairos-release.tmpl b/image-assets/kairos-release.tmpl new file mode 100644 index 0000000..0e74e5f --- /dev/null +++ b/image-assets/kairos-release.tmpl @@ -0,0 +1,13 @@ +KAIROS_NAME="${OS_NAME}" +KAIROS_VERSION="${OS_VERSION}" +KAIROS_ID="${OS_ID}" +KAIROS_ID_LIKE="${OS_NAME}" +KAIROS_VERSION_ID="${OS_VERSION}" +KAIROS_PRETTY_NAME="${OS_NAME} ${OS_VERSION}" +KAIROS_BUG_REPORT_URL="${BUG_REPORT_URL}" +KAIROS_HOME_URL="${HOME_URL}" +KAIROS_IMAGE_REPO="${OS_REPO}" +KAIROS_IMAGE_LABEL="${OS_LABEL}" +KAIROS_GITHUB_REPO="${GITHUB_REPO}" +KAIROS_VARIANT="${VARIANT}" +KAIROS_FLAVOR="${FLAVOR}" diff --git a/image-assets/luet-amd64.yaml b/image-assets/luet-amd64.yaml new file mode 100644 index 0000000..2ab555a --- /dev/null +++ b/image-assets/luet-amd64.yaml @@ -0,0 +1,16 @@ +general: + debug: false + spinner_charset: 9 +logging: + enable_emoji: false +repositories: + - name: "kairos" + description: "kairos repository" + type: "docker" + cached: true + enable: true + priority: 2 + urls: + - "quay.io/kairos/packages" + # renovate: datasource=docker depName=quay.io/kairos/packages + reference: 202411120825-git6d3d6215-repository.yaml diff --git a/image-assets/luet-arm64.yaml b/image-assets/luet-arm64.yaml new file mode 100644 index 0000000..a60210f --- /dev/null +++ b/image-assets/luet-arm64.yaml @@ -0,0 +1,16 @@ +general: + debug: false + spinner_charset: 9 +logging: + enable_emoji: false +repositories: + - name: "kairos-arm64" + description: "kairos repository arm64" + type: "docker" + cached: true + enable: true + priority: 2 + urls: + - "quay.io/kairos/packages-arm64" + # renovate: datasource=docker depName=quay.io/kairos/packages-arm64 + reference: 202411120835-git6d3d6215-repository.yaml diff --git a/image-assets/netboot.sh b/image-assets/netboot.sh new file mode 100755 index 0000000..34da4cc --- /dev/null +++ b/image-assets/netboot.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Extracts squashfs, kernel, initrd and generates a ipxe template script + +ISO=$1 +OUTPUT_NAME=$2 +ARTIFACT_NAME=$(basename $OUTPUT_NAME) + +isoinfo -x /rootfs.squashfs -R -i $ISO > $OUTPUT_NAME.squashfs +isoinfo -x /boot/kernel -R -i $ISO > $OUTPUT_NAME-kernel +isoinfo -x /boot/initrd -R -i $ISO > $OUTPUT_NAME-initrd + +URL=${URL:-https://github.com/kairos-io/kairos/releases/download} + +cat > $OUTPUT_NAME.ipxe << EOF +#!ipxe +set url ${URL}/ +set kernel $ARTIFACT_NAME-kernel +set initrd $ARTIFACT_NAME-initrd +set rootfs $ARTIFACT_NAME.squashfs +# set config https://example.com/machine-config +# set cmdline extra.values=1 +kernel \${url}/\${kernel} initrd=\${initrd} ip=dhcp rd.cos.disable root=live:\${url}/\${rootfs} netboot install-mode config_url=\${config} console=tty1 console=ttyS0 \${cmdline} +initrd \${url}/\${initrd} +boot +EOF \ No newline at end of file diff --git a/image-assets/prepare_arm_images.sh b/image-assets/prepare_arm_images.sh new file mode 100755 index 0000000..4f5de43 --- /dev/null +++ b/image-assets/prepare_arm_images.sh @@ -0,0 +1,140 @@ +#!/bin/bash +# This script prepares Kairos state, recovery, oem and pesistent partitions as img files. + +set -e + +# Temp dir used during build +WORKDIR=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX) +TARGET=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX) +STATEDIR=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX) + +: "${OEM_LABEL:=COS_OEM}" +: "${RECOVERY_LABEL:=COS_RECOVERY}" +: "${ACTIVE_LABEL:=COS_ACTIVE}" +: "${PASSIVE_LABEL:=COS_PASSIVE}" +: "${PERSISTENT_LABEL:=COS_PERSISTENT}" +: "${SYSTEM_LABEL:=COS_SYSTEM}" +: "${STATE_LABEL:=COS_STATE}" + +size="${SIZE:-7544}" +state_size="${STATE_SIZE:-4992}" +recovery_size="${RECOVERY_SIZE:-2192}" +default_active_size="${DEFAULT_ACTIVE_SIZE:-2400}" +menu_entry="${DEFAULT_MENU_ENTRY:-Kairos}" + +container_image="${container_image:-quay.io/kairos/kairos-opensuse-leap-arm-rpi:v1.5.1-k3sv1.25.6-k3s1}" + +ensure_dir_structure() { + local target=$1 + for mnt in /sys /proc /dev /tmp /boot /usr/local /oem + do + if [ ! -d "${target}${mnt}" ]; then + mkdir -p ${target}${mnt} + fi + done +} + +mkdir -p $WORKDIR/tmpefi + +# Create the EFI partition FAT16 and include the EFI image and a basic grub.cfg +truncate -s $((20*1024*1024)) bootloader/efi.img +cp -rfv /arm/raw/grubefi/* $WORKDIR/tmpefi +mkfs.fat -F16 -n COS_GRUB bootloader/efi.img +mcopy -s -i bootloader/efi.img $WORKDIR/tmpefi/EFI ::EFI + +mkdir -p ${STATEDIR}/cOS + +dd if=/dev/zero of=${STATEDIR}/cOS/active.img bs=1M count=$default_active_size + +mkfs.ext2 ${STATEDIR}/cOS/active.img -L ${ACTIVE_LABEL} + + +LOOP=$(losetup --show -f ${STATEDIR}/cOS/active.img) +if [ -z "$LOOP" ]; then + echo "No device" + exit 1 +fi + +mount -t ext2 $LOOP $TARGET + +ensure_dir_structure $TARGET + +# Download the container image +if [ -z "$directory" ]; then + echo ">>> Downloading container image" + luet util unpack $container_image $TARGET +else + echo ">>> Copying files from $directory" + rsync -axq --exclude='host' --exclude='mnt' --exclude='proc' --exclude='sys' --exclude='dev' --exclude='tmp' ${directory}/ $TARGET +fi + +# We copy the grubmenu.cfg to a temporary location to be copied later in the state partition +# https://github.com/kairos-io/kairos/blob/62c67e3e61d49435c362014522e5c6696335376f/overlay/files/system/oem/08_grub.yaml#L105 +# This is a hack and we need a better way: https://github.com/kairos-io/kairos/issues/1427 +tmpgrubconfig=$(mktemp /tmp/grubmeny.cfg.XXXXXX) +cp -rfv $TARGET/etc/kairos/branding/grubmenu.cfg "${tmpgrubconfig}" + +umount $TARGET +sync + +losetup -d $LOOP + + +echo ">> Preparing passive.img" +cp -rfv ${STATEDIR}/cOS/active.img ${STATEDIR}/cOS/passive.img +tune2fs -L ${PASSIVE_LABEL} ${STATEDIR}/cOS/passive.img + + +# Preparing recovery +echo ">> Preparing recovery.img" +RECOVERY=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX) +mkdir -p ${RECOVERY}/cOS +cp -rfv ${STATEDIR}/cOS/active.img ${RECOVERY}/cOS/recovery.img +tune2fs -L ${SYSTEM_LABEL} ${RECOVERY}/cOS/recovery.img + +# Install real grub config to recovery +cp -rfv /arm/raw/grubconfig/* $RECOVERY +mkdir -p $RECOVERY/grub2/fonts +cp -rfv /arm/raw/grubartifacts/* $RECOVERY/grub2 +mv $RECOVERY/grub2/*pf2 $RECOVERY/grub2/fonts + +dd if=/dev/zero of=recovery_partition.img bs=1M count=$recovery_size +dd if=/dev/zero of=state_partition.img bs=1M count=$state_size + +mkfs.ext4 -F -L ${RECOVERY_LABEL} recovery_partition.img +LOOP=$(losetup --show -f recovery_partition.img) +mkdir -p $WORKDIR/recovery +mount $LOOP $WORKDIR/recovery +cp -arf $RECOVERY/* $WORKDIR/recovery +umount $WORKDIR/recovery +losetup -d $LOOP + +mkfs.ext4 -F -L ${STATE_LABEL} state_partition.img +LOOP=$(losetup --show -f state_partition.img) +mkdir -p $WORKDIR/state +mount $LOOP $WORKDIR/state +cp -arf $STATEDIR/* $WORKDIR/state +grub2-editenv $WORKDIR/state/grub_oem_env set "default_menu_entry=$menu_entry" + +# We copy the file we saved earier to the STATE partition +cp -rfv "${tmpgrubconfig}" $WORKDIR/state/grubmenu + +umount $WORKDIR/state +losetup -d $LOOP + +cp -rfv state_partition.img bootloader/ +cp -rfv recovery_partition.img bootloader/ + +## Optional, prepare COS_OEM and COS_PERSISTENT + +# Create the grubenv forcing first boot to be on recovery system +mkdir -p $WORKDIR/oem +cp -rfv /defaults.yaml $WORKDIR/oem/01_defaults.yaml + +# Create a 64MB filesystem for OEM volume +truncate -s $((64*1024*1024)) bootloader/oem.img +mkfs.ext2 -L "${OEM_LABEL}" -d $WORKDIR/oem bootloader/oem.img + +# Create a 2GB filesystem for COS_PERSISTENT volume +truncate -s $((2048*1024*1024)) bootloader/persistent.img +mkfs.ext2 -L "${PERSISTENT_LABEL}" bootloader/persistent.img diff --git a/image-assets/raw-images.sh b/image-assets/raw-images.sh new file mode 100755 index 0000000..47f613c --- /dev/null +++ b/image-assets/raw-images.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Generates EFI bootable images (statically) +# This is a re-adaptation of https://github.com/rancher/elemental-toolkit/blob/v0.8.10-1/images/img-builder.sh, which was dropped +# How to use: +# First extract the image which you want to create an image from: +### luet util unpack rootfs +# Then convert it to a raw disk (EFI only): +### docker run -v $PWD:/output --entrypoint /raw-images.sh -ti --rm test-image /output/rootfs /output/foo.raw cloud-init.yaml + +: "${OEM_LABEL:=COS_OEM}" +: "${RECOVERY_LABEL:=COS_RECOVERY}" +: "${EXTEND:=}" +: "${RECOVERY_SIZE:=2048}" + +DIRECTORY=$1 +OUT=${2:-disk.raw} +CONFIG=$3 + +echo "Output: $OUT" + +set -e + +mkdir -p /build/root/grub2 +mkdir /build/root/cOS +mkdir /build/efi + +cp -rf /raw/grub/* /build/efi +cp -rf /raw/grubconfig/* /build/root +cp -rf /raw/grubartifacts/* /build/root/grub2 + +echo "Generating squashfs from $DIRECTORY" +mksquashfs $DIRECTORY recovery.squashfs -b 1024k -comp xz -Xbcj x86 +mv recovery.squashfs /build/root/cOS/recovery.squashfs + +grub2-editenv /build/root/grub_oem_env set "default_menu_entry=Kairos" + +# Create a 2GB filesystem for RECOVERY including the contents for root (grub config and squasfs container) +# shellcheck disable=SC2004 +truncate -s $(($RECOVERY_SIZE*1024*1024)) rootfs.part +mkfs.ext2 -L "${RECOVERY_LABEL}" -d /build/root rootfs.part + +# Create the EFI partition FAT16 and include the EFI image and a basic grub.cfg +truncate -s $((20*1024*1024)) efi.part + +mkfs.fat -F16 -n COS_GRUB efi.part +mcopy -s -i efi.part /build/efi/EFI ::EFI + +# Create the grubenv forcing first boot to be on recovery system +mkdir -p /build/oem +cp /build/root/etc/cos/grubenv_firstboot /build/oem/grubenv +if [ -n "$CONFIG" ]; then + echo "Copying config file ($CONFIG)" + cp $CONFIG /build/oem +fi + +# Create a 64MB filesystem for OEM volume +truncate -s $((64*1024*1024)) oem.part +mkfs.ext2 -L "${OEM_LABEL}" -d /build/oem oem.part + +echo "Generating image $OUT" +# Create disk image, add 3MB of initial free space to disk, 1MB is for proper alignement, 2MB are for the hybrid legacy boot. +truncate -s $((3*1024*1024)) $OUT +{ + cat efi.part + cat oem.part + cat rootfs.part +} >> $OUT + +# Add an extra MB at the end of the disk for the gpt headers, in fact 34 sectors would be enough, but adding some more does not hurt. +truncate -s "+$((1024*1024))" $OUT + +if [ -n "$EXTEND" ]; then + echo "Extending image of $EXTEND MB" + truncate -s "+$((EXTEND*1024*1024))" $OUT +fi + +# Create the partition table in $OUT (assumes sectors of 512 bytes) +sgdisk -n 1:2048:+2M -c 1:legacy -t 1:EF02 $OUT +sgdisk -n 2:0:+20M -c 2:UEFI -t 2:EF00 $OUT +sgdisk -n 3:0:+64M -c 3:oem -t 3:8300 $OUT +sgdisk -n 4:0:+${RECOVERY_SIZE}M -c 4:root -t 4:8300 $OUT diff --git a/image-assets/update-os-release.sh b/image-assets/update-os-release.sh new file mode 100755 index 0000000..00f064f --- /dev/null +++ b/image-assets/update-os-release.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# usage: +# docker run --rm -ti --entrypoint /update-os-release.sh \ +# -v /etc:/workspace \ # mount the directory where your os-release is, this is by default in /etc but you can mount a different dir for testing +# -e OS_NAME=kairos-core-opensuse-leap \ +# -e OS_VERSION=v2.2.0 \ +# -e OS_ID="kairos" \ +# -e OS_NAME=kairos-core-opensuse-leap \ +# -e BUG_REPORT_URL="https://github.com/kairos-io/kairos/issues" \ +# -e HOME_URL="https://github.com/kairos-io/kairos" \ +# -e OS_REPO="quay.io/kairos/core-opensuse-leap" \ +# -e OS_LABEL="latest" \ +# -e GITHUB_REPO="kairos-io/kairos" \ +# -e VARIANT="core" \ +# -e FLAVOR="opensuse-leap" +# quay.io/kairos/osbuilder-tools:latest + +set -ex + +[ -f "/workspace/kairos-release" ] && sed -i -n '/KAIROS_/!p' /workspace/kairos-release +# Clean up old os-release just in case so we dont have stuff lying around +sed -i -n '/KAIROS_/!p' /workspace/os-release +envsubst >>/workspace/kairos-release < /kairos-release.tmpl + +cat /workspace/kairos-release diff --git a/internal/cmd/build-iso.go b/internal/cmd/build-iso.go index a721fdc..1ec0a8c 100644 --- a/internal/cmd/build-iso.go +++ b/internal/cmd/build-iso.go @@ -113,7 +113,7 @@ var BuildISOCmd = cli.Command{ d.StepPrepTmpRootDir, d.StepPrepISODir, d.StepCopyCloudConfig, - d.StepPullContainer, + d.StepDumpSource, d.StepGenISO, } { if err := step(); err != nil { diff --git a/pkg/ops/container.go b/pkg/ops/container.go index 0be4638..eebff89 100644 --- a/pkg/ops/container.go +++ b/pkg/ops/container.go @@ -3,37 +3,32 @@ package ops import ( "context" "fmt" - "os" - "github.com/distribution/reference" - "github.com/kairos-io/AuroraBoot/internal" - sdkUtils "github.com/kairos-io/kairos-sdk/utils" + "github.com/kairos-io/kairos-agent/v2/pkg/elemental" + v1 "github.com/kairos-io/kairos-agent/v2/pkg/types/v1" + sdkTypes "github.com/kairos-io/kairos-sdk/types" ) -// PullContainerImage pulls a container image either remotely or locally from a docker daemon. -func PullContainerImage(image, dst string) func(ctx context.Context) error { +// DumpSource pulls a container image either remotely or locally from a docker daemon +// or simply copies the directory to the destination. +// Supports these prefixes: +// https://github.com/kairos-io/kairos-agent/blob/1e81cdef38677c8a36cae50d3334559976f66481/pkg/types/v1/common.go#L30-L33 +func DumpSource(image, dst string) func(ctx context.Context) error { return func(ctx context.Context) error { - _, err := reference.ParseNormalizedNamed(image) - if err != nil { - return fmt.Errorf("invalid image reference %s", image) - } + cfg := NewConfig( + WithImageExtractor(v1.OCIImageExtractor{}), + WithLogger(sdkTypes.NewKairosLogger("auroraboot-dump-source", "debug", false)), + ) + e := elemental.NewElemental(cfg) - img, err := sdkUtils.GetImage(image, "", nil, nil) + imgSource, err := v1.NewSrcFromURI(image) if err != nil { - internal.Log.Logger.Error().Err(err).Str("image", image).Msg("failed to pull image") return err } - internal.Log.Logger.Info().Msgf("Pulling container image '%s' to '%s')", image, dst) - - // This method already first tries the local registry and then moves to remote, so no need to pass local - err = os.MkdirAll(dst, os.ModeDir|os.ModePerm) - if err != nil { - internal.Log.Logger.Error().Err(err).Str("image", image).Msg("failed to create directory") + if _, err := e.DumpSource(dst, imgSource); err != nil { + return fmt.Errorf("dumping the source image %s to %s: %w", image, dst, err) } - err = sdkUtils.ExtractOCIImage(img, dst) - if err != nil { - internal.Log.Logger.Error().Err(err).Str("image", image).Msg("failed to extract OCI image") - } - return err + + return nil } } diff --git a/renovate.json b/renovate.json index 542a19e..25ad4e1 100644 --- a/renovate.json +++ b/renovate.json @@ -9,7 +9,9 @@ "every weekend" ], "timezone": "Europe/Brussels", - "reviewers": [ "team:maintainers" ], + "reviewers": [ + "team:maintainers" + ], "rebaseWhen": "behind-base-branch", "packageRules": [ { @@ -18,5 +20,16 @@ ], "automerge": true } + ], + "regexManagers": [ + { + "fileMatch": [ + "image-assets/luet-amd64.yaml$", + "image-assets/luet-arm64.yaml$" + ], + "matchStrings": [ + "#\\s*renovate:\\s*datasource=(?.*?) depName=(?.*?)?\\s+reference:\\s(?.*?)\\s" + ] + } ] } diff --git a/tests/e2e/disks_test.go b/tests/e2e/disks_test.go index 0c06c90..22e159a 100644 --- a/tests/e2e/disks_test.go +++ b/tests/e2e/disks_test.go @@ -43,8 +43,8 @@ var _ = Describe("Disk image generation", Label("raw-disks"), func() { Expect(out).To(ContainSubstring("gen-raw-disk"), out) Expect(out).To(ContainSubstring("download-squashfs"), out) Expect(out).To(ContainSubstring("extract-squashfs"), out) - Expect(out).ToNot(ContainSubstring("container-pull"), out) - Expect(err).ToNot(HaveOccurred()) + Expect(out).ToNot(ContainSubstring("dump-source"), out) + Expect(err).ToNot(HaveOccurred(), out) _, err = os.Stat(filepath.Join(tempDir, "build/build/disk.raw")) Expect(err).ToNot(HaveOccurred()) }) @@ -65,8 +65,8 @@ var _ = Describe("Disk image generation", Label("raw-disks"), func() { Expect(out).To(ContainSubstring("gen-raw-disk"), out) Expect(out).To(ContainSubstring("download-squashfs"), out) Expect(out).To(ContainSubstring("extract-squashfs"), out) - Expect(out).ToNot(ContainSubstring("container-pull"), out) - Expect(err).ToNot(HaveOccurred()) + Expect(out).ToNot(ContainSubstring("dump-source"), out) + Expect(err).ToNot(HaveOccurred(), out) _, err = os.Stat(filepath.Join(tempDir, "build/build/disk.raw.gce")) Expect(err).ToNot(HaveOccurred()) }) @@ -87,8 +87,8 @@ var _ = Describe("Disk image generation", Label("raw-disks"), func() { Expect(out).To(ContainSubstring("gen-raw-disk"), out) Expect(out).To(ContainSubstring("download-squashfs"), out) Expect(out).To(ContainSubstring("extract-squashfs"), out) - Expect(out).ToNot(ContainSubstring("container-pull"), out) - Expect(err).ToNot(HaveOccurred()) + Expect(out).ToNot(ContainSubstring("dump-source"), out) + Expect(err).ToNot(HaveOccurred(), out) _, err = os.Stat(filepath.Join(tempDir, "build/build/disk.raw.vhd")) Expect(err).ToNot(HaveOccurred()) }) @@ -109,8 +109,8 @@ var _ = Describe("Disk image generation", Label("raw-disks"), func() { // Expect(out).To(ContainSubstring("gen-raw-mbr-disk"), out) // Expect(out).To(ContainSubstring("download-squashfs"), out) // Expect(out).To(ContainSubstring("extract-squashfs"), out) - // Expect(out).ToNot(ContainSubstring("container-pull"), out) - // Expect(err).ToNot(HaveOccurred()) + // Expect(out).ToNot(ContainSubstring("dump-source"), out) + // Expect(err).ToNot(HaveOccurred(), out) // _, err = os.Stat(filepath.Join(tempDir, "build/build/disk.raw.gce")) // Expect(err).ToNot(HaveOccurred()) // }) @@ -184,8 +184,8 @@ stages: Expect(out).To(ContainSubstring("Generating raw disk"), out) Expect(out).ToNot(ContainSubstring("build-arm-image"), out) Expect(out).To(ContainSubstring("gen-raw-disk"), out) - Expect(out).To(ContainSubstring("container-pull"), out) - Expect(err).ToNot(HaveOccurred()) + Expect(out).To(ContainSubstring("dump-source"), out) + Expect(err).ToNot(HaveOccurred(), out) _, err = os.Stat(filepath.Join(tempDir, "build/build/disk.raw")) Expect(err).ToNot(HaveOccurred()) }) @@ -205,8 +205,8 @@ stages: Expect(out).ToNot(ContainSubstring("build-arm-image"), out) Expect(out).To(ContainSubstring("gen-raw-disk"), out) Expect(out).To(ContainSubstring("convert-gce"), out) - Expect(out).To(ContainSubstring("container-pull"), out) - Expect(err).ToNot(HaveOccurred()) + Expect(out).To(ContainSubstring("dump-source"), out) + Expect(err).ToNot(HaveOccurred(), out) _, err = os.Stat(filepath.Join(tempDir, "build/build/disk.raw.gce")) Expect(err).ToNot(HaveOccurred()) }) @@ -226,8 +226,8 @@ stages: Expect(out).ToNot(ContainSubstring("build-arm-image"), out) Expect(out).To(ContainSubstring("gen-raw-disk"), out) Expect(out).To(ContainSubstring("convert-vhd"), out) - Expect(out).To(ContainSubstring("container-pull"), out) - Expect(err).ToNot(HaveOccurred()) + Expect(out).To(ContainSubstring("dump-source"), out) + Expect(err).ToNot(HaveOccurred(), out) _, err = os.Stat(filepath.Join(tempDir, "build/build/disk.raw.vhd")) Expect(err).ToNot(HaveOccurred()) }) @@ -247,8 +247,8 @@ stages: Expect(out).To(ContainSubstring("Generating MBR disk"), out) Expect(out).ToNot(ContainSubstring("build-arm-image"), out) Expect(out).To(ContainSubstring("gen-raw-mbr-disk"), out) - Expect(out).To(ContainSubstring("container-pull"), out) - Expect(err).ToNot(HaveOccurred()) + Expect(out).To(ContainSubstring("dump-source"), out) + Expect(err).ToNot(HaveOccurred(), out) _, err = os.Stat(filepath.Join(tempDir, "build/build/disk.raw")) Expect(err).ToNot(HaveOccurred()) }) diff --git a/tests/e2e/iso_test.go b/tests/e2e/iso_test.go index 31f14f4..272eec6 100644 --- a/tests/e2e/iso_test.go +++ b/tests/e2e/iso_test.go @@ -45,14 +45,16 @@ var _ = Describe("ISO image generation", Label("iso"), func() { }) It("fails if cloud config is empty", func() { + image := "quay.io/kairos/core-rockylinux:latest" + err := WriteConfig("", tempDir) Expect(err).ToNot(HaveOccurred()) - out, err := RunAurora(fmt.Sprintf(`--set container_image=quay.io/kairos/core-rockylinux:latest \ + out, err := RunAurora(fmt.Sprintf(`--set container_image=oci://%s \ --set "disable_http_server=true" \ --set "disable_netboot=true" \ --cloud-config /config.yaml \ - --set "state_dir=/tmp/auroraboot"`), tempDir) + --set "state_dir=/tmp/auroraboot"`, image), tempDir) Expect(err).To(HaveOccurred(), out) Expect(out).To(MatchRegexp("cloud config set but contents are empty")) })