Merge branch 'main' into PE-3397

.arg.template

@@ -12,6 +12,7 @@ PROXY_CERT_PATH=
 UPDATE_KERNEL=false
 CLUSTERCONFIG=spc.tgz
 CIS_HARDENING=false
+EDGE_CUSTOM_CONFIG=.edge-custom-config.yaml
 
 # If you have Ubuntu Pro, use the UBUNTU_PRO_KEY variable to activate it as part of the image build
 # UBUNTU_PRO_KEY=your-key

.edge-custom-config.yaml.template

@@ -0,0 +1,5 @@
+content:
+ signing:
+  publicKey:
+   base64EncodedValue: ""
+   description: "This is a public key used for signing content and cluster-config"

.gitignore

@@ -9,6 +9,12 @@ content-*/*
 *.arg
 .idea
 .DS_Store
+
+hack/*.img
+test/.env
+two-node-create.json
+two-node-update.json
+
 build/
 local/
 keys/

Dockerfile

@@ -17,7 +17,7 @@ RUN if [ "${OS_DISTRIBUTION}" = "opensuse-leap" ] && [ "${PROXY_CERT_PATH}" != "
     update-ca-certificates; \
     fi
 
-###########################Add any other image customizations here #######################
+########################### Add any other image customizations here #######################
 
 ####  Examples  ####
 
@@ -37,8 +37,18 @@ RUN if [ "${OS_DISTRIBUTION}" = "opensuse-leap" ] && [ "${PROXY_CERT_PATH}" != "
 #     && apt-get clean
 
 # RUN apt-get update && apt-get install nginx -y
-### or
 
 ### To install the nginx package for opensuse ###
 
 # RUN zypper refresh && zypper install nginx -y
+
+### To add a custom health script for two-node liveness checks ###
+
+# ADD overlay/files/opt/spectrocloud/bin/check-disk-size.sh /opt/spectrocloud/bin/
+
+### To install wifi prerequisites for Ubuntu ###
+
+# RUN apt-get update && apt-get install wpasupplicant -y && \
+#    apt-get update && apt-get install network-manager -y && \
+#    apt-get install iputils-ping -y && \
+#    mkdir /var/lib/wpa

Earthfile

@@ -2,28 +2,28 @@ VERSION 0.6
 ARG TARGETOS
 ARG TARGETARCH
 
-## Default Image Repos Used in the Builds. 
+# Default image repositories used in the builds.
 ARG ALPINE_IMG=gcr.io/spectro-images-public/alpine:3.16.2
 ARG SPECTRO_PUB_REPO=gcr.io/spectro-images-public
 ARG SPECTRO_LUET_REPO=gcr.io/spectro-dev-public
 ARG KAIROS_BASE_IMAGE_URL=gcr.io/spectro-images-public
 ARG ETCD_REPO=https://github.com/etcd-io
 FROM $SPECTRO_PUB_REPO/canvos/alpine-cert:v1.0.0
 
-## Spectro Cloud and Kairos Tags ##
-ARG PE_VERSION=v4.4.1-rc5
+# Spectro Cloud and Kairos tags.
+ARG PE_VERSION=v4.4.1
 ARG SPECTRO_LUET_VERSION=v1.3.1
 ARG KAIROS_VERSION=v3.0.12
 ARG K3S_FLAVOR_TAG=k3s1
 ARG RKE2_FLAVOR_TAG=rke2r1
 ARG BASE_IMAGE_URL=quay.io/kairos
 ARG OSBUILDER_VERSION=v0.201.0
 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION
-ARG K3S_PROVIDER_VERSION=v4.4.0
+ARG K3S_PROVIDER_VERSION=v4.4.1
 ARG KUBEADM_PROVIDER_VERSION=v4.4.0
 ARG RKE2_PROVIDER_VERSION=v4.4.0
 
-# Variables used in the builds.  Update for ADVANCED use cases only Modify in .arg file or via CLI arguements
+# Variables used in the builds. Update for ADVANCED use cases only. Modify in .arg file or via CLI arguments.
 ARG OS_DISTRIBUTION
 ARG OS_VERSION
 ARG K8S_VERSION
@@ -33,6 +33,7 @@ ARG ISO_NAME=installer
 ARG K8S_DISTRIBUTION
 ARG CUSTOM_TAG
 ARG CLUSTERCONFIG
+ARG EDGE_CUSTOM_CONFIG=.edge-custom-config.yaml
 ARG ARCH
 ARG DISABLE_SELINUX=true
 ARG CIS_HARDENING=true
@@ -48,6 +49,11 @@ ARG no_proxy=${NO_PROXY}
 ARG PROXY_CERT_PATH
 
 ARG UPDATE_KERNEL=false
+ARG ETCD_VERSION="v3.5.13"
+
+# Two node variables
+ARG TWO_NODE=false
+ARG KINE_VERSION=0.11.4
 
 # UKI Variables
 ARG IS_UKI=false
@@ -57,7 +63,6 @@ ARG UKI_BRING_YOUR_OWN_KEYS=false
 
 ARG CMDLINE="stylus.registration"
 ARG BRANDING="Palette eXtended Kubernetes Edge"
-ARG ETCD_VERSION="v3.5.13"
 
 # EFI size check
 ARG EFI_MAX_SIZE=2048
@@ -270,6 +275,7 @@ uki-provider-image:
     COPY +kairos-agent/kairos-agent /usr/bin/kairos-agent
     COPY --platform=linux/${ARCH} +trust-boot-unpack/ /trusted-boot
     COPY --platform=linux/${ARCH} +install-k8s/ /k8s
+    COPY --if-exists "$EDGE_CUSTOM_CONFIG" /oem/.edge_custom_config.yaml
     SAVE IMAGE --push $IMAGE_PATH
 
 trust-boot-unpack:
@@ -339,6 +345,7 @@ build-uki-iso:
     COPY --platform=linux/${ARCH} +luet/luet /overlay/luet
 
     COPY --if-exists content-*/*.zst /overlay/opt/spectrocloud/content/
+    COPY --if-exists "$EDGE_CUSTOM_CONFIG" /overlay/.edge_custom_config.yaml
     RUN if [ -n "$(ls /overlay/opt/spectrocloud/content/*.zst 2>/dev/null)" ]; then \
         for file in /overlay/opt/spectrocloud/content/*.zst; do \
             split --bytes=3GB --numeric-suffixes "$file" /overlay/opt/spectrocloud/content/$(basename "$file")_part; \
@@ -392,6 +399,7 @@ build-iso:
     COPY overlay/files-iso/ /overlay/
     COPY --if-exists user-data /overlay/files-iso/config.yaml
     COPY --if-exists content-*/*.zst /overlay/opt/spectrocloud/content/
+    COPY --if-exists "$EDGE_CUSTOM_CONFIG" /overlay/.edge_custom_config.yaml
     RUN if [ -n "$(ls /overlay/opt/spectrocloud/content/*.zst 2>/dev/null)" ]; then \
         for file in /overlay/opt/spectrocloud/content/*.zst; do \
             split --bytes=3GB --numeric-suffixes "$file" /overlay/opt/spectrocloud/content/$(basename "$file")_part; \
@@ -530,7 +538,7 @@ secure-boot-dirs:
     RUN chmod 0644 /secure-boot/public-keys
     SAVE ARTIFACT --keep-ts /secure-boot AS LOCAL ./secure-boot
 
-# Used to create the provider images.  The --K8S_VERSION will be passed in the earthly build
+# Used to create the provider images. The --K8S_VERSION will be passed in the earthly build.
 provider-image:
     FROM --platform=linux/${ARCH} +base-image
     # added PROVIDER_K8S_VERSION to fix missing image in ghcr.io/kairos-io/provider-*
@@ -553,6 +561,7 @@ provider-image:
     COPY +stylus-image/etc/kairos/branding /etc/kairos/branding
     COPY +stylus-image/oem/stylus_config.yaml /etc/kairos/branding/stylus_config.yaml
     COPY +stylus-image/etc/elemental/config.yaml /etc/elemental/config.yaml
+    COPY --if-exists "$EDGE_CUSTOM_CONFIG" /oem/.edge_custom_config.yaml
 
     IF [ "$IS_UKI" = "true" ]
         COPY +internal-slink/slink /usr/bin/slink
@@ -572,8 +581,30 @@ provider-image:
     RUN touch /etc/machine-id \
         && chmod 444 /etc/machine-id
 
-    SAVE IMAGE --push $IMAGE_PATH
+    IF $TWO_NODE
+        # Install postgresql 16
+        IF [ "$OS_DISTRIBUTION" = "ubuntu" ] &&  [ "$ARCH" = "amd64" ]
+            RUN apt install -y ca-certificates curl && \
+                install -d /usr/share/postgresql-common/pgdg && \
+                curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc && \
+                echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \
+                apt update && \
+                apt install -y postgresql-16 postgresql-contrib-16 iputils-ping
+        ELSE IF [ "$OS_DISTRIBUTION" = "opensuse-leap" ] && [ "$ARCH" = "amd64" ]
+            RUN zypper --non-interactive --quiet addrepo --refresh -p 90 http://download.opensuse.org/repositories/server:database:postgresql/openSUSE_Tumbleweed/ PostgreSQL && \
+                zypper --gpg-auto-import-keys ref && \
+                zypper install -y postgresql-16 postgresql-server-16 postgresql-contrib iputils
+        END
+
+        # Install kine
+        RUN mkdir -p /opt/spectrocloud/bin && \
+            curl -L https://github.com/k3s-io/kine/releases/download/v${KINE_VERSION}/kine-amd64 | install -m 755 /dev/stdin /opt/spectrocloud/bin/kine
+
+        # Ensure psql works ootb for the postgres user
+        RUN su postgres -c 'echo "export PERL5LIB=/usr/share/perl/5.34:/usr/share/perl5:/usr/lib/x86_64-linux-gnu/perl/5.34" > ~/.bash_profile'
+    END
 
+    SAVE IMAGE --push $IMAGE_PATH
 
 provider-image-rootfs:
     FROM --platform=linux/${ARCH} +provider-image
@@ -647,6 +678,7 @@ base-image:
         ARG BASE_K8S_VERSION=$K8S_VERSION-$K8S_DISTRIBUTION_TAG
     END
 
+    # OS == Ubuntu
     IF [ "$OS_DISTRIBUTION" = "ubuntu" ] &&  [ "$ARCH" = "amd64" ]
         IF [ ! -z "$UBUNTU_PRO_KEY" ]
             RUN sed -i '/^[[:space:]]*$/d' /etc/os-release && \
@@ -696,27 +728,27 @@ base-image:
             RUN pro detach --assume-yes
         END
 
-        # IF OS Type is Opensuse
+    # OS == Opensuse
     ELSE IF [ "$OS_DISTRIBUTION" = "opensuse-leap" ] && [ "$ARCH" = "amd64" ]
         # Add proxy certificate if present
         IF [ ! -z $PROXY_CERT_PATH ]
             COPY sc.crt /usr/share/pki/trust/anchors
-            RUN  update-ca-certificates
+            RUN update-ca-certificates
         END
         # Enable or Disable Kernel Updates
         IF [ "$UPDATE_KERNEL" = "false" ]
             RUN zypper al kernel-de*
         END
 
-        RUN zypper refresh && \
-            zypper update -y
+        RUN zypper refresh && zypper update -y
 
-           IF [ -e "/usr/bin/dracut" ]
-             RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && depmod -a "${kernel}"
-             RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && dracut -f "/boot/initrd-${kernel}" "${kernel}" && ln -sf "initrd-${kernel}" /boot/initrd
-           END
-        RUN zypper install -y zstd vim iputils bridge-utils curl ethtool tcpdump
-        RUN zypper cc && \
+        IF [ -e "/usr/bin/dracut" ]
+            RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && depmod -a "${kernel}"
+            RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && dracut -f "/boot/initrd-${kernel}" "${kernel}" && ln -sf "initrd-${kernel}" /boot/initrd
+        END
+
+        RUN zypper install -y zstd vim iputils bridge-utils curl ethtool tcpdump && \
+            zypper cc && \
             zypper clean
     END
 
@@ -726,6 +758,7 @@ base-image:
             zypper clean
         RUN if [ ! -e /usr/bin/apparmor_parser ]; then cp /sbin/apparmor_parser /usr/bin/apparmor_parser; fi
     END
+
     IF [ "$ARCH" = "arm64" ]
         ARG LUET_REPO=luet-repo-arm
     ELSE IF [ "$ARCH" = "amd64" ]
@@ -738,7 +771,7 @@ base-image:
     RUN --no-cache if [ -f spectro-luet-auth.yaml ]; then cat spectro-luet-auth.yaml >> /etc/luet/repos.conf.d/spectro.yaml; fi
     RUN --no-cache luet repo update
 
-     IF [ "$OS_DISTRIBUTION" = "rhel" ]
+    IF [ "$OS_DISTRIBUTION" = "rhel" ]
         RUN yum install -y openssl
     END
 
@@ -762,7 +795,7 @@ base-image:
             if grep "selinux=1" /etc/cos/bootargs.cfg > /dev/null; then sed -i 's/selinux=1/selinux=0/g' /etc/cos/bootargs.cfg; fi
     END
 
-# Used to build the installer image.  The installer ISO will be created from this.
+# Used to build the installer image. The installer ISO will be created from this.
 iso-image:
     FROM --platform=linux/${ARCH} +base-image
     IF [ "$IS_UKI" = "false" ]

README.md

@@ -138,6 +138,7 @@ cp .arg.template .arg
 | UKI_BRING_YOUR_OWN_KEYS     | Bring your own public/private key pairs if this is set to true. Otherwise, CanvOS will generate the key pair.                                                                                                                                                                                                                                  | boolean | `false`                  |
 | INCLUDE_MS_SECUREBOOT_KEYS  | Include Microsoft 3rd Party UEFI CA certificate in generated keys                                                                                                                                                                                                                                                                              | boolean | `true`                   |
 | AUTO_ENROLL_SECUREBOOT_KEYS | Auto enroll SecureBoot keys when device boots up and is in setup mode of secure boot                                                                                                                                                                                                                                                           | boolean | `true`                   |
+| EDGE_CUSTOM_CONFIG | Path to edge custom configuration file                                                                                                                                                                                                                                                            | string | `.edge-custom-config.yaml`                   |
 
 1. (Optional) If you are building the images behind a proxy server, you may need to modify your docker daemon settings to let it use your proxy server. You can refer this [tutorial](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy).
 
@@ -322,3 +323,26 @@ cp spectro-luet-auth.yaml.template spectro-luet-auth.yaml
 ```shell
 earthly --push +build-all-images
 ```
+
+### Building Installer Image with public key for verifying signed content
+
+1. Copy the .edge.custom-config.yaml.template file to .edge.custom-config.yaml 
+
+```shell
+cp .edge.custom-config.yaml.template .edge.custom-config.yaml
+```
+
+2. Edit the property signing.publicKey in `.edge.custom-config.yaml`
+
+3. Include the following property in `.arg` file
+```
+...
+
+EDGE_CUSTOM_CONFIG=/path/to/.edge.custom-config.yaml
+```
+
+4. Build the image using earthly installed on the host
+
+```shell
+earthly --push +build-all-images
+```

hack/Earthfile

@@ -0,0 +1,16 @@
+VERSION 0.6
+
+ARG OSBUILDER_VERSION=v0.7.11
+ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION
+ARG ISO_NAME=debug
+
+# replace with your CanvOS provider image
+ARG PROVIDER_IMAGE=oci:tylergillson/ubuntu:k3s-1.26.4-v4.0.4-071c2c23
+
+build:
+    FROM $OSBUILDER_IMAGE
+    WORKDIR /build
+    COPY . ./
+
+    RUN /entrypoint.sh --name $ISO_NAME --debug build-iso --squash-no-compression --date=false $PROVIDER_IMAGE --output /build/
+    SAVE ARTIFACT /build/$ISO_NAME.iso kairos.iso AS LOCAL build/$ISO_NAME.iso

hack/README.md

@@ -0,0 +1,19 @@
+# Debugging Kairos
+
+If you're facing hard-to-diagnose issues with your custom provider image, you can use the scripts in this directory to obtain verbose Kairos output.
+
+## Steps
+1. Use earthly to generate an ISO from your CanvOS provider image:
+    ```
+    earthly +build --PROVIDER_IMAGE=<your_provider_image>  # e.g., oci:tylergillson/ubuntu:k3s-1.26.4-v4.0.4-071c2c23
+    ```
+    If successful, `build/debug.iso` will be created.
+
+2. Launch a local VM based on the debug ISO using QEMU and pipe all output to a log file:
+    ```
+    ./launch-qemu.sh build/debug.iso | tee out.log
+    ```
+
+3. Boot the VM in `Kairos (manual)` mode. Once booted, create `userdata.yaml` with your desired Kairos config and execute a manual Kairos installation: `kairos-agent --debug manual-install --device auto userdata.yaml`.
+
+4. The VM should eventually reboot itself once the installation completes. Rather than waiting, execute `reboot` to return to the GRUB menu, select `Palette eXtended Kubernetes Edge` and hit `e` to edit it. Add `rd.debug rd.immucore.debug` to the end of the `linux` line, then hit `CTRL+x` to boot with your edits. You should see verbose Kairos debug logs and they will be persisted to `out.log`.

hack/launch-qemu.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Screenshot capability:
+# https://unix.stackexchange.com/a/476617
+
+if [ ! -e disk.img ]; then
+    qemu-img create -f qcow2 disk.img 60g
+fi
+
+#    -nic bridge,br=br0,model=virtio-net-pci \
+qemu-system-x86_64 \
+    -enable-kvm \
+    -cpu "${CPU:=host}" \
+    -nographic \
+    -spice port=9000,addr=127.0.0.1,disable-ticketing=yes \
+    -m ${MEMORY:=10096} \
+    -smp ${CORES:=5} \
+    -monitor unix:/tmp/qemu-monitor.sock,server=on,wait=off \
+    -serial mon:stdio \
+    -rtc base=utc,clock=rt \
+    -chardev socket,path=qga.sock,server=on,wait=off,id=qga0 \
+    -device virtio-serial \
+    -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \
+    -drive if=virtio,media=disk,file=disk.img \
+    -drive if=ide,media=cdrom,file="${1}"

overlay/files/opt/spectrocloud/bin/check-disk-size.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+REQUIRED_FREE_DISK=$1
+
+FREE=$(df -h --output=pcent /var/ | tail -n 1 | tr -d '\% ')
+
+if (( $FREE < $REQUIRED_FREE_DISK )); then
+   echo "Not enough free disk, required: $1. Free: $FREE"
+   exit 1
+fi
+
+echo "Free disk ok, required: $1. Free: $FREE"
+exit 0

rhel-fips/Dockerfile

@@ -6,15 +6,14 @@ ARG USERNAME
 ARG PASSWORD
 
 # Generate os-release file
-FROM quay.io/kairos/osbuilder-tools:v0.200.11 as osbuilder
+FROM quay.io/kairos/osbuilder-tools:v0.7.11 as osbuilder
 RUN zypper install -y gettext && zypper clean
 RUN mkdir /workspace
 COPY --from=base /etc/os-release /workspace/os-release
 # You should change the following values according to your own versioning and other details
 RUN OS_NAME=kairos-core-rhel-fips \
-  OS_VERSION=v1.0.0 \
+  OS_VERSION=v3.0.11 \
   OS_ID="kairos" \
-  OS_NAME=kairos-fedora-fips \
   BUG_REPORT_URL="https://github.com/spectrocloud/pxke-samples/issues" \
   HOME_URL="https://github.com/spectrocloud/pxke-samples" \
   OS_REPO="quay.io/spectrocloud/fedora-fips" \