PAD-173: 2 node HA - Tech Preview (#131)

* PAD-156: Add custom health check script
* PAD-156: Update user-data template, include health-check-script
* Add E2E automation, debug helpers (#94)
* automate e2e provisioning
* chore: bump provider-k3s version
---------

Signed-off-by: Oz Tiram <[email protected]>
Signed-off-by: Tyler Gillson <[email protected]>
Signed-off-by: Oz N Tiram <[email protected]>
Co-authored-by: Oz Tiram <[email protected]>

.gitignore

@@ -9,6 +9,12 @@ content-*/*
 *.arg
 .idea
 .DS_Store
+
+hack/*.img
+test/.env
+two-node-create.json
+two-node-update.json
+
 build/
 local/
 keys/

Dockerfile

@@ -17,7 +17,7 @@ RUN if [ "${OS_DISTRIBUTION}" = "opensuse-leap" ] && [ "${PROXY_CERT_PATH}" != "
     update-ca-certificates; \
     fi
 
-###########################Add any other image customizations here #######################
+########################### Add any other image customizations here #######################
 
 ####  Examples  ####
 
@@ -37,8 +37,18 @@ RUN if [ "${OS_DISTRIBUTION}" = "opensuse-leap" ] && [ "${PROXY_CERT_PATH}" != "
 #     && apt-get clean
 
 # RUN apt-get update && apt-get install nginx -y
-### or
 
 ### To install the nginx package for opensuse ###
 
 # RUN zypper refresh && zypper install nginx -y
+
+### To add a custom health script for two-node liveness checks ###
+
+# ADD overlay/files/opt/spectrocloud/bin/check-disk-size.sh /opt/spectrocloud/bin/
+
+### To install wifi prerequisites for Ubuntu ###
+
+# RUN apt-get update && apt-get install wpasupplicant -y && \
+#    apt-get update && apt-get install network-manager -y && \
+#    apt-get install iputils-ping -y && \
+#    mkdir /var/lib/wpa

Earthfile

@@ -2,15 +2,15 @@ VERSION 0.6
 ARG TARGETOS
 ARG TARGETARCH
 
-## Default Image Repos Used in the Builds. 
+# Default image repositories used in the builds.
 ARG ALPINE_IMG=gcr.io/spectro-images-public/alpine:3.16.2
 ARG SPECTRO_PUB_REPO=gcr.io/spectro-images-public
 ARG SPECTRO_LUET_REPO=gcr.io/spectro-dev-public
 ARG KAIROS_BASE_IMAGE_URL=gcr.io/spectro-images-public
 ARG ETCD_REPO=https://github.com/etcd-io
 FROM $SPECTRO_PUB_REPO/canvos/alpine-cert:v1.0.0
 
-## Spectro Cloud and Kairos Tags ##
+# Spectro Cloud and Kairos tags.
 ARG PE_VERSION=v4.4.1
 ARG SPECTRO_LUET_VERSION=v1.3.1
 ARG KAIROS_VERSION=v3.0.11
@@ -19,11 +19,11 @@ ARG RKE2_FLAVOR_TAG=rke2r1
 ARG BASE_IMAGE_URL=quay.io/kairos
 ARG OSBUILDER_VERSION=v0.201.0
 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION
-ARG K3S_PROVIDER_VERSION=v4.4.0
+ARG K3S_PROVIDER_VERSION=v4.4.1
 ARG KUBEADM_PROVIDER_VERSION=v4.4.0
 ARG RKE2_PROVIDER_VERSION=v4.4.0
 
-# Variables used in the builds.  Update for ADVANCED use cases only Modify in .arg file or via CLI arguements
+# Variables used in the builds. Update for ADVANCED use cases only. Modify in .arg file or via CLI arguments.
 ARG OS_DISTRIBUTION
 ARG OS_VERSION
 ARG K8S_VERSION
@@ -48,6 +48,11 @@ ARG no_proxy=${NO_PROXY}
 ARG PROXY_CERT_PATH
 
 ARG UPDATE_KERNEL=false
+ARG ETCD_VERSION="v3.5.13"
+
+# Two node variables
+ARG TWO_NODE=false
+ARG KINE_VERSION=0.11.4
 
 # UKI Variables
 ARG IS_UKI=false
@@ -57,7 +62,6 @@ ARG UKI_BRING_YOUR_OWN_KEYS=false
 
 ARG CMDLINE="stylus.registration"
 ARG BRANDING="Palette eXtended Kubernetes Edge"
-ARG ETCD_VERSION="v3.5.13"
 
 # EFI size check
 ARG EFI_MAX_SIZE=2048
@@ -530,7 +534,7 @@ secure-boot-dirs:
     RUN chmod 0644 /secure-boot/public-keys
     SAVE ARTIFACT --keep-ts /secure-boot AS LOCAL ./secure-boot
 
-# Used to create the provider images.  The --K8S_VERSION will be passed in the earthly build
+# Used to create the provider images. The --K8S_VERSION will be passed in the earthly build.
 provider-image:
     FROM --platform=linux/${ARCH} +base-image
     # added PROVIDER_K8S_VERSION to fix missing image in ghcr.io/kairos-io/provider-*
@@ -572,8 +576,30 @@ provider-image:
     RUN touch /etc/machine-id \
         && chmod 444 /etc/machine-id
 
-    SAVE IMAGE --push $IMAGE_PATH
+    IF $TWO_NODE
+        # Install postgresql 16
+        IF [ "$OS_DISTRIBUTION" = "ubuntu" ] &&  [ "$ARCH" = "amd64" ]
+            RUN apt install -y ca-certificates curl && \
+                install -d /usr/share/postgresql-common/pgdg && \
+                curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc && \
+                echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \
+                apt update && \
+                apt install -y postgresql-16 postgresql-contrib-16 iputils-ping
+        ELSE IF [ "$OS_DISTRIBUTION" = "opensuse-leap" ] && [ "$ARCH" = "amd64" ]
+            RUN zypper --non-interactive --quiet addrepo --refresh -p 90 http://download.opensuse.org/repositories/server:database:postgresql/openSUSE_Tumbleweed/ PostgreSQL && \
+                zypper --gpg-auto-import-keys ref && \
+                zypper install -y postgresql-16 postgresql-server-16 postgresql-contrib iputils
+        END
+
+        # Install kine
+        RUN mkdir -p /opt/spectrocloud/bin && \
+            curl -L https://github.com/k3s-io/kine/releases/download/v${KINE_VERSION}/kine-amd64 | install -m 755 /dev/stdin /opt/spectrocloud/bin/kine
+
+        # Ensure psql works ootb for the postgres user
+        RUN su postgres -c 'echo "export PERL5LIB=/usr/share/perl/5.34:/usr/share/perl5:/usr/lib/x86_64-linux-gnu/perl/5.34" > ~/.bash_profile'
+    END
 
+    SAVE IMAGE --push $IMAGE_PATH
 
 provider-image-rootfs:
     FROM --platform=linux/${ARCH} +provider-image
@@ -647,6 +673,7 @@ base-image:
         ARG BASE_K8S_VERSION=$K8S_VERSION-$K8S_DISTRIBUTION_TAG
     END
 
+    # OS == Ubuntu
     IF [ "$OS_DISTRIBUTION" = "ubuntu" ] &&  [ "$ARCH" = "amd64" ]
         IF [ ! -z "$UBUNTU_PRO_KEY" ]
             RUN sed -i '/^[[:space:]]*$/d' /etc/os-release && \
@@ -696,27 +723,27 @@ base-image:
             RUN pro detach --assume-yes
         END
 
-        # IF OS Type is Opensuse
+    # OS == Opensuse
     ELSE IF [ "$OS_DISTRIBUTION" = "opensuse-leap" ] && [ "$ARCH" = "amd64" ]
         # Add proxy certificate if present
         IF [ ! -z $PROXY_CERT_PATH ]
             COPY sc.crt /usr/share/pki/trust/anchors
-            RUN  update-ca-certificates
+            RUN update-ca-certificates
         END
         # Enable or Disable Kernel Updates
         IF [ "$UPDATE_KERNEL" = "false" ]
             RUN zypper al kernel-de*
         END
 
-        RUN zypper refresh && \
-            zypper update -y
+        RUN zypper refresh && zypper update -y
 
-           IF [ -e "/usr/bin/dracut" ]
-             RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && depmod -a "${kernel}"
-             RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && dracut -f "/boot/initrd-${kernel}" "${kernel}" && ln -sf "initrd-${kernel}" /boot/initrd
-           END
-        RUN zypper install -y zstd vim iputils bridge-utils curl ethtool tcpdump
-        RUN zypper cc && \
+        IF [ -e "/usr/bin/dracut" ]
+            RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && depmod -a "${kernel}"
+            RUN --no-cache kernel=$(ls /lib/modules | tail -n1) && dracut -f "/boot/initrd-${kernel}" "${kernel}" && ln -sf "initrd-${kernel}" /boot/initrd
+        END
+
+        RUN zypper install -y zstd vim iputils bridge-utils curl ethtool tcpdump && \
+            zypper cc && \
             zypper clean
     END
 
@@ -726,6 +753,7 @@ base-image:
             zypper clean
         RUN if [ ! -e /usr/bin/apparmor_parser ]; then cp /sbin/apparmor_parser /usr/bin/apparmor_parser; fi
     END
+
     IF [ "$ARCH" = "arm64" ]
         ARG LUET_REPO=luet-repo-arm
     ELSE IF [ "$ARCH" = "amd64" ]
@@ -738,7 +766,7 @@ base-image:
     RUN --no-cache if [ -f spectro-luet-auth.yaml ]; then cat spectro-luet-auth.yaml >> /etc/luet/repos.conf.d/spectro.yaml; fi
     RUN --no-cache luet repo update
 
-     IF [ "$OS_DISTRIBUTION" = "rhel" ]
+    IF [ "$OS_DISTRIBUTION" = "rhel" ]
         RUN yum install -y openssl
     END
 
@@ -762,7 +790,7 @@ base-image:
             if grep "selinux=1" /etc/cos/bootargs.cfg > /dev/null; then sed -i 's/selinux=1/selinux=0/g' /etc/cos/bootargs.cfg; fi
     END
 
-# Used to build the installer image.  The installer ISO will be created from this.
+# Used to build the installer image. The installer ISO will be created from this.
 iso-image:
     FROM --platform=linux/${ARCH} +base-image
     IF [ "$IS_UKI" = "false" ]

hack/Earthfile

@@ -0,0 +1,16 @@
+VERSION 0.6
+
+ARG OSBUILDER_VERSION=v0.7.11
+ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION
+ARG ISO_NAME=debug
+
+# replace with your CanvOS provider image
+ARG PROVIDER_IMAGE=oci:tylergillson/ubuntu:k3s-1.26.4-v4.0.4-071c2c23
+
+build:
+    FROM $OSBUILDER_IMAGE
+    WORKDIR /build
+    COPY . ./
+
+    RUN /entrypoint.sh --name $ISO_NAME --debug build-iso --squash-no-compression --date=false $PROVIDER_IMAGE --output /build/
+    SAVE ARTIFACT /build/$ISO_NAME.iso kairos.iso AS LOCAL build/$ISO_NAME.iso

hack/README.md

@@ -0,0 +1,19 @@
+# Debugging Kairos
+
+If you're facing hard-to-diagnose issues with your custom provider image, you can use the scripts in this directory to obtain verbose Kairos output.
+
+## Steps
+1. Use earthly to generate an ISO from your CanvOS provider image:
+    ```
+    earthly +build --PROVIDER_IMAGE=<your_provider_image>  # e.g., oci:tylergillson/ubuntu:k3s-1.26.4-v4.0.4-071c2c23
+    ```
+    If successful, `build/debug.iso` will be created.
+
+2. Launch a local VM based on the debug ISO using QEMU and pipe all output to a log file:
+    ```
+    ./launch-qemu.sh build/debug.iso | tee out.log
+    ```
+
+3. Boot the VM in `Kairos (manual)` mode. Once booted, create `userdata.yaml` with your desired Kairos config and execute a manual Kairos installation: `kairos-agent --debug manual-install --device auto userdata.yaml`.
+
+4. The VM should eventually reboot itself once the installation completes. Rather than waiting, execute `reboot` to return to the GRUB menu, select `Palette eXtended Kubernetes Edge` and hit `e` to edit it. Add `rd.debug rd.immucore.debug` to the end of the `linux` line, then hit `CTRL+x` to boot with your edits. You should see verbose Kairos debug logs and they will be persisted to `out.log`.

hack/launch-qemu.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Screenshot capability:
+# https://unix.stackexchange.com/a/476617
+
+if [ ! -e disk.img ]; then
+    qemu-img create -f qcow2 disk.img 60g
+fi
+
+#    -nic bridge,br=br0,model=virtio-net-pci \
+qemu-system-x86_64 \
+    -enable-kvm \
+    -cpu "${CPU:=host}" \
+    -nographic \
+    -spice port=9000,addr=127.0.0.1,disable-ticketing=yes \
+    -m ${MEMORY:=10096} \
+    -smp ${CORES:=5} \
+    -monitor unix:/tmp/qemu-monitor.sock,server=on,wait=off \
+    -serial mon:stdio \
+    -rtc base=utc,clock=rt \
+    -chardev socket,path=qga.sock,server=on,wait=off,id=qga0 \
+    -device virtio-serial \
+    -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \
+    -drive if=virtio,media=disk,file=disk.img \
+    -drive if=ide,media=cdrom,file="${1}"

overlay/files/opt/spectrocloud/bin/check-disk-size.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+REQUIRED_FREE_DISK=$1
+
+FREE=$(df -h --output=pcent /var/ | tail -n 1 | tr -d '\% ')
+
+if (( $FREE < $REQUIRED_FREE_DISK )); then
+   echo "Not enough free disk, required: $1. Free: $FREE"
+   exit 1
+fi
+
+echo "Free disk ok, required: $1. Free: $FREE"
+exit 0

test/env.example

@@ -0,0 +1,42 @@
+# govc vars
+export GOVC_USERNAME=<YOUR_NAME>@vsphere.local
+export GOVC_PASSWORD=<YOUR_VSPHERE_PASSWORD>
+export GOVC_URL=10.10.128.10 # IP address of USDC; edit as needed
+export GOVC_INSECURE=true
+export GOVC_DATACENTER=Datacenter
+export GOVC_DATASTORE=vsanDatastore2
+export GOVC_NETWORK=VM-NETWORK
+export GOVC_RESOURCE_POOL=<YOUR_RESOURCE_POOL>
+export GOVC_FOLDER=<YOUR_FOLDER>
+
+# vSphere vars
+export HOST_SUFFIX=<YOUR_NAME>-$(git -C ../stylus describe --always) # required to ensure unique edge host IDs
+export ISO_FOLDER=<YOUR_FOLDER> e.g. "ISO/01-tyler"
+export STYLUS_ISO="${ISO_FOLDER}/stylus-dev-amd64.iso"
+export NIC_NAME=ens160
+
+# networking vars
+export PROXY= # set any value to use Spectro's basic proxy
+export WIFI_NETWORK=
+export WIFI_PASSWORD=
+
+# palette vars
+export API_KEY=<YOUR_PALETTE_API_KEY>
+export PROJECT_UID=<YOUR_PROJECT_ID>
+export EDGE_REGISTRATION_TOKEN=<YOUR_REGISTRATION_TOKEN>
+export DOMAIN=dev.spectrocloud.com
+export PUBLIC_PACK_REPO_UID=<YOUR_PUBLIC_PACK_REPO_UID> # this varies per Palette tenant, identify via Chrome inspector on Tenant Admin -> Pack Registries page
+export CLUSTER_NAME=two-node-<YOUR_NAME>-$(git -C ../stylus describe --always)
+export CLUSTER_PROFILE_UID= # if left blank, a cluster profile will be created
+export CLUSTER_VIP= # choose an unassigned VIP
+
+# image vars
+export EARTHLY_BUILDKIT_CACHE_SIZE_MB=500000
+export OCI_REGISTRY=${OCI_REGISTRY:-ttl.sh}
+export STYLUS_BRANCH=${STYLUS_BRANCH:-2-node}
+export PROVIDER_K3S_BRANCH=${PROVIDER_K3S_BRANCH:-two-node}
+export K3S_VERSION="1.28.5"
+export PE_VERSION="4.3.0-2node"
+
+# two node vars
+export TWO_NODE_BACKEND=postgres