spectrocloud · TylerGillson · Jul 4, 2024 · Jul 8, 2024 · Jul 8, 2024 · Jul 8, 2024
diff --git a/.gitignore b/.gitignore
@@ -11,6 +11,7 @@ content-*/*
 .DS_Store
 
 hack/*.img
+hack/eks-hybrid
 test/.env
 two-node-create.json
 two-node-update.json

diff --git a/Earthfile b/Earthfile
@@ -21,7 +21,7 @@ ARG RKE2_FLAVOR_TAG=rke2r1
 ARG BASE_IMAGE_URL=quay.io/kairos
 ARG OSBUILDER_VERSION=v0.201.0
 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION
-ARG K3S_PROVIDER_VERSION=v4.4.2
+ARG K3S_PROVIDER_VERSION=v4.4.2-2node
 ARG KUBEADM_PROVIDER_VERSION=v4.4.1
 ARG RKE2_PROVIDER_VERSION=v4.4.1
 
@@ -54,7 +54,7 @@ ARG UPDATE_KERNEL=false
 ARG ETCD_VERSION="v3.5.13"
 
 # Two node variables
-ARG TWO_NODE=false
+ARG TWO_NODE=true
 ARG KINE_VERSION=0.11.4
 
 # UKI Variables

diff --git a/test/env.example b/test/env.example
@@ -28,15 +28,15 @@ export DOMAIN=dev.spectrocloud.com
 export PUBLIC_PACK_REPO_UID=<YOUR_PUBLIC_PACK_REPO_UID> # this varies per Palette tenant, identify via Chrome inspector on Tenant Admin -> Pack Registries page
 export CLUSTER_NAME=two-node-<YOUR_NAME>-$(git -C ../stylus describe --always)
 export CLUSTER_PROFILE_UID= # if left blank, a cluster profile will be created
-export CLUSTER_VIP= # choose an unassigned VIP
+export CLUSTER_VIP= # choose an unassigned VIP from your static IP range
 
 # image vars
 export EARTHLY_BUILDKIT_CACHE_SIZE_MB=500000
 export OCI_REGISTRY=${OCI_REGISTRY:-ttl.sh}
 export STYLUS_BRANCH=${STYLUS_BRANCH:-2-node}
 export PROVIDER_K3S_BRANCH=${PROVIDER_K3S_BRANCH:-two-node}
-export K3S_VERSION="1.28.5"
-export PE_VERSION="4.3.0-2node"
+export PE_VERSION=4.4.12
+export K8S_VERSION=1.29.6
 
 # two node vars
 export TWO_NODE_BACKEND=postgres
diff --git a/test/templates/two-node-cluster-profile.json.tmpl b/test/templates/two-node-cluster-profile.json.tmpl
@@ -16,7 +16,7 @@
                     "layer": "os",
                     "version": "1.0.0",
                     "tag": "1.0.0",
-                    "values": "pack:\n  content:\n    images:\n      - image: \"{{.spectro.pack.edge-native-byoi.options.system.uri}}\"\n  # Below config is default value, please uncomment if you want to modify default values\n  #drain:\n    #cordon: true\n    #timeout: 60 # The length of time to wait before giving up, zero means infinite\n    #gracePeriod: 60 # Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used\n    #ignoreDaemonSets: true\n    #deleteLocalData: true # Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained)\n    #force: true # Continue even if there are pods that do not declare a controller\n    #disableEviction: false # Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution\n    #skipWaitForDeleteTimeout: 60 # If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.\nstylusPackage: container://OCI_REGISTRY/stylus-linux-amd64:v0.0.0-STYLUS_HASH\noptions:\n  system.uri: \"OCI_REGISTRY/ubuntu:k3s-K3S_VERSION-vPE_VERSION-STYLUS_HASH\"",
+                    "values": "pack:\n  content:\n    images:\n      - image: \"{{.spectro.pack.edge-native-byoi.options.system.uri}}\"\n  # Below config is default value, please uncomment if you want to modify default values\n  #drain:\n    #cordon: true\n    #timeout: 60 # The length of time to wait before giving up, zero means infinite\n    #gracePeriod: 60 # Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used\n    #ignoreDaemonSets: true\n    #deleteLocalData: true # Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained)\n    #force: true # Continue even if there are pods that do not declare a controller\n    #disableEviction: false # Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution\n    #skipWaitForDeleteTimeout: 60 # If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.\nstylusPackage: container://OCI_REGISTRY/stylus-linux-amd64:v0.0.0-STYLUS_HASH\noptions:\n  system.uri: \"OCI_REGISTRY/ubuntu:k3s-K8S_VERSION-vPE_VERSION-STYLUS_HASH\"",
                     "registry": {
                         "metadata": {
                             "uid": "_____place_holder_____",

diff --git a/test/templates/two-node-master-master.json.tmpl b/test/templates/two-node-master-master.json.tmpl
@@ -67,7 +67,7 @@
                                                 "tag": "1.0.0",
                                                 "name": "edge-native-byoi",
                                                 "type": "spectro",
-                                                "values": "pack:\n  content:\n    images:\n      - image: \"{{.spectro.pack.edge-native-byoi.options.system.uri}}\"\n  # Below config is default value, please uncomment if you want to modify default values\n  #drain:\n    #cordon: true\n    #timeout: 60 # The length of time to wait before giving up, zero means infinite\n    #gracePeriod: 60 # Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used\n    #ignoreDaemonSets: true\n    #deleteLocalData: true # Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained)\n    #force: true # Continue even if there are pods that do not declare a controller\n    #disableEviction: false # Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution\n    #skipWaitForDeleteTimeout: 60 # If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.\nstylusPackage: container://OCI_REGISTRY/stylus-linux-amd64:v0.0.0-STYLUS_HASH\noptions:\n  system.uri: \"OCI_REGISTRY/ubuntu:k3s-K3S_VERSION-vPE_VERSION-STYLUS_HASH\"",
+                                                "values": "pack:\n  content:\n    images:\n      - image: \"{{.spectro.pack.edge-native-byoi.options.system.uri}}\"\n  # Below config is default value, please uncomment if you want to modify default values\n  #drain:\n    #cordon: true\n    #timeout: 60 # The length of time to wait before giving up, zero means infinite\n    #gracePeriod: 60 # Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used\n    #ignoreDaemonSets: true\n    #deleteLocalData: true # Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained)\n    #force: true # Continue even if there are pods that do not declare a controller\n    #disableEviction: false # Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution\n    #skipWaitForDeleteTimeout: 60 # If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.\nstylusPackage: container://OCI_REGISTRY/stylus-linux-amd64:v0.0.0-STYLUS_HASH\noptions:\n  system.uri: \"OCI_REGISTRY/ubuntu:k3s-K8S_VERSION-vPE_VERSION-STYLUS_HASH\"",
                                                 "manifests": []
                                         },
                                         {

diff --git a/test/test-two-node.sh b/test/test-two-node.sh
@@ -2,6 +2,8 @@
 
 set -e
 
+source .arg
+
 # Usage
 # -----
 #
@@ -14,21 +16,24 @@ set -e
 #    - jq (https://jqlang.github.io/jq/download/)
 #    - mkisofs (https://command-not-found.com/mkisofs)
 #
-# 2. Clone CanvOS and checkout this branch.
+# 2. Clone CanvOS and 'git checkout' this branch.
+#
+# 3. Configure your Earthly argument file by running: cp .arg.template .arg.
+#    Ensure the following vars are customized:
 #
-# 3. Configure your Earthly argument file by running: cp .arg.template .arg
-#    No modifications to the template are required.
+#    K3S_PROVIDER_VERSION=v4.4.2-2node
+#    K8S_VERSION=1.28.7 (pick any supported K3s version)
 #
 # 4. Create a .netrc file in the stylus repo root with GitHub
 #    credentials capable of cloning Spectro Cloud internal repos.
 #
 # 5. Copy the test/env.example file to test/.env and edit test/.env
 #    as required.
 #
-# 6. Source and execute this script:
+# 6. Source this script and execute 'deploy':
 #
 #    source ./test/test-two-node.sh
-#    ./test/test-two-node.sh
+#    deploy
 
 # Do not edit anything below
 
@@ -78,6 +83,9 @@ stylus:
   users:
   - name: kairos
     passwd: kairos
+  includeTui: true
+  twoNode:
+    enable: true
   site:
     edgeHostToken: "$EDGE_REGISTRATION_TOKEN"
     paletteEndpoint: "$DOMAIN"
@@ -321,24 +329,23 @@ function prepare_cluster_profile() {
     jq '
       .metadata.name = env.CLUSTER_NAME |
       .spec.template.packs[0].registry.metadata.uid = env.PUBLIC_PACK_REPO_UID |
-      .spec.template.packs[1].version = env.K3S_VERSION |
-      .spec.template.packs[1].tag = env.K3S_VERSION |
+      .spec.template.packs[1].version = env.K8S_VERSION |
+      .spec.template.packs[1].tag = env.K8S_VERSION |
       .spec.template.packs[1].registry.metadata.uid = env.PUBLIC_PACK_REPO_UID |
       .spec.template.packs[2].registry.metadata.uid = env.PUBLIC_PACK_REPO_UID |
       .spec.template.packs[0].values |= gsub("OCI_REGISTRY"; env.OCI_REGISTRY) |
       .spec.template.packs[0].values |= gsub("PE_VERSION"; env.PE_VERSION) |
-      .spec.template.packs[0].values |= gsub("K3S_VERSION"; env.K3S_VERSION) |
+      .spec.template.packs[0].values |= gsub("K8S_VERSION"; env.K8S_VERSION) |
       .spec.template.packs[0].values |= gsub("STYLUS_HASH"; env.STYLUS_HASH)
-    ' test/templates/two-node-cluster-profile.json.tmpl > two-node-cluster-profile.json
+    ' test/templates/two-node-cluster-profile.json.tmpl > /tmp/two-node-cluster-profile.json
 }
 
 function create_cluster_profile() {
     export CLUSTER_PROFILE_UID=$(curl -s -X POST https://$DOMAIN/v1/clusterprofiles/import?publish=true \
         -H "ApiKey: $API_KEY" \
         -H "Content-Type: application/json" \
         -H "ProjectUid: $PROJECT_UID" \
-        -d @two-node-cluster-profile.json | jq -r .uid)
-    rm -f two-node-cluster-profile.json
+        -d @/tmp/two-node-cluster-profile.json | jq -r .uid)
     if [ "$CLUSTER_PROFILE_UID" = "null" ]; then
         echo Cluster Profile creation failed as it already exists. Please delete it and retry.
         return 1
@@ -376,23 +383,22 @@ function prepare_master_master_cluster() {
       .spec.profiles[0].uid = env.CLUSTER_PROFILE_UID |
       .spec.profiles[0].packValues[0].values |= gsub("OCI_REGISTRY"; env.OCI_REGISTRY) |
       .spec.profiles[0].packValues[0].values |= gsub("PE_VERSION"; env.PE_VERSION) |
-      .spec.profiles[0].packValues[0].values |= gsub("K3S_VERSION"; env.K3S_VERSION) |
+      .spec.profiles[0].packValues[0].values |= gsub("K8S_VERSION"; env.K8S_VERSION) |
       .spec.profiles[0].packValues[0].values |= gsub("STYLUS_HASH"; env.STYLUS_HASH) |
-      .spec.profiles[0].packValues[1].tag = env.K3S_VERSION
-    ' test/templates/two-node-master-master.json.tmpl > two-node-create.json
+      .spec.profiles[0].packValues[1].tag = env.K8S_VERSION
+    ' test/templates/two-node-master-master.json.tmpl > /tmp/two-node-create.json
 }
 
 function create_cluster() {
     uid=$(curl -s -X POST https://$DOMAIN/v1/spectroclusters/edge-native?ProjectUid=$PROJECT_UID \
         -H "ApiKey: $API_KEY" \
         -H "Content-Type: application/json" \
         -H "ProjectUid: $PROJECT_UID" \
-        -d @two-node-create.json | jq -r .uid)
+        -d @/tmp/two-node-create.json | jq -r .uid)
     if [ "$uid" = "null" ]; then
         echo "Cluster creation failed. Please check two-node-create.json and retry creation manually to see Hubble's response."
         return 1
     else
-        rm -f two-node-create.json
         echo "Cluster $uid created"
     fi
 }
@@ -443,37 +449,32 @@ function update_cluster() {
 
 function build_provider_k3s() {
     echo "Building provider-k3s image..."
-    earthly +build-provider-package \
+    earthly --push +build-provider-package \
         --platform=linux/amd64 \
         --IMAGE_REPOSITORY=${OCI_REGISTRY} \
         --VERSION=${PROVIDER_K3S_HASH}
-    docker push ${OCI_REGISTRY}/provider-k3s:${PROVIDER_K3S_HASH}
 }
 
 function build_stylus_package_and_framework() {
     echo "Building stylus image and stylus framework image..."
-    earthly --allow-privileged +package \
+    earthly --allow-privileged --push +package \
         --platform=linux/amd64 \
         --IMAGE_REPOSITORY=${OCI_REGISTRY} \
         --BASE_IMAGE=quay.io/kairos/core-opensuse-leap:v2.3.2 \
         --VERSION=v0.0.0-${STYLUS_HASH}
-    docker push ${OCI_REGISTRY}/stylus-linux-amd64:v0.0.0-${STYLUS_HASH}
-    docker push ${OCI_REGISTRY}/stylus-framework-linux-amd64:v0.0.0-${STYLUS_HASH}
 }
 
 function build_canvos() {
     echo "Building provider image & installer ISO..."
-    earthly +build-all-images \
+    earthly --push +build-all-images \
         --ARCH=amd64 \
         --PROVIDER_BASE=${OCI_REGISTRY}/provider-k3s:${PROVIDER_K3S_HASH} \
         --STYLUS_BASE=${OCI_REGISTRY}/stylus-framework-linux-amd64:v0.0.0-${STYLUS_HASH} \
         --ISO_NAME=palette-edge-installer-stylus-${STYLUS_HASH}-k3s-${PROVIDER_K3S_HASH} \
         --IMAGE_REGISTRY=${OCI_REGISTRY} \
         --TWO_NODE=true \
         --TWO_NODE_BACKEND=${TWO_NODE_BACKEND} \
-        --CUSTOM_TAG=${STYLUS_HASH} \
-	--PE_VERSION=v${PE_VERSION}
-    docker push ${OCI_REGISTRY}/ubuntu:k3s-${K3S_VERSION}-v${PE_VERSION}-${STYLUS_HASH}
+        --CUSTOM_TAG=${STYLUS_HASH}
 }
 
 function build_all() {
@@ -501,7 +502,7 @@ function build_all() {
     (
         test -f build/palette-edge-installer-stylus-${STYLUS_HASH}-k3s-${PROVIDER_K3S_HASH}.iso && \
         docker image ls --format "{{.Repository}}:{{.Tag}}" | \
-        grep -q ${OCI_REGISTRY}/ubuntu:k3s-${K3S_VERSION}-v${PE_VERSION}-${STYLUS_HASH}
+        grep -q ${OCI_REGISTRY}/ubuntu:k3s-${K8S_VERSION}-v${PE_VERSION}-${STYLUS_HASH}
     ) || ( build_canvos )
 }
 
@@ -512,6 +513,36 @@ function clean_all() {
     docker system prune --all --volumes --force
 }
 
+# Builds 2-node provider image and ISO, uploads the ISO to vCenter, and provisions edge hosts
+# for deploying a 2-node cluster.
+function deploy() {
+    set +e
+
+    earthly --push +build-all-images \
+        --ARCH=amd64 \
+        --ISO_NAME=palette-edge-installer-stylus-k3s-twonode \
+        --IMAGE_REGISTRY=${OCI_REGISTRY} \
+        --TWO_NODE=true \
+        --TWO_NODE_BACKEND=${TWO_NODE_BACKEND} \
+        --CUSTOM_TAG=twonode
+
+    # create & upload user-data ISOs, configured to enable two node mode
+    create_userdata_iso
+    upload_userdata_iso
+
+    echo Uploading installer ISO $iso...
+    iso=palette-edge-installer-stylus-k3s-twonode.iso
+    govc datastore.upload --ds=$GOVC_DATASTORE --dc=$GOVC_DATACENTER build/$iso $STYLUS_ISO
+
+    vm_array=("two-node-sep16-1" "two-node-sep16-2")
+    create_vms
+    wait_for_vms_to_power_off
+    reboot_vms
+
+    echo Edge hosts should register with Palette shortly. Please proceed to deploying an Edge cluster via the UI.
+    set -e
+}
+
 function main() {
     init_globals