diff --git a/.gitignore b/.gitignore index c67f2b5..d0c5b46 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ content-*/* .DS_Store hack/*.img +hack/eks-hybrid test/.env two-node-create.json two-node-update.json diff --git a/Earthfile b/Earthfile index 675f633..065ab91 100644 --- a/Earthfile +++ b/Earthfile @@ -21,7 +21,7 @@ ARG RKE2_FLAVOR_TAG=rke2r1 ARG BASE_IMAGE_URL=quay.io/kairos ARG OSBUILDER_VERSION=v0.201.0 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION -ARG K3S_PROVIDER_VERSION=v4.4.2 +ARG K3S_PROVIDER_VERSION=v4.4.2-2node ARG KUBEADM_PROVIDER_VERSION=v4.4.1 ARG RKE2_PROVIDER_VERSION=v4.4.1 @@ -54,7 +54,7 @@ ARG UPDATE_KERNEL=false ARG ETCD_VERSION="v3.5.13" # Two node variables -ARG TWO_NODE=false +ARG TWO_NODE=true ARG KINE_VERSION=0.11.4 # UKI Variables diff --git a/test/env.example b/test/env.example index 6de9f39..befe472 100644 --- a/test/env.example +++ b/test/env.example @@ -28,15 +28,15 @@ export DOMAIN=dev.spectrocloud.com export PUBLIC_PACK_REPO_UID= # this varies per Palette tenant, identify via Chrome inspector on Tenant Admin -> Pack Registries page export CLUSTER_NAME=two-node--$(git -C ../stylus describe --always) export CLUSTER_PROFILE_UID= # if left blank, a cluster profile will be created -export CLUSTER_VIP= # choose an unassigned VIP +export CLUSTER_VIP= # choose an unassigned VIP from your static IP range # image vars export EARTHLY_BUILDKIT_CACHE_SIZE_MB=500000 export OCI_REGISTRY=${OCI_REGISTRY:-ttl.sh} export STYLUS_BRANCH=${STYLUS_BRANCH:-2-node} export PROVIDER_K3S_BRANCH=${PROVIDER_K3S_BRANCH:-two-node} -export K3S_VERSION="1.28.5" -export PE_VERSION="4.3.0-2node" +export PE_VERSION=4.4.12 +export K8S_VERSION=1.29.6 # two node vars export TWO_NODE_BACKEND=postgres diff --git a/test/templates/two-node-cluster-profile.json.tmpl b/test/templates/two-node-cluster-profile.json.tmpl index 5f68898..ecf729b 100644 --- a/test/templates/two-node-cluster-profile.json.tmpl +++ b/test/templates/two-node-cluster-profile.json.tmpl @@ -16,7 +16,7 @@ "layer": "os", "version": "1.0.0", "tag": "1.0.0", - "values": "pack:\n content:\n images:\n - image: \"{{.spectro.pack.edge-native-byoi.options.system.uri}}\"\n # Below config is default value, please uncomment if you want to modify default values\n #drain:\n #cordon: true\n #timeout: 60 # The length of time to wait before giving up, zero means infinite\n #gracePeriod: 60 # Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used\n #ignoreDaemonSets: true\n #deleteLocalData: true # Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained)\n #force: true # Continue even if there are pods that do not declare a controller\n #disableEviction: false # Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution\n #skipWaitForDeleteTimeout: 60 # If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.\nstylusPackage: container://OCI_REGISTRY/stylus-linux-amd64:v0.0.0-STYLUS_HASH\noptions:\n system.uri: \"OCI_REGISTRY/ubuntu:k3s-K3S_VERSION-vPE_VERSION-STYLUS_HASH\"", + "values": "pack:\n content:\n images:\n - image: \"{{.spectro.pack.edge-native-byoi.options.system.uri}}\"\n # Below config is default value, please uncomment if you want to modify default values\n #drain:\n #cordon: true\n #timeout: 60 # The length of time to wait before giving up, zero means infinite\n #gracePeriod: 60 # Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used\n #ignoreDaemonSets: true\n #deleteLocalData: true # Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained)\n #force: true # Continue even if there are pods that do not declare a controller\n #disableEviction: false # Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution\n #skipWaitForDeleteTimeout: 60 # If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.\nstylusPackage: container://OCI_REGISTRY/stylus-linux-amd64:v0.0.0-STYLUS_HASH\noptions:\n system.uri: \"OCI_REGISTRY/ubuntu:k3s-K8S_VERSION-vPE_VERSION-STYLUS_HASH\"", "registry": { "metadata": { "uid": "_____place_holder_____", diff --git a/test/templates/two-node-master-master.json.tmpl b/test/templates/two-node-master-master.json.tmpl index 1dcd625..719676b 100644 --- a/test/templates/two-node-master-master.json.tmpl +++ b/test/templates/two-node-master-master.json.tmpl @@ -67,7 +67,7 @@ "tag": "1.0.0", "name": "edge-native-byoi", "type": "spectro", - "values": "pack:\n content:\n images:\n - image: \"{{.spectro.pack.edge-native-byoi.options.system.uri}}\"\n # Below config is default value, please uncomment if you want to modify default values\n #drain:\n #cordon: true\n #timeout: 60 # The length of time to wait before giving up, zero means infinite\n #gracePeriod: 60 # Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used\n #ignoreDaemonSets: true\n #deleteLocalData: true # Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained)\n #force: true # Continue even if there are pods that do not declare a controller\n #disableEviction: false # Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution\n #skipWaitForDeleteTimeout: 60 # If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.\nstylusPackage: container://OCI_REGISTRY/stylus-linux-amd64:v0.0.0-STYLUS_HASH\noptions:\n system.uri: \"OCI_REGISTRY/ubuntu:k3s-K3S_VERSION-vPE_VERSION-STYLUS_HASH\"", + "values": "pack:\n content:\n images:\n - image: \"{{.spectro.pack.edge-native-byoi.options.system.uri}}\"\n # Below config is default value, please uncomment if you want to modify default values\n #drain:\n #cordon: true\n #timeout: 60 # The length of time to wait before giving up, zero means infinite\n #gracePeriod: 60 # Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used\n #ignoreDaemonSets: true\n #deleteLocalData: true # Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained)\n #force: true # Continue even if there are pods that do not declare a controller\n #disableEviction: false # Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution\n #skipWaitForDeleteTimeout: 60 # If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.\nstylusPackage: container://OCI_REGISTRY/stylus-linux-amd64:v0.0.0-STYLUS_HASH\noptions:\n system.uri: \"OCI_REGISTRY/ubuntu:k3s-K8S_VERSION-vPE_VERSION-STYLUS_HASH\"", "manifests": [] }, { diff --git a/test/test-two-node.sh b/test/test-two-node.sh index c84607e..173718a 100755 --- a/test/test-two-node.sh +++ b/test/test-two-node.sh @@ -2,6 +2,8 @@ set -e +source .arg + # Usage # ----- # @@ -14,10 +16,13 @@ set -e # - jq (https://jqlang.github.io/jq/download/) # - mkisofs (https://command-not-found.com/mkisofs) # -# 2. Clone CanvOS and checkout this branch. +# 2. Clone CanvOS and 'git checkout' this branch. +# +# 3. Configure your Earthly argument file by running: cp .arg.template .arg. +# Ensure the following vars are customized: # -# 3. Configure your Earthly argument file by running: cp .arg.template .arg -# No modifications to the template are required. +# K3S_PROVIDER_VERSION=v4.4.2-2node +# K8S_VERSION=1.28.7 (pick any supported K3s version) # # 4. Create a .netrc file in the stylus repo root with GitHub # credentials capable of cloning Spectro Cloud internal repos. @@ -25,10 +30,10 @@ set -e # 5. Copy the test/env.example file to test/.env and edit test/.env # as required. # -# 6. Source and execute this script: +# 6. Source this script and execute 'deploy': # # source ./test/test-two-node.sh -# ./test/test-two-node.sh +# deploy # Do not edit anything below @@ -78,6 +83,9 @@ stylus: users: - name: kairos passwd: kairos + includeTui: true + twoNode: + enable: true site: edgeHostToken: "$EDGE_REGISTRATION_TOKEN" paletteEndpoint: "$DOMAIN" @@ -321,15 +329,15 @@ function prepare_cluster_profile() { jq ' .metadata.name = env.CLUSTER_NAME | .spec.template.packs[0].registry.metadata.uid = env.PUBLIC_PACK_REPO_UID | - .spec.template.packs[1].version = env.K3S_VERSION | - .spec.template.packs[1].tag = env.K3S_VERSION | + .spec.template.packs[1].version = env.K8S_VERSION | + .spec.template.packs[1].tag = env.K8S_VERSION | .spec.template.packs[1].registry.metadata.uid = env.PUBLIC_PACK_REPO_UID | .spec.template.packs[2].registry.metadata.uid = env.PUBLIC_PACK_REPO_UID | .spec.template.packs[0].values |= gsub("OCI_REGISTRY"; env.OCI_REGISTRY) | .spec.template.packs[0].values |= gsub("PE_VERSION"; env.PE_VERSION) | - .spec.template.packs[0].values |= gsub("K3S_VERSION"; env.K3S_VERSION) | + .spec.template.packs[0].values |= gsub("K8S_VERSION"; env.K8S_VERSION) | .spec.template.packs[0].values |= gsub("STYLUS_HASH"; env.STYLUS_HASH) - ' test/templates/two-node-cluster-profile.json.tmpl > two-node-cluster-profile.json + ' test/templates/two-node-cluster-profile.json.tmpl > /tmp/two-node-cluster-profile.json } function create_cluster_profile() { @@ -337,8 +345,7 @@ function create_cluster_profile() { -H "ApiKey: $API_KEY" \ -H "Content-Type: application/json" \ -H "ProjectUid: $PROJECT_UID" \ - -d @two-node-cluster-profile.json | jq -r .uid) - rm -f two-node-cluster-profile.json + -d @/tmp/two-node-cluster-profile.json | jq -r .uid) if [ "$CLUSTER_PROFILE_UID" = "null" ]; then echo Cluster Profile creation failed as it already exists. Please delete it and retry. return 1 @@ -376,10 +383,10 @@ function prepare_master_master_cluster() { .spec.profiles[0].uid = env.CLUSTER_PROFILE_UID | .spec.profiles[0].packValues[0].values |= gsub("OCI_REGISTRY"; env.OCI_REGISTRY) | .spec.profiles[0].packValues[0].values |= gsub("PE_VERSION"; env.PE_VERSION) | - .spec.profiles[0].packValues[0].values |= gsub("K3S_VERSION"; env.K3S_VERSION) | + .spec.profiles[0].packValues[0].values |= gsub("K8S_VERSION"; env.K8S_VERSION) | .spec.profiles[0].packValues[0].values |= gsub("STYLUS_HASH"; env.STYLUS_HASH) | - .spec.profiles[0].packValues[1].tag = env.K3S_VERSION - ' test/templates/two-node-master-master.json.tmpl > two-node-create.json + .spec.profiles[0].packValues[1].tag = env.K8S_VERSION + ' test/templates/two-node-master-master.json.tmpl > /tmp/two-node-create.json } function create_cluster() { @@ -387,12 +394,11 @@ function create_cluster() { -H "ApiKey: $API_KEY" \ -H "Content-Type: application/json" \ -H "ProjectUid: $PROJECT_UID" \ - -d @two-node-create.json | jq -r .uid) + -d @/tmp/two-node-create.json | jq -r .uid) if [ "$uid" = "null" ]; then echo "Cluster creation failed. Please check two-node-create.json and retry creation manually to see Hubble's response." return 1 else - rm -f two-node-create.json echo "Cluster $uid created" fi } @@ -443,27 +449,24 @@ function update_cluster() { function build_provider_k3s() { echo "Building provider-k3s image..." - earthly +build-provider-package \ + earthly --push +build-provider-package \ --platform=linux/amd64 \ --IMAGE_REPOSITORY=${OCI_REGISTRY} \ --VERSION=${PROVIDER_K3S_HASH} - docker push ${OCI_REGISTRY}/provider-k3s:${PROVIDER_K3S_HASH} } function build_stylus_package_and_framework() { echo "Building stylus image and stylus framework image..." - earthly --allow-privileged +package \ + earthly --allow-privileged --push +package \ --platform=linux/amd64 \ --IMAGE_REPOSITORY=${OCI_REGISTRY} \ --BASE_IMAGE=quay.io/kairos/core-opensuse-leap:v2.3.2 \ --VERSION=v0.0.0-${STYLUS_HASH} - docker push ${OCI_REGISTRY}/stylus-linux-amd64:v0.0.0-${STYLUS_HASH} - docker push ${OCI_REGISTRY}/stylus-framework-linux-amd64:v0.0.0-${STYLUS_HASH} } function build_canvos() { echo "Building provider image & installer ISO..." - earthly +build-all-images \ + earthly --push +build-all-images \ --ARCH=amd64 \ --PROVIDER_BASE=${OCI_REGISTRY}/provider-k3s:${PROVIDER_K3S_HASH} \ --STYLUS_BASE=${OCI_REGISTRY}/stylus-framework-linux-amd64:v0.0.0-${STYLUS_HASH} \ @@ -471,9 +474,7 @@ function build_canvos() { --IMAGE_REGISTRY=${OCI_REGISTRY} \ --TWO_NODE=true \ --TWO_NODE_BACKEND=${TWO_NODE_BACKEND} \ - --CUSTOM_TAG=${STYLUS_HASH} \ - --PE_VERSION=v${PE_VERSION} - docker push ${OCI_REGISTRY}/ubuntu:k3s-${K3S_VERSION}-v${PE_VERSION}-${STYLUS_HASH} + --CUSTOM_TAG=${STYLUS_HASH} } function build_all() { @@ -501,7 +502,7 @@ function build_all() { ( test -f build/palette-edge-installer-stylus-${STYLUS_HASH}-k3s-${PROVIDER_K3S_HASH}.iso && \ docker image ls --format "{{.Repository}}:{{.Tag}}" | \ - grep -q ${OCI_REGISTRY}/ubuntu:k3s-${K3S_VERSION}-v${PE_VERSION}-${STYLUS_HASH} + grep -q ${OCI_REGISTRY}/ubuntu:k3s-${K8S_VERSION}-v${PE_VERSION}-${STYLUS_HASH} ) || ( build_canvos ) } @@ -512,6 +513,36 @@ function clean_all() { docker system prune --all --volumes --force } +# Builds 2-node provider image and ISO, uploads the ISO to vCenter, and provisions edge hosts +# for deploying a 2-node cluster. +function deploy() { + set +e + + earthly --push +build-all-images \ + --ARCH=amd64 \ + --ISO_NAME=palette-edge-installer-stylus-k3s-twonode \ + --IMAGE_REGISTRY=${OCI_REGISTRY} \ + --TWO_NODE=true \ + --TWO_NODE_BACKEND=${TWO_NODE_BACKEND} \ + --CUSTOM_TAG=twonode + + # create & upload user-data ISOs, configured to enable two node mode + create_userdata_iso + upload_userdata_iso + + echo Uploading installer ISO $iso... + iso=palette-edge-installer-stylus-k3s-twonode.iso + govc datastore.upload --ds=$GOVC_DATASTORE --dc=$GOVC_DATACENTER build/$iso $STYLUS_ISO + + vm_array=("two-node-sep16-1" "two-node-sep16-2") + create_vms + wait_for_vms_to_power_off + reboot_vms + + echo Edge hosts should register with Palette shortly. Please proceed to deploying an Edge cluster via the UI. + set -e +} + function main() { init_globals