From 1c4e43aa724c931e08844c943a26bfb32b7cd3cc Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Thu, 13 Jun 2024 14:36:16 -0700
Subject: [PATCH 01/32] Override the nginx image properly in tinkerbell stack
 chart (#8319)

Co-authored-by: ahreehong <46465244+ahreehong@users.noreply.github.com>
---
 pkg/providers/tinkerbell/stack/stack.go                          | 1 +
 pkg/providers/tinkerbell/stack/stack_test.go                     | 1 +
 pkg/providers/tinkerbell/stack/testdata/expected_upgrade.yaml    | 1 +
 .../tinkerbell/stack/testdata/expected_upgrade_with_proxy.yaml   | 1 +
 .../tinkerbell/stack/testdata/expected_with_boots_on_docker.yaml | 1 +
 .../stack/testdata/expected_with_boots_on_kubernetes.yaml        | 1 +
 .../tinkerbell/stack/testdata/expected_with_docker_options.yaml  | 1 +
 .../tinkerbell/stack/testdata/expected_with_hook_override.yaml   | 1 +
 .../stack/testdata/expected_with_host_port_enabled_false.yaml    | 1 +
 .../stack/testdata/expected_with_host_port_enabled_true.yaml     | 1 +
 .../stack/testdata/expected_with_kubernetes_options.yaml         | 1 +
 .../testdata/expected_with_load_balancer_enabled_false.yaml      | 1 +
 .../stack/testdata/expected_with_load_balancer_enabled_true.yaml | 1 +
 .../tinkerbell/stack/testdata/expected_with_proxy_config.yaml    | 1 +
 .../tinkerbell/stack/testdata/expected_with_registry_mirror.yaml | 1 +
 15 files changed, 15 insertions(+)

diff --git a/pkg/providers/tinkerbell/stack/stack.go b/pkg/providers/tinkerbell/stack/stack.go
index b52eca69ac86..d5eaac07a08b 100644
--- a/pkg/providers/tinkerbell/stack/stack.go
+++ b/pkg/providers/tinkerbell/stack/stack.go
@@ -558,6 +558,7 @@ func (s *Installer) createValuesOverride(bundle releasev1alpha1.TinkerbellBundle
 			},
 		},
 		stack: map[string]interface{}{
+			image: bundle.TinkerbellStack.Tink.Nginx.URI,
 			kubevip: map[string]interface{}{
 				image:   bundle.KubeVip.URI,
 				enabled: s.loadBalancer,
diff --git a/pkg/providers/tinkerbell/stack/stack_test.go b/pkg/providers/tinkerbell/stack/stack_test.go
index d69dd8f14252..9e774a1efe6f 100644
--- a/pkg/providers/tinkerbell/stack/stack_test.go
+++ b/pkg/providers/tinkerbell/stack/stack_test.go
@@ -42,6 +42,7 @@ func getTinkBundle() releasev1alpha1.TinkerbellBundle {
 	return releasev1alpha1.TinkerbellBundle{
 		TinkerbellStack: releasev1alpha1.TinkerbellStackBundle{
 			Tink: releasev1alpha1.TinkBundle{
+				Nginx:          releasev1alpha1.Image{URI: "public.ecr.aws/eks-anywhere/nginx:latest"},
 				TinkController: releasev1alpha1.Image{URI: "public.ecr.aws/eks-anywhere/tink-controller:latest"},
 				TinkServer:     releasev1alpha1.Image{URI: "public.ecr.aws/eks-anywhere/tink-server:latest"},
 				TinkWorker:     releasev1alpha1.Image{URI: "public.ecr.aws/eks-anywhere/tink-worker:latest"},
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_upgrade.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_upgrade.yaml
index c1409f6eb11d..9dec402eedde 100755
--- a/pkg/providers/tinkerbell/stack/testdata/expected_upgrade.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_upgrade.yaml
@@ -34,6 +34,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: false
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_upgrade_with_proxy.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_upgrade_with_proxy.yaml
index 8d4c2205f4b1..390bc5ca4e8d 100755
--- a/pkg/providers/tinkerbell/stack/testdata/expected_upgrade_with_proxy.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_upgrade_with_proxy.yaml
@@ -35,6 +35,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: false
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_docker.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_docker.yaml
index bf5092ed0044..b1275d5a09b7 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_docker.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_docker.yaml
@@ -34,6 +34,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: false
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_kubernetes.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_kubernetes.yaml
index c1409f6eb11d..9dec402eedde 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_kubernetes.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_kubernetes.yaml
@@ -34,6 +34,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: false
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_docker_options.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_docker_options.yaml
index d287970e85c9..dfd89a1b52d8 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_docker_options.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_docker_options.yaml
@@ -34,6 +34,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: true
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_hook_override.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_hook_override.yaml
index b4f11d16b798..6c667401a21e 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_hook_override.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_hook_override.yaml
@@ -34,6 +34,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: false
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_false.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_false.yaml
index c1409f6eb11d..9dec402eedde 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_false.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_false.yaml
@@ -34,6 +34,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: false
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_true.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_true.yaml
index 015dc6590217..ae610ad2bb85 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_true.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_true.yaml
@@ -34,6 +34,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: true
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_kubernetes_options.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_kubernetes_options.yaml
index c256ae1728ce..d2b41811295f 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_kubernetes_options.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_kubernetes_options.yaml
@@ -34,6 +34,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: false
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_false.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_false.yaml
index c1409f6eb11d..9dec402eedde 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_false.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_false.yaml
@@ -34,6 +34,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: false
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_true.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_true.yaml
index c256ae1728ce..d2b41811295f 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_true.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_true.yaml
@@ -34,6 +34,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: false
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_proxy_config.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_proxy_config.yaml
index 9f961714a77e..a99b684be43e 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_proxy_config.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_proxy_config.yaml
@@ -35,6 +35,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: false
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_registry_mirror.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_registry_mirror.yaml
index 4bf94f047961..d2a0be25b640 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_registry_mirror.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_registry_mirror.yaml
@@ -38,6 +38,7 @@ stack:
   hook:
     enabled: false
   hostNetwork: false
+  image: public.ecr.aws/eks-anywhere/nginx:latest
   kubevip:
     additionalEnv:
     - name: prometheus_server

From d0b1628340c1e6f8b5cb4b70048886c7baaa7ddc Mon Sep 17 00:00:00 2001
From: Rahul Ganesh <31204974+rahulbabu95@users.noreply.github.com>
Date: Thu, 13 Jun 2024 15:07:16 -0700
Subject: [PATCH 02/32] Update branch name to release-0.20 in Makefile (#8320)

Signed-off-by: Rahul Ganesh <rahulgab@amazon.com>
Co-authored-by: Rahul Ganesh <rahulgab@amazon.com>
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 820c8b6720ea..34eee816d73f 100644
--- a/Makefile
+++ b/Makefile
@@ -29,7 +29,7 @@ UNIT_TEST_PACKAGE_EXCLUSION_REGEX ?=mocks$
 UNIT_TEST_PACKAGES ?= $$($(GO) list ./... | grep -vE "$(UNIT_TEST_PACKAGE_EXCLUSION_REGEX)")
 
 ## ensure local execution uses the 'main' branch bundle
-BRANCH_NAME?=main
+BRANCH_NAME?=release-0.20
 # DEV_GIT_VERSION should be something like v0.19.0-dev+latest, depending on the base branch
 # https://github.com/aws/eks-anywhere/blob/main/docs/developer/manifests.md#locally-building-the-cli
 ifneq ($(PULL_BASE_REF),) # PULL_BASE_REF originates from prow

From 434226d09da18d2a46ddb1afeff92c3e32084388 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Fri, 14 Jun 2024 16:35:17 -0700
Subject: [PATCH 03/32] Fix e2e tests creating old bundle workload clusters
 (#8330)

Co-authored-by: Guillermo Gaston <gaslor@amazon.com>
---
 test/e2e/docker_test.go         |  1 +
 test/e2e/suite.go               | 18 +++++++++++++++++
 test/e2e/upgrade_from_latest.go | 34 +++++++++++++++++++++++++++------
 test/e2e/vsphere_test.go        |  1 +
 4 files changed, 48 insertions(+), 6 deletions(-)
 create mode 100644 test/e2e/suite.go

diff --git a/test/e2e/docker_test.go b/test/e2e/docker_test.go
index 3be6d8c25633..fe9c915d4659 100644
--- a/test/e2e/docker_test.go
+++ b/test/e2e/docker_test.go
@@ -1050,6 +1050,7 @@ func TestDockerKubernetes129to130UpgradeFromLatestMinorReleaseAPI(t *testing.T)
 	runMulticlusterUpgradeFromReleaseFlowAPI(
 		test,
 		release,
+		wc.ClusterConfig.Cluster.Spec.KubernetesVersion,
 		v1alpha1.Kube130,
 		"",
 	)
diff --git a/test/e2e/suite.go b/test/e2e/suite.go
new file mode 100644
index 000000000000..ade1f87d6b84
--- /dev/null
+++ b/test/e2e/suite.go
@@ -0,0 +1,18 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+)
+
+func init() {
+	// This is necessary because the test framework builds controller-runtime
+	// k8s client, and the library requires SetLogger to be called before
+	// it's used. Otherwise it prints a confusing warning and it hides any
+	// other client log.
+	// There might a better way for this, but this will do for now.
+	ctrl.SetLogger(klog.Background())
+}
diff --git a/test/e2e/upgrade_from_latest.go b/test/e2e/upgrade_from_latest.go
index 42c9334da524..f3235fcd7219 100644
--- a/test/e2e/upgrade_from_latest.go
+++ b/test/e2e/upgrade_from_latest.go
@@ -66,10 +66,23 @@ func runInPlaceUpgradeFromReleaseFlow(test *framework.ClusterE2ETest, latestRele
 	test.DeleteCluster()
 }
 
-func runMulticlusterUpgradeFromReleaseFlowAPI(test *framework.MulticlusterE2ETest, release *releasev1.EksARelease, kubeVersion anywherev1.KubernetesVersion, os framework.OS) {
+// runMulticlusterUpgradeFromReleaseFlowAPI tests the ability to create workload clusters with an old Bundle in a management cluster
+// that has been updated to a new Bundle. It follows the following steps:
+// 1. Create a management cluster with the old Bundle.
+// 2. Create workload clusters with the old Bundle.
+// 3. Upgrade the management cluster to the new Bundle and new Kubernetes version (newVersion).
+// 4. Upgrade the workload clusters to the new Bundle and new Kubernetes version (newVersion).
+// 5. Delete the workload clusters.
+// 6. Re-create the workload clusters with the old Bundle and previous Kubernetes version (oldVersion). It's necessary to sometimes
+//    use a different kube version because the old Bundle might not support the new kubernetes version.
+// 7. Delete the workload clusters.
+// 8. Delete the management cluster.
+func runMulticlusterUpgradeFromReleaseFlowAPI(test *framework.MulticlusterE2ETest, release *releasev1.EksARelease, oldVersion, newVersion anywherev1.KubernetesVersion, os framework.OS) {
 	provider := test.ManagementCluster.Provider
+	// 1. Create management cluster
 	test.CreateManagementCluster(framework.ExecuteWithEksaRelease(release))
 
+	// 2. Create workload clusters with the old Bundle  
 	test.RunConcurrentlyInWorkloadClusters(func(wc *framework.WorkloadCluster) {
 		wc.CreateCluster(framework.ExecuteWithEksaRelease(release))
 		wc.ValidateCluster(wc.ClusterConfig.Cluster.Spec.KubernetesVersion)
@@ -79,18 +92,19 @@ func runMulticlusterUpgradeFromReleaseFlowAPI(test *framework.MulticlusterE2ETes
 	oldCluster := test.ManagementCluster.GetEKSACluster()
 
 	test.ManagementCluster.UpdateClusterConfig(
-		provider.WithKubeVersionAndOS(kubeVersion, os, nil),
+		provider.WithKubeVersionAndOS(newVersion, os, nil),
 	)
+	// 3. Upgrade management cluster to new Bundle and new Kubernetes version
 	test.ManagementCluster.UpgradeCluster()
 	test.ManagementCluster.ValidateCluster(test.ManagementCluster.ClusterConfig.Cluster.Spec.KubernetesVersion)
 	test.ManagementCluster.StopIfFailed()
 
 	cluster := test.ManagementCluster.GetEKSACluster()
 
-	// Upgrade bundle workload clusters now because they still have the old versions of the bundle.
 	test.RunConcurrentlyInWorkloadClusters(func(wc *framework.WorkloadCluster) {
+		// 4. Upgrade the workload clusters to the new Bundle and new Kubernetes version (newVersion).
 		wc.UpdateClusterConfig(
-			provider.WithKubeVersionAndOS(kubeVersion, os, nil),
+			provider.WithKubeVersionAndOS(newVersion, os, nil),
 			api.ClusterToConfigFiller(
 				api.WithEksaVersion(cluster.Spec.EksaVersion),
 			),
@@ -98,28 +112,36 @@ func runMulticlusterUpgradeFromReleaseFlowAPI(test *framework.MulticlusterE2ETes
 		wc.ApplyClusterManifest()
 		wc.ValidateClusterState()
 		wc.StopIfFailed()
+		// 5. Delete the workload clusters.
 		wc.DeleteClusterWithKubectl()
 		wc.ValidateClusterDelete()
 		wc.StopIfFailed()
 	})
 
-	// Create workload cluster with old bundle
 	test.RunConcurrentlyInWorkloadClusters(func(wc *framework.WorkloadCluster) {
 		wc.UpdateClusterConfig(
-			provider.WithKubeVersionAndOS(kubeVersion, os, release),
+			provider.WithKubeVersionAndOS(oldVersion, os, release),
 			api.ClusterToConfigFiller(
 				api.WithEksaVersion(oldCluster.Spec.EksaVersion),
 			),
 		)
+		// 6. Re-create the workload clusters with the old Bundle and previous Kubernetes version (oldVersion).
 		wc.ApplyClusterManifest()
 		wc.WaitForKubeconfig()
 		wc.ValidateClusterState()
 		wc.StopIfFailed()
+		// 7. Delete the workload clusters.
 		wc.DeleteClusterWithKubectl()
 		wc.ValidateClusterDelete()
+	})
+
+	// It's necessary to call stop here because if any of the workload clusters failed,
+	// their panic was thrown in a go routine, which doesn't stop the main test routine.
+	test.RunInWorkloadClusters(func(wc *framework.WorkloadCluster) {
 		wc.StopIfFailed()
 	})
 
+	// 8. Delete the management cluster.
 	test.DeleteManagementCluster()
 }
 
diff --git a/test/e2e/vsphere_test.go b/test/e2e/vsphere_test.go
index 281121fbdbaa..89fab09673af 100644
--- a/test/e2e/vsphere_test.go
+++ b/test/e2e/vsphere_test.go
@@ -4329,6 +4329,7 @@ func TestVSphereKubernetes127to128UpgradeFromLatestMinorReleaseBottleRocketAPI(t
 	runMulticlusterUpgradeFromReleaseFlowAPI(
 		test,
 		release,
+		wc.ClusterConfig.Cluster.Spec.KubernetesVersion,
 		v1alpha1.Kube128,
 		framework.Bottlerocket1,
 	)

From d1b4222aa7beffd6b3fe118a19f474df4f6a6cec Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 17 Jun 2024 09:19:20 -0700
Subject: [PATCH 04/32] Updated prometheus pod labels for e2e test (#8332)

Co-authored-by: Jhaanvi Golani <jhaanvi@amazon.com>
---
 docs/content/en/docs/packages/prometheus/prometheus_grafana.md | 2 +-
 test/framework/cluster.go                                      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/content/en/docs/packages/prometheus/prometheus_grafana.md b/docs/content/en/docs/packages/prometheus/prometheus_grafana.md
index 335b4222db0d..8547d83fb6ca 100644
--- a/docs/content/en/docs/packages/prometheus/prometheus_grafana.md
+++ b/docs/content/en/docs/packages/prometheus/prometheus_grafana.md
@@ -125,7 +125,7 @@ To ensure the Prometheus package is installed correctly in the cluster, a user c
 
 Port forward Prometheus to local host `9090`:
 ```bash
-export PROM_SERVER_POD_NAME=$(kubectl get pods --namespace <namespace> -l "app=prometheus,component=server" -o jsonpath="{.items[0].metadata.name")
+export PROM_SERVER_POD_NAME=$(kubectl get pods --namespace <namespace> -l "app.kubernetes.io/name=prometheus,app.kubernetes.io/component=server" -o jsonpath="{.items[0].metadata.name")
 kubectl port-forward $PROM_SERVER_POD_NAME -n <namespace> 9090
 ```
 
diff --git a/test/framework/cluster.go b/test/framework/cluster.go
index d72fd041720e..0bf3e865442a 100644
--- a/test/framework/cluster.go
+++ b/test/framework/cluster.go
@@ -1812,7 +1812,7 @@ func (e *ClusterE2ETest) VerifyPrometheusPrometheusServerStates(packageName, tar
 	}
 
 	e.T.Log("Reading package", packageName, "pod prometheus-server logs")
-	podName, err := e.KubectlClient.GetPodNameByLabel(context.TODO(), targetNamespace, "app=prometheus,component=server", e.KubeconfigFilePath())
+	podName, err := e.KubectlClient.GetPodNameByLabel(context.TODO(), targetNamespace, "app.kubernetes.io/name=prometheus,app.kubernetes.io/component=server", e.KubeconfigFilePath())
 	if err != nil {
 		e.T.Fatalf("unable to get name of the prometheus-server pod: %s", err)
 	}

From 38737d884c043c720a645c7ad498caee5b635d18 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 17 Jun 2024 15:58:21 -0700
Subject: [PATCH 05/32] Ensure Chart.yaml is modified for Tinkerbell stack and
 CRDs Helm charts (#8336)

Co-authored-by: Abhay Krishna Arunachalam <arnchlm@amazon.com>
---
 release/cli/pkg/bundles/tinkerbell.go         |  5 ++-
 .../testdata/main-bundle-release.yaml         | 40 +++++++++----------
 release/cli/pkg/operations/upload.go          |  3 +-
 3 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/release/cli/pkg/bundles/tinkerbell.go b/release/cli/pkg/bundles/tinkerbell.go
index 714a974d76ad..4a0199337c09 100644
--- a/release/cli/pkg/bundles/tinkerbell.go
+++ b/release/cli/pkg/bundles/tinkerbell.go
@@ -61,9 +61,10 @@ func GetTinkerbellBundle(r *releasetypes.ReleaseConfig, imageDigests releasetype
 				// If the artifact is a helm chart, handle differently. Helm charts can have both suffix "-chart" and "-helm",
 				// so we need to handle both cases.
 				if strings.HasSuffix(imageArtifact.AssetName, "chart") || strings.HasSuffix(imageArtifact.AssetName, "helm") {
+					assetName := strings.TrimSuffix(imageArtifact.AssetName, "-helm")
 					bundleImageArtifact = anywherev1alpha1.Image{
-						Name:        imageArtifact.AssetName,
-						Description: fmt.Sprintf("Helm chart for %s", imageArtifact.AssetName),
+						Name:        assetName,
+						Description: fmt.Sprintf("Helm chart for %s", assetName),
 						URI:         imageArtifact.ReleaseImageURI,
 						ImageDigest: imageDigest,
 					}
diff --git a/release/cli/pkg/operations/testdata/main-bundle-release.yaml b/release/cli/pkg/operations/testdata/main-bundle-release.yaml
index be660d3f55a5..c7f1b0b58c20 100644
--- a/release/cli/pkg/operations/testdata/main-bundle-release.yaml
+++ b/release/cli/pkg/operations/testdata/main-bundle-release.yaml
@@ -699,9 +699,9 @@ spec:
           os: linux
           uri: public.ecr.aws/release-container-registry/tinkerbell/rufio:v0.3.3-eks-a-v0.0.0-dev-build.1
         stack:
-          description: Helm chart for stack-helm
+          description: Helm chart for stack
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-          name: stack-helm
+          name: stack
           uri: public.ecr.aws/release-container-registry/tinkerbell/stack:0.4.5-eks-a-v0.0.0-dev-build.1
         tink:
           nginx:
@@ -746,9 +746,9 @@ spec:
           name: tinkerbell-chart
           uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.6-eks-a-v0.0.0-dev-build.1
         tinkerbellCrds:
-          description: Helm chart for tinkerbell-crds-helm
+          description: Helm chart for tinkerbell-crds
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-          name: tinkerbell-crds-helm
+          name: tinkerbell-crds
           uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-crds:0.2.5-eks-a-v0.0.0-dev-build.1
       version: v0.5.3+abcdef1
     upgrader:
@@ -1496,9 +1496,9 @@ spec:
           os: linux
           uri: public.ecr.aws/release-container-registry/tinkerbell/rufio:v0.3.3-eks-a-v0.0.0-dev-build.1
         stack:
-          description: Helm chart for stack-helm
+          description: Helm chart for stack
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-          name: stack-helm
+          name: stack
           uri: public.ecr.aws/release-container-registry/tinkerbell/stack:0.4.5-eks-a-v0.0.0-dev-build.1
         tink:
           nginx:
@@ -1543,9 +1543,9 @@ spec:
           name: tinkerbell-chart
           uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.6-eks-a-v0.0.0-dev-build.1
         tinkerbellCrds:
-          description: Helm chart for tinkerbell-crds-helm
+          description: Helm chart for tinkerbell-crds
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-          name: tinkerbell-crds-helm
+          name: tinkerbell-crds
           uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-crds:0.2.5-eks-a-v0.0.0-dev-build.1
       version: v0.5.3+abcdef1
     upgrader:
@@ -2293,9 +2293,9 @@ spec:
           os: linux
           uri: public.ecr.aws/release-container-registry/tinkerbell/rufio:v0.3.3-eks-a-v0.0.0-dev-build.1
         stack:
-          description: Helm chart for stack-helm
+          description: Helm chart for stack
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-          name: stack-helm
+          name: stack
           uri: public.ecr.aws/release-container-registry/tinkerbell/stack:0.4.5-eks-a-v0.0.0-dev-build.1
         tink:
           nginx:
@@ -2340,9 +2340,9 @@ spec:
           name: tinkerbell-chart
           uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.6-eks-a-v0.0.0-dev-build.1
         tinkerbellCrds:
-          description: Helm chart for tinkerbell-crds-helm
+          description: Helm chart for tinkerbell-crds
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-          name: tinkerbell-crds-helm
+          name: tinkerbell-crds
           uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-crds:0.2.5-eks-a-v0.0.0-dev-build.1
       version: v0.5.3+abcdef1
     upgrader:
@@ -3090,9 +3090,9 @@ spec:
           os: linux
           uri: public.ecr.aws/release-container-registry/tinkerbell/rufio:v0.3.3-eks-a-v0.0.0-dev-build.1
         stack:
-          description: Helm chart for stack-helm
+          description: Helm chart for stack
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-          name: stack-helm
+          name: stack
           uri: public.ecr.aws/release-container-registry/tinkerbell/stack:0.4.5-eks-a-v0.0.0-dev-build.1
         tink:
           nginx:
@@ -3137,9 +3137,9 @@ spec:
           name: tinkerbell-chart
           uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.6-eks-a-v0.0.0-dev-build.1
         tinkerbellCrds:
-          description: Helm chart for tinkerbell-crds-helm
+          description: Helm chart for tinkerbell-crds
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-          name: tinkerbell-crds-helm
+          name: tinkerbell-crds
           uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-crds:0.2.5-eks-a-v0.0.0-dev-build.1
       version: v0.5.3+abcdef1
     upgrader:
@@ -3887,9 +3887,9 @@ spec:
           os: linux
           uri: public.ecr.aws/release-container-registry/tinkerbell/rufio:v0.3.3-eks-a-v0.0.0-dev-build.1
         stack:
-          description: Helm chart for stack-helm
+          description: Helm chart for stack
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-          name: stack-helm
+          name: stack
           uri: public.ecr.aws/release-container-registry/tinkerbell/stack:0.4.5-eks-a-v0.0.0-dev-build.1
         tink:
           nginx:
@@ -3934,9 +3934,9 @@ spec:
           name: tinkerbell-chart
           uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.6-eks-a-v0.0.0-dev-build.1
         tinkerbellCrds:
-          description: Helm chart for tinkerbell-crds-helm
+          description: Helm chart for tinkerbell-crds
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-          name: tinkerbell-crds-helm
+          name: tinkerbell-crds
           uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-crds:0.2.5-eks-a-v0.0.0-dev-build.1
       version: v0.5.3+abcdef1
     upgrader:
diff --git a/release/cli/pkg/operations/upload.go b/release/cli/pkg/operations/upload.go
index d5fa633a1f28..06fb98f99c70 100644
--- a/release/cli/pkg/operations/upload.go
+++ b/release/cli/pkg/operations/upload.go
@@ -133,7 +133,8 @@ func handleManifestUpload(_ context.Context, r *releasetypes.ReleaseConfig, arti
 func handleImageUpload(_ context.Context, r *releasetypes.ReleaseConfig, packagesArtifacts map[string][]releasetypes.Artifact, artifact releasetypes.Artifact, sourceEcrAuthConfig, releaseEcrAuthConfig *docker.AuthConfiguration) error {
 	// If the artifact is a helm chart, skip the skopeo copy. Instead, modify the Chart.yaml to match the release tag
 	// and then use Helm package and push commands to upload chart to ECR Public
-	if !r.DryRun && ((strings.HasSuffix(artifact.Image.AssetName, "helm") && !r.DevRelease) || strings.HasSuffix(artifact.Image.AssetName, "chart")) {
+	// Packages Helm chart modification for dev-release is handled elsewhere, so we are checking for that case and skipping
+	if !r.DryRun && ((strings.HasSuffix(artifact.Image.AssetName, "helm") || strings.HasSuffix(artifact.Image.AssetName, "chart")) && !(artifact.Image.AssetName == "eks-anywhere-packages-helm" && r.DevRelease)) {
 		// Trim -helm on the packages helm chart, but don't need to trim tinkerbell chart since the AssetName is the same as the repoName
 		trimmedAsset := strings.TrimSuffix(artifact.Image.AssetName, "-helm")
 

From 44868d0ccd559bf8d4391eb6e7ecf235768180f2 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 17 Jun 2024 17:55:21 -0700
Subject: [PATCH 06/32] Fix issue where incorrect flag is getting passed to
 helm commands (#8341)

Co-authored-by: ahreehong <46465244+ahreehong@users.noreply.github.com>
---
 pkg/providers/tinkerbell/stack/stack.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pkg/providers/tinkerbell/stack/stack.go b/pkg/providers/tinkerbell/stack/stack.go
index d5eaac07a08b..d60b3ead5a81 100644
--- a/pkg/providers/tinkerbell/stack/stack.go
+++ b/pkg/providers/tinkerbell/stack/stack.go
@@ -432,6 +432,7 @@ func (s *Installer) UpgradeInstallCRDs(ctx context.Context, bundle releasev1alph
 	if s.proxyConfig != nil {
 		envMap["NO_PROXY"] = strings.Join(s.proxyConfig.NoProxy, ",")
 	}
+
 	return s.helm.UpgradeInstallChartWithValuesFile(
 		ctx,
 		bundle.TinkerbellStack.TinkerbellCrds.Name,
@@ -441,6 +442,7 @@ func (s *Installer) UpgradeInstallCRDs(ctx context.Context, bundle releasev1alph
 		s.namespace,
 		"",
 		helm.WithProxyConfig(envMap),
+		helm.WithExtraFlags([]string{}),
 	)
 }
 

From 08145359e02e311a41125e6ca6810dd33245a2e4 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 17 Jun 2024 18:56:21 -0700
Subject: [PATCH 07/32] ensure release branch is included in teest cluster name
 (#8342)

Co-authored-by: Cavaughn Browne <cxbrowne@amazon.com>
---
 internal/test/e2e/setup.go | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/internal/test/e2e/setup.go b/internal/test/e2e/setup.go
index ee66c8bcd3c8..fee99bb0e8e9 100644
--- a/internal/test/e2e/setup.go
+++ b/internal/test/e2e/setup.go
@@ -285,10 +285,6 @@ func clusterPrefix(branch, instanceId string) (clusterPrefix string) {
 		sanitizedBranch = strings.ReplaceAll(sanitizedBranch, char, "-")
 	}
 
-	if len(sanitizedBranch) > 7 {
-		sanitizedBranch = sanitizedBranch[:7]
-	}
-
 	if len(instanceId) > 7 {
 		instanceId = instanceId[:7]
 	}
@@ -299,9 +295,9 @@ func clusterPrefix(branch, instanceId string) (clusterPrefix string) {
 
 func (e *E2ESession) clusterName(branch, instanceId, testName string) (clusterName string) {
 	clusterName = fmt.Sprintf("%s-%s", clusterPrefix(branch, instanceId), e2etests.GetTestNameHash(testName))
-	if len(clusterName) > 63 {
-		e.logger.Info("Cluster name is longer than 63 characters; truncating to 63 characters.", "original cluster name", clusterName, "truncated cluster name", clusterName[:63])
-		clusterName = clusterName[:63]
+	if len(clusterName) > 35 {
+		e.logger.Info("Cluster name is longer than 35 characters; truncating to 35 characters.", "original cluster name", clusterName, "truncated cluster name", clusterName[:35])
+		clusterName = clusterName[:35]
 	}
 	return clusterName
 }

From 42482d648148986053fec58fbc19b6ea5713d834 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Wed, 19 Jun 2024 09:05:55 -0700
Subject: [PATCH 08/32] Log ssm command details when an e2e test finishes
 (#8352)

Co-authored-by: Xu Deng <xudeng@amazon.com>
---
 internal/pkg/ssm/run_output.go | 5 +++++
 internal/test/e2e/run.go       | 1 +
 2 files changed, 6 insertions(+)

diff --git a/internal/pkg/ssm/run_output.go b/internal/pkg/ssm/run_output.go
index 1061cb6481b6..d6f3be8ee7f2 100644
--- a/internal/pkg/ssm/run_output.go
+++ b/internal/pkg/ssm/run_output.go
@@ -20,3 +20,8 @@ func buildRunOutput(commandOut *ssm.GetCommandInvocationOutput) *RunOutput {
 func (r *RunOutput) Successful() bool {
 	return *r.commandOut.Status == ssm.CommandInvocationStatusSuccess
 }
+
+// StatusDetails returns the status details of the ssm command.
+func (r *RunOutput) StatusDetails() string {
+	return *r.commandOut.StatusDetails
+}
diff --git a/internal/test/e2e/run.go b/internal/test/e2e/run.go
index e5b97abaa5b9..1f7a550e9987 100644
--- a/internal/test/e2e/run.go
+++ b/internal/test/e2e/run.go
@@ -172,6 +172,7 @@ func RunTestsInParallel(conf ParallelRunConf) error {
 			"completedInstances", completedInstances,
 			"totalInstances", totalInstances,
 			"succeeded", succeeded,
+			"ssmStatusDetails", r.testCommandResult.StatusDetails(),
 		)
 		putInstanceTestResultMetrics(r)
 	}

From 11fcfb5c395ac9ef785efbea87149a97d628a429 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Wed, 19 Jun 2024 19:14:54 -0700
Subject: [PATCH 09/32] Set registry username and password as expected with new
 tink stack (#8354)

Co-authored-by: ahreehong <46465244+ahreehong@users.noreply.github.com>
---
 pkg/providers/tinkerbell/stack/stack.go       | 52 ++++++++-----------
 pkg/providers/tinkerbell/stack/stack_test.go  |  2 -
 .../stack/testdata/expected_upgrade.yaml      |  8 +--
 .../testdata/expected_upgrade_with_proxy.yaml | 12 ++---
 .../expected_with_boots_on_docker.yaml        |  9 +---
 .../expected_with_boots_on_kubernetes.yaml    |  8 +--
 .../expected_with_docker_options.yaml         |  9 +---
 .../testdata/expected_with_hook_override.yaml |  8 +--
 ...expected_with_host_port_enabled_false.yaml |  8 +--
 .../expected_with_host_port_enabled_true.yaml |  8 +--
 .../expected_with_kubernetes_options.yaml     |  8 +--
 ...cted_with_load_balancer_enabled_false.yaml |  8 +--
 ...ected_with_load_balancer_enabled_true.yaml |  8 +--
 .../testdata/expected_with_proxy_config.yaml  | 12 ++---
 .../expected_with_registry_mirror.yaml        | 15 ++----
 15 files changed, 47 insertions(+), 128 deletions(-)

diff --git a/pkg/providers/tinkerbell/stack/stack.go b/pkg/providers/tinkerbell/stack/stack.go
index d60b3ead5a81..a545be64957e 100644
--- a/pkg/providers/tinkerbell/stack/stack.go
+++ b/pkg/providers/tinkerbell/stack/stack.go
@@ -161,7 +161,7 @@ func (s *Installer) Install(ctx context.Context, bundle releasev1alpha1.Tinkerbe
 		option(s)
 	}
 
-	bootEnv := s.getBootsEnv(bundle.TinkerbellStack, tinkerbellIP)
+	bootEnv := s.getSmeeKernelArgs(bundle.TinkerbellStack)
 
 	osieURI, err := getURIDir(bundle.TinkerbellStack.Hook.Initramfs.Amd.URI)
 	if err != nil {
@@ -232,9 +232,9 @@ func (s *Installer) installBootsOnDocker(ctx context.Context, bundle releasev1al
 		"-e", fmt.Sprintf("BOOTS_KUBE_NAMESPACE=%v", s.namespace),
 	}
 
-	for _, e := range s.getBootsEnv(bundle, tinkServerIP) {
-		flags = append(flags, "-e", fmt.Sprintf("%s=%s", e["name"], e["value"]))
-	}
+	extraKernelArgList := s.getSmeeKernelArgs(bundle)
+	extraKernelArgs := strings.Join(extraKernelArgList, " ")
+	flags = append(flags, "-e", fmt.Sprintf("%s=%s", "SMEE_EXTRA_KERNEL_ARGS", extraKernelArgs))
 
 	osiePath, err := getURIDir(bundle.Hook.Initramfs.Amd.URI)
 	if err != nil {
@@ -258,38 +258,32 @@ func (s *Installer) installBootsOnDocker(ctx context.Context, bundle releasev1al
 	return nil
 }
 
-func (s *Installer) getBootsEnv(bundle releasev1alpha1.TinkerbellStackBundle, tinkServerIP string) []map[string]string {
-	env := []map[string]string{
-		toEnvEntry("DATA_MODEL_VERSION", "kubernetes"),
-		toEnvEntry("TINKERBELL_TLS", "false"),
+func (s *Installer) getSmeeKernelArgs(bundle releasev1alpha1.TinkerbellStackBundle) []string {
+	extraKernelArgs := []string{}
+	if s.bootsOnDocker {
+		extraKernelArgs = append(extraKernelArgs, fmt.Sprintf("tink_worker_image=%s", s.localRegistryURL(bundle.Tink.TinkWorker.URI)))
 	}
 
-	extraKernelArgs := fmt.Sprintf("tink_worker_image=%s", s.localRegistryURL(bundle.Tink.TinkWorker.URI))
-
 	if s.registryMirror != nil {
 		localRegistry := s.registryMirror.BaseRegistry
-		extraKernelArgs = fmt.Sprintf("%s insecure_registries=%s", extraKernelArgs, localRegistry)
+		extraKernelArgs = append(extraKernelArgs, fmt.Sprintf("insecure_registries=%s", localRegistry))
 		if s.registryMirror.Auth {
 			username, password, _ := config.ReadCredentials()
-			env = append(env,
-				toEnvEntry("REGISTRY_USERNAME", username),
-				toEnvEntry("REGISTRY_PASSWORD", password))
+			username = fmt.Sprintf("registry_username=%s", username)
+			password = fmt.Sprintf("registry_password=%s", password)
+			extraKernelArgs = append(extraKernelArgs, username, password)
 		}
 	}
 
 	if s.proxyConfig != nil {
 		noProxy := strings.Join(s.proxyConfig.NoProxy, ",")
-		extraKernelArgs = fmt.Sprintf("%s HTTP_PROXY=%s HTTPS_PROXY=%s NO_PROXY=%s", extraKernelArgs, s.proxyConfig.HttpProxy, s.proxyConfig.HttpsProxy, noProxy)
+		httpProxy := fmt.Sprintf("HTTP_PROXY=%s", s.proxyConfig.HttpProxy)
+		httpsProxy := fmt.Sprintf("HTTPS_PROXY=%s", s.proxyConfig.HttpsProxy)
+		noProxy = fmt.Sprintf("NO_PROXY=%s", noProxy)
+		extraKernelArgs = append(extraKernelArgs, httpProxy, httpsProxy, noProxy)
 	}
 
-	return append(env, toEnvEntry("SMEE_EXTRA_KERNEL_ARGS", extraKernelArgs))
-}
-
-func toEnvEntry(k, v string) map[string]string {
-	return map[string]string{
-		"name":  k,
-		"value": v,
-	}
+	return extraKernelArgs
 }
 
 // UninstallLocal currently removes local docker container running Boots.
@@ -363,7 +357,7 @@ func (s *Installer) Upgrade(ctx context.Context, bundle releasev1alpha1.Tinkerbe
 		option(s)
 	}
 
-	bootEnv := s.getBootsEnv(bundle.TinkerbellStack, tinkerbellIP)
+	bootEnv := s.getSmeeKernelArgs(bundle.TinkerbellStack)
 
 	osieURI, err := getURIDir(bundle.TinkerbellStack.Hook.Initramfs.Amd.URI)
 	if err != nil {
@@ -514,7 +508,7 @@ func (s *Installer) HasLegacyChart(ctx context.Context, bundle releasev1alpha1.T
 }
 
 // createValuesOverride generates the values override file to send to helm.
-func (s *Installer) createValuesOverride(bundle releasev1alpha1.TinkerbellBundle, bootEnv []map[string]string, tinkerbellIP string, osiePath *url.URL) map[string]interface{} {
+func (s *Installer) createValuesOverride(bundle releasev1alpha1.TinkerbellBundle, bootEnv []string, tinkerbellIP string, osiePath *url.URL) map[string]interface{} {
 	valuesMap := map[string]interface{}{
 		tink: map[string]interface{}{
 			controller: map[string]interface{}{
@@ -531,10 +525,9 @@ func (s *Installer) createValuesOverride(bundle releasev1alpha1.TinkerbellBundle
 			},
 		},
 		smee: map[string]interface{}{
-			deploy:        !s.bootsOnDocker,
-			image:         bundle.TinkerbellStack.Boots.URI,
-			additionalEnv: bootEnv,
-			"publicIP":    tinkerbellIP,
+			deploy:     !s.bootsOnDocker,
+			image:      bundle.TinkerbellStack.Boots.URI,
+			"publicIP": tinkerbellIP,
 			"trustedProxies": []string{
 				s.podCidrRange,
 			},
@@ -549,6 +542,7 @@ func (s *Installer) createValuesOverride(bundle releasev1alpha1.TinkerbellBundle
 					"port":   osiePath.Port(),
 					"path":   osiePath.Path,
 				},
+				"additionalKernelArgs": bootEnv,
 			},
 			"hostNetwork":     true,
 			"tinkWorkerImage": s.localRegistryURL(bundle.TinkerbellStack.Tink.TinkWorker.URI),
diff --git a/pkg/providers/tinkerbell/stack/stack_test.go b/pkg/providers/tinkerbell/stack/stack_test.go
index 9e774a1efe6f..5e824f607160 100644
--- a/pkg/providers/tinkerbell/stack/stack_test.go
+++ b/pkg/providers/tinkerbell/stack/stack_test.go
@@ -229,8 +229,6 @@ func TestTinkerbellStackInstallWithDifferentOptions(t *testing.T) {
 					"-e", gomock.Any(),
 					"-e", gomock.Any(),
 					"-e", gomock.Any(),
-					"-e", gomock.Any(),
-					"-e", gomock.Any(),
 				)
 			}
 
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_upgrade.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_upgrade.yaml
index 9dec402eedde..51212757466e 100755
--- a/pkg/providers/tinkerbell/stack/testdata/expected_upgrade.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_upgrade.yaml
@@ -7,16 +7,10 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
   deploy: true
   hostNetwork: true
   http:
+    additionalKernelArgs: []
     osieUrl:
       host: anywhere-assests.eks.amazonaws.com
       path: /tinkerbell/hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_upgrade_with_proxy.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_upgrade_with_proxy.yaml
index 390bc5ca4e8d..e58eeef9458e 100755
--- a/pkg/providers/tinkerbell/stack/testdata/expected_upgrade_with_proxy.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_upgrade_with_proxy.yaml
@@ -7,17 +7,13 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest HTTP_PROXY=1.2.3.4
-      HTTPS_PROXY=1.2.3.4 NO_PROXY=localhost,.svc
   deploy: true
   hostNetwork: true
   http:
+    additionalKernelArgs:
+    - HTTP_PROXY=1.2.3.4
+    - HTTPS_PROXY=1.2.3.4
+    - NO_PROXY=localhost,.svc
     osieUrl:
       host: my-local-web-server
       path: /hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_docker.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_docker.yaml
index b1275d5a09b7..d6adcd0702e1 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_docker.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_docker.yaml
@@ -7,16 +7,11 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
   deploy: false
   hostNetwork: true
   http:
+    additionalKernelArgs:
+    - tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
     osieUrl:
       host: anywhere-assests.eks.amazonaws.com
       path: /tinkerbell/hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_kubernetes.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_kubernetes.yaml
index 9dec402eedde..51212757466e 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_kubernetes.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_boots_on_kubernetes.yaml
@@ -7,16 +7,10 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
   deploy: true
   hostNetwork: true
   http:
+    additionalKernelArgs: []
     osieUrl:
       host: anywhere-assests.eks.amazonaws.com
       path: /tinkerbell/hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_docker_options.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_docker_options.yaml
index dfd89a1b52d8..c12a74cab5d7 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_docker_options.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_docker_options.yaml
@@ -7,16 +7,11 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
   deploy: false
   hostNetwork: true
   http:
+    additionalKernelArgs:
+    - tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
     osieUrl:
       host: anywhere-assests.eks.amazonaws.com
       path: /tinkerbell/hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_hook_override.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_hook_override.yaml
index 6c667401a21e..628bb3f34055 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_hook_override.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_hook_override.yaml
@@ -7,16 +7,10 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
   deploy: true
   hostNetwork: true
   http:
+    additionalKernelArgs: []
     osieUrl:
       host: my-local-web-server
       path: /hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_false.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_false.yaml
index 9dec402eedde..51212757466e 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_false.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_false.yaml
@@ -7,16 +7,10 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
   deploy: true
   hostNetwork: true
   http:
+    additionalKernelArgs: []
     osieUrl:
       host: anywhere-assests.eks.amazonaws.com
       path: /tinkerbell/hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_true.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_true.yaml
index ae610ad2bb85..081270116ff8 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_true.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_host_port_enabled_true.yaml
@@ -7,16 +7,10 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
   deploy: true
   hostNetwork: true
   http:
+    additionalKernelArgs: []
     osieUrl:
       host: anywhere-assests.eks.amazonaws.com
       path: /tinkerbell/hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_kubernetes_options.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_kubernetes_options.yaml
index d2b41811295f..6df0f6f9fee8 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_kubernetes_options.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_kubernetes_options.yaml
@@ -7,16 +7,10 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
   deploy: true
   hostNetwork: true
   http:
+    additionalKernelArgs: []
     osieUrl:
       host: anywhere-assests.eks.amazonaws.com
       path: /tinkerbell/hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_false.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_false.yaml
index 9dec402eedde..51212757466e 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_false.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_false.yaml
@@ -7,16 +7,10 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
   deploy: true
   hostNetwork: true
   http:
+    additionalKernelArgs: []
     osieUrl:
       host: anywhere-assests.eks.amazonaws.com
       path: /tinkerbell/hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_true.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_true.yaml
index d2b41811295f..6df0f6f9fee8 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_true.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_load_balancer_enabled_true.yaml
@@ -7,16 +7,10 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest
   deploy: true
   hostNetwork: true
   http:
+    additionalKernelArgs: []
     osieUrl:
       host: anywhere-assests.eks.amazonaws.com
       path: /tinkerbell/hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_proxy_config.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_proxy_config.yaml
index a99b684be43e..01f772d1baa6 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_proxy_config.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_proxy_config.yaml
@@ -7,17 +7,13 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=public.ecr.aws/eks-anywhere/tink-worker:latest HTTP_PROXY=1.2.3.4:3128
-      HTTPS_PROXY=1.2.3.4:3128 NO_PROXY=
   deploy: true
   hostNetwork: true
   http:
+    additionalKernelArgs:
+    - HTTP_PROXY=1.2.3.4:3128
+    - HTTPS_PROXY=1.2.3.4:3128
+    - NO_PROXY=
     osieUrl:
       host: anywhere-assests.eks.amazonaws.com
       path: /tinkerbell/hook
diff --git a/pkg/providers/tinkerbell/stack/testdata/expected_with_registry_mirror.yaml b/pkg/providers/tinkerbell/stack/testdata/expected_with_registry_mirror.yaml
index d2a0be25b640..603538084b82 100644
--- a/pkg/providers/tinkerbell/stack/testdata/expected_with_registry_mirror.yaml
+++ b/pkg/providers/tinkerbell/stack/testdata/expected_with_registry_mirror.yaml
@@ -7,20 +7,13 @@ rufio:
   - -metrics-bind-address=127.0.0.1:8080
   image: public.ecr.aws/eks-anywhere/rufio:latest
 smee:
-  additionalEnv:
-  - name: DATA_MODEL_VERSION
-    value: kubernetes
-  - name: TINKERBELL_TLS
-    value: "false"
-  - name: REGISTRY_USERNAME
-    value: username
-  - name: REGISTRY_PASSWORD
-    value: password
-  - name: SMEE_EXTRA_KERNEL_ARGS
-    value: tink_worker_image=1.2.3.4:443/custom/eks-anywhere/tink-worker:latest insecure_registries=1.2.3.4:443
   deploy: true
   hostNetwork: true
   http:
+    additionalKernelArgs:
+    - insecure_registries=1.2.3.4:443
+    - registry_username=username
+    - registry_password=password
     osieUrl:
       host: anywhere-assests.eks.amazonaws.com
       path: /tinkerbell/hook

From 14d38123adea7dad6db3e798da0e8df19d9e23ad Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Fri, 21 Jun 2024 09:27:55 -0700
Subject: [PATCH 10/32] Support installing previous version of EKS-A CLI using
 Homebrew (#8360)

Co-authored-by: Abhay Krishna Arunachalam <arnchlm@amazon.com>
---
 scripts/brew_formula_pr.sh     |  6 ++----
 scripts/brew_formula_update.sh | 35 ++++++++++++++++++++++++----------
 scripts/common.sh              | 32 +++++++++++++++++++++++++++++++
 scripts/eks-anywhere.rb.tmpl   |  2 +-
 4 files changed, 60 insertions(+), 15 deletions(-)

diff --git a/scripts/brew_formula_pr.sh b/scripts/brew_formula_pr.sh
index 528cd68f67bd..b00616badbe4 100755
--- a/scripts/brew_formula_pr.sh
+++ b/scripts/brew_formula_pr.sh
@@ -32,8 +32,6 @@ By submitting this pull request, I confirm that you can use, modify, copy, and r
 EOF
 )
 
-cd ${SCRIPT_ROOT}/
-cd ../../
 gh auth login --with-token < /secrets/github-secrets/token
 
 cd ${SCRIPT_ROOT}
@@ -44,7 +42,7 @@ then
   exit 1
 fi
 
-LATEST_VERSION=$(echo $(/home/prow/go/src/github.com/aws/eks-anywhere/scripts/$BREW_UPDATE_SCRIPT))
+LATEST_VERSION=$(echo $($SCRIPT_ROOT/$BREW_UPDATE_SCRIPT))
 
 cd ${SCRIPT_ROOT}/../../../${ORIGIN_ORG}/${REPO}
 git config --global push.default current
@@ -62,7 +60,7 @@ PR_BRANCH="eks-anywhere-formula-update"
 git checkout -b $PR_BRANCH
 
 git diff
-git add Formula/eks-anywhere.rb
+git add Formula/eks-anywhere*.rb
 # If some other files get modified, the changes should be ignored
 git restore .
 FILES_ADDED=$(git diff --staged --name-only)
diff --git a/scripts/brew_formula_update.sh b/scripts/brew_formula_update.sh
index 55d29fc3a059..5f8f23a32fe8 100755
--- a/scripts/brew_formula_update.sh
+++ b/scripts/brew_formula_update.sh
@@ -16,7 +16,9 @@ SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 REPO="homebrew-tap"
 ORIGIN_ORG="eks-anywhere-brew-pr-bot"
 UPSTREAM_ORG="aws"
+EKS_ANYWHERE_RELEASES_MANIFEST_URL="https://anywhere-assets.eks.amazonaws.com/releases/eks-a/manifest.yaml"
 YQ_LATEST_RELEASE_URL="https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64"
+source "$SCRIPT_ROOT/common.sh"
 
 if ! command -v yq &> /dev/null
 then
@@ -24,9 +26,21 @@ then
   chmod a+x /usr/local/bin/yq
 fi
 
-curl --silent 'https://anywhere-assets.eks.amazonaws.com/releases/eks-a/manifest.yaml' -o release.yaml
+curl --silent $EKS_ANYWHERE_RELEASES_MANIFEST_URL -o release.yaml
 
-latest_release_version=$(cat release.yaml | yq e '.spec.latestVersion')
+BRANCH_NAME=$PULL_BASE_REF
+ALL_RELEASE_BRANCHES=($(build::common::get_release_branches))
+LATEST_RELEASE_BRANCHES=($(get_latest_release_branches "${ALL_RELEASE_BRANCHES[@]}"))
+
+# Get the latest release version corresponding to this release branch
+if [[ "$BRANCH_NAME" == "${LATEST_RELEASE_BRANCHES[0]}" ]]; then
+  latest_release_version=$(cat release.yaml | yq e '.spec.latestVersion')
+elif [[ "$BRANCH_NAME" == "${LATEST_RELEASE_BRANCHES[1]}" ]]; then
+  latest_release_version=$(build::common::get_latest_release_for_release_branch $BRANCH_NAME)
+else
+  echo "Unsupported EKS Anywhere release branch $BRANCH_NAME!"
+  exit 1
+fi
 
 latest_release=$(cat release.yaml | yq e '.spec.releases[] | select(.version == "'$latest_release_version'")' >> latest_release.yaml)
 
@@ -46,7 +60,14 @@ latest_release_version="${latest_release_version:1}"
 export VERSION=$latest_release_version
 
 EKSA_TEMPLATE="eks-anywhere.rb.tmpl"
-EKSA_FORMULA="${SCRIPT_ROOT}/../../../${ORIGIN_ORG}/${REPO}/Formula/eks-anywhere.rb"
+if [[ "$BRANCH_NAME" == "${LATEST_RELEASE_BRANCHES[0]}" ]]; then
+  EKSA_FORMULA="${SCRIPT_ROOT}/../../../${ORIGIN_ORG}/${REPO}/Formula/eks-anywhere.rb"
+  export VERSION_SUFFIX=""
+elif [[ "$BRANCH_NAME" == "${LATEST_RELEASE_BRANCHES[1]}" ]]; then
+  minor_version="${BRANCH_NAME##release-}"
+  EKSA_FORMULA="${SCRIPT_ROOT}/../../../${ORIGIN_ORG}/${REPO}/Formula/eks-anywhere@$minor_version.rb"
+  export VERSION_SUFFIX="AT${minor_version/.}"
+fi
 
 if [ ! -f "$EKSA_TEMPLATE" ]
 then
@@ -54,13 +75,7 @@ then
   exit 1
 fi
 
-if [ ! -f "$EKSA_FORMULA" ]
-then
-  echo "Can not find the ${EKSA_FORMULA} file, exiting.."
-  exit 1
-fi
-
-envsubst '$VERSION:$darwin_arm64_url:$darwin_arm64_sha256:$darwin_amd64_url:$darwin_amd64_sha256:$linux_arm64_url:$linux_arm64_sha256:$linux_amd64_url:$linux_amd64_sha256' \
+envsubst '$VERSION:$VERSION_SUFFIX:$darwin_arm64_url:$darwin_arm64_sha256:$darwin_amd64_url:$darwin_amd64_sha256:$linux_arm64_url:$linux_arm64_sha256:$linux_amd64_url:$linux_amd64_sha256' \
  < "${EKSA_TEMPLATE}" \
  > "${EKSA_FORMULA}"
 
diff --git a/scripts/common.sh b/scripts/common.sh
index 902b5093c735..8e8cc1da2e58 100755
--- a/scripts/common.sh
+++ b/scripts/common.sh
@@ -209,3 +209,35 @@ function build::common::re_quote() {
     local -r to_escape=$1
     sed 's/[][()\.^$\/?*+]/\\&/g' <<< "$to_escape"
 }
+
+# This function returns all release branches of EKS Anywhere from GitHub.
+function build::common::get_release_branches() {
+  echo "$(git ls-remote https://github.com/aws/eks-anywhere.git | grep -oE 'refs/heads/release-[0-9]+\.[0-9]+' | xargs -I_ echo _ | cut -d/ -f3)"
+}
+
+# This function returns the two latest release branches of EKS Anywhere,
+# that is, release branches for versions N and N-1 in that order.
+function build::common::get_latest_release_branches() {
+  local -r release_branches=("$@")
+
+  latest_minor=0
+  for branch in "${release_branches[@]}"; do
+    minor_version=${branch##release-0.}
+    if [ $minor_version -gt $latest_minor ] && [ "$(get_latest_release_for_release_branch $branch)" != "" ]; then
+      latest_minor=$minor_version
+    fi
+  done
+
+  echo "release-0.${latest_minor} release-0.$(($latest_minor-1))"
+}
+
+# This function returns the latest release version for the given release branch
+# by parsing the EKS Anywhere releases manifest.
+function build::common::get_latest_release_for_release_branch() {
+  release_branch=$1
+
+  minor_version="v${release_branch##release-}"
+  latest_release=$(curl --silent "https://anywhere-assets.eks.amazonaws.com/releases/eks-a/manifest.yaml" | yq ".spec.releases[].version" | grep $minor_version | tail -1)
+
+  echo $latest_release
+}
diff --git a/scripts/eks-anywhere.rb.tmpl b/scripts/eks-anywhere.rb.tmpl
index 821ac491e268..da142df3bd93 100644
--- a/scripts/eks-anywhere.rb.tmpl
+++ b/scripts/eks-anywhere.rb.tmpl
@@ -1,4 +1,4 @@
-class EksAnywhere < Formula
+class EksAnywhere$VERSION_SUFFIX < Formula
   desc "CLI for managing EKS Anywhere Kubernetes clusters"
   homepage "https://github.com/aws/eks-anywhere"
   version "$VERSION"

From 0215fb3d7d78b6d79fd1cb484e2c03d8880bfac7 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Fri, 21 Jun 2024 17:21:56 -0700
Subject: [PATCH 11/32] Kubelet config e2e tests for vsphere and Docker (#8363)

Co-authored-by: Mitali Paygude <mitalipaygude@gmail.com>
---
 internal/pkg/api/cluster.go                   | 18 ++++
 .../cloudstackisolatednetwork_types.go        |  3 +-
 test/e2e/docker_test.go                       | 21 ++++
 test/e2e/kubeletconfig.go                     | 16 +++
 test/e2e/upgrade_from_latest.go               | 20 ++--
 test/e2e/vsphere_test.go                      | 21 ++++
 test/framework/kubeletconfig.go               | 99 +++++++++++++++++++
 test/framework/network.go                     |  2 +-
 test/framework/vsphere.go                     |  6 ++
 9 files changed, 194 insertions(+), 12 deletions(-)
 create mode 100644 test/e2e/kubeletconfig.go
 create mode 100644 test/framework/kubeletconfig.go

diff --git a/internal/pkg/api/cluster.go b/internal/pkg/api/cluster.go
index a2fd6f03b58a..3a7793d56a61 100644
--- a/internal/pkg/api/cluster.go
+++ b/internal/pkg/api/cluster.go
@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
 	anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1"
 	"github.com/aws/eks-anywhere/pkg/cluster"
@@ -137,6 +138,13 @@ func WithControlPlaneAPIServerExtraArgs() ClusterFiller {
 	}
 }
 
+// WithControlPlaneKubeletConfig adds the Kubelet config to the control plane in cluster spec.
+func WithControlPlaneKubeletConfig(kc *unstructured.Unstructured) ClusterFiller {
+	return func(c *anywherev1.Cluster) {
+		c.Spec.ControlPlaneConfiguration.KubeletConfiguration = kc
+	}
+}
+
 // RemoveAllAPIServerExtraArgs removes all the API server flags from the cluster spec.
 func RemoveAllAPIServerExtraArgs() ClusterFiller {
 	return func(c *anywherev1.Cluster) {
@@ -186,6 +194,16 @@ func workerNodeWithKubernetesVersion(name string, version *anywherev1.Kubernetes
 	}
 }
 
+// WithWorkerNodeKubeletConfig adds the Kubelet config to the worker node groups in cluster spec.
+func WithWorkerNodeKubeletConfig(kc *unstructured.Unstructured) ClusterFiller {
+	return func(c *anywherev1.Cluster) {
+		if len(c.Spec.WorkerNodeGroupConfigurations) == 0 {
+			c.Spec.WorkerNodeGroupConfigurations = []anywherev1.WorkerNodeGroupConfiguration{{}}
+		}
+		c.Spec.WorkerNodeGroupConfigurations[0].KubeletConfiguration = kc
+	}
+}
+
 func WithWorkerNodeCount(r int) ClusterFiller {
 	return func(c *anywherev1.Cluster) {
 		if len(c.Spec.WorkerNodeGroupConfigurations) == 0 {
diff --git a/internal/thirdparty/capc/api/v1beta3/cloudstackisolatednetwork_types.go b/internal/thirdparty/capc/api/v1beta3/cloudstackisolatednetwork_types.go
index 76c15995bbbf..db2630c1be65 100644
--- a/internal/thirdparty/capc/api/v1beta3/cloudstackisolatednetwork_types.go
+++ b/internal/thirdparty/capc/api/v1beta3/cloudstackisolatednetwork_types.go
@@ -57,7 +57,8 @@ func (n *CloudStackIsolatedNetwork) Network() *Network {
 	return &Network{
 		Name: n.Spec.Name,
 		Type: "IsolatedNetwork",
-		ID:   n.Spec.ID}
+		ID:   n.Spec.ID,
+	}
 }
 
 //+kubebuilder:object:root=true
diff --git a/test/e2e/docker_test.go b/test/e2e/docker_test.go
index fe9c915d4659..1557f73b5d57 100644
--- a/test/e2e/docker_test.go
+++ b/test/e2e/docker_test.go
@@ -1459,3 +1459,24 @@ func TestDockerKubernetes129to130EtcdScaleDown(t *testing.T) {
 		),
 	)
 }
+
+// Kubelet Configuration e2e tests
+func TestDockerKubernetes129KubeletConfigurationSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewDocker(t),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}
+
+func TestDockerKubernetes130KubeletConfigurationSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewDocker(t),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube130)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}
diff --git a/test/e2e/kubeletconfig.go b/test/e2e/kubeletconfig.go
new file mode 100644
index 000000000000..d10630abcede
--- /dev/null
+++ b/test/e2e/kubeletconfig.go
@@ -0,0 +1,16 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"github.com/aws/eks-anywhere/test/framework"
+)
+
+func runKubeletConfigurationFlow(test *framework.ClusterE2ETest) {
+	test.GenerateClusterConfig()
+	test.CreateCluster()
+	test.ValidateKubeletConfig()
+	test.StopIfFailed()
+	test.DeleteCluster()
+}
diff --git a/test/e2e/upgrade_from_latest.go b/test/e2e/upgrade_from_latest.go
index f3235fcd7219..0ba4407cf5dd 100644
--- a/test/e2e/upgrade_from_latest.go
+++ b/test/e2e/upgrade_from_latest.go
@@ -68,21 +68,21 @@ func runInPlaceUpgradeFromReleaseFlow(test *framework.ClusterE2ETest, latestRele
 
 // runMulticlusterUpgradeFromReleaseFlowAPI tests the ability to create workload clusters with an old Bundle in a management cluster
 // that has been updated to a new Bundle. It follows the following steps:
-// 1. Create a management cluster with the old Bundle.
-// 2. Create workload clusters with the old Bundle.
-// 3. Upgrade the management cluster to the new Bundle and new Kubernetes version (newVersion).
-// 4. Upgrade the workload clusters to the new Bundle and new Kubernetes version (newVersion).
-// 5. Delete the workload clusters.
-// 6. Re-create the workload clusters with the old Bundle and previous Kubernetes version (oldVersion). It's necessary to sometimes
-//    use a different kube version because the old Bundle might not support the new kubernetes version.
-// 7. Delete the workload clusters.
-// 8. Delete the management cluster.
+//  1. Create a management cluster with the old Bundle.
+//  2. Create workload clusters with the old Bundle.
+//  3. Upgrade the management cluster to the new Bundle and new Kubernetes version (newVersion).
+//  4. Upgrade the workload clusters to the new Bundle and new Kubernetes version (newVersion).
+//  5. Delete the workload clusters.
+//  6. Re-create the workload clusters with the old Bundle and previous Kubernetes version (oldVersion). It's necessary to sometimes
+//     use a different kube version because the old Bundle might not support the new kubernetes version.
+//  7. Delete the workload clusters.
+//  8. Delete the management cluster.
 func runMulticlusterUpgradeFromReleaseFlowAPI(test *framework.MulticlusterE2ETest, release *releasev1.EksARelease, oldVersion, newVersion anywherev1.KubernetesVersion, os framework.OS) {
 	provider := test.ManagementCluster.Provider
 	// 1. Create management cluster
 	test.CreateManagementCluster(framework.ExecuteWithEksaRelease(release))
 
-	// 2. Create workload clusters with the old Bundle  
+	// 2. Create workload clusters with the old Bundle
 	test.RunConcurrentlyInWorkloadClusters(func(wc *framework.WorkloadCluster) {
 		wc.CreateCluster(framework.ExecuteWithEksaRelease(release))
 		wc.ValidateCluster(wc.ClusterConfig.Cluster.Spec.KubernetesVersion)
diff --git a/test/e2e/vsphere_test.go b/test/e2e/vsphere_test.go
index 89fab09673af..8eb3b0399792 100644
--- a/test/e2e/vsphere_test.go
+++ b/test/e2e/vsphere_test.go
@@ -5993,3 +5993,24 @@ func TestVSphereKubernetes129to130UbuntuEtcdScaleDown(t *testing.T) {
 		provider.WithProviderUpgrade(provider.Ubuntu130Template()),
 	)
 }
+
+// Kubelet Configuration e2e tests
+func TestVSphereKubernetes129KubeletConfigurationSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewVSphere(t, framework.WithUbuntu129()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}
+
+func TestVSphereKubernetes130KubeletConfigurationSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewVSphere(t, framework.WithUbuntu130()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube130)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}
diff --git a/test/framework/kubeletconfig.go b/test/framework/kubeletconfig.go
new file mode 100644
index 000000000000..c851743629c5
--- /dev/null
+++ b/test/framework/kubeletconfig.go
@@ -0,0 +1,99 @@
+package framework
+
+import (
+	"context"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	"github.com/aws/eks-anywhere/internal/pkg/api"
+)
+
+const (
+	maxPod50 = 50
+	maxPod60 = 60
+)
+
+// WithKubeletConfig returns the default kubelet config set for e2e test.
+func WithKubeletConfig() ClusterE2ETestOpt {
+	return func(e *ClusterE2ETest) {
+		e.addClusterConfigFillers(WithKubeletClusterConfig())
+	}
+}
+
+// WithKubeletClusterConfig returns a ClusterConfigFiller that adds the default
+// KubeletConfig for E2E tests to the cluster Config.
+func WithKubeletClusterConfig() api.ClusterConfigFiller {
+	cpKubeletConfiguration := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"maxPods": maxPod50,
+			"kind":    "KubeletConfiguration",
+		},
+	}
+
+	wnKubeletConfiguration := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"maxPods": maxPod60,
+			"kind":    "KubeletConfiguration",
+		},
+	}
+
+	return api.JoinClusterConfigFillers(
+		api.ClusterToConfigFiller(api.WithControlPlaneKubeletConfig(cpKubeletConfiguration)),
+		api.ClusterToConfigFiller(api.WithWorkerNodeKubeletConfig(wnKubeletConfiguration)),
+	)
+}
+
+// ValidateKubeletConfig validates the kubelet config specified in the cluster spec has been applied to the nodes.
+func (e *ClusterE2ETest) ValidateKubeletConfig() {
+	ctx := context.Background()
+	kubectlClient := buildLocalKubectl()
+
+	e.T.Log("Getting control plane nodes for kubelet max pod verification")
+	nodes, err := kubectlClient.GetControlPlaneNodes(ctx,
+		e.KubeconfigFilePath(),
+	)
+	if err != nil {
+		e.T.Fatalf("Error getting nodes: %v", err)
+	}
+	if len(nodes) == 0 {
+		e.T.Fatalf("no control plane nodes found")
+	}
+
+	got, _ := nodes[0].Status.Capacity.Pods().AsInt64()
+	if got != int64(maxPod50) {
+		e.T.Fatalf("Node capacity for control plane pods not equal to %v", maxPod50)
+	}
+
+	e.T.Log("Successfully verified Kubelet Configuration for control plane nodes")
+
+	e.T.Log("Getting control plane nodes for kubelet max pod verification")
+	allNodes, err := kubectlClient.GetNodes(ctx,
+		e.KubeconfigFilePath(),
+	)
+	if err != nil {
+		e.T.Fatalf("Error getting nodes: %v", err)
+	}
+	if len(allNodes) == 0 {
+		e.T.Fatalf("no nodes found")
+	}
+
+	e.T.Log("Getting worker nodes for kubelet max pod verification")
+	var workerNode corev1.Node
+	for i := range allNodes {
+		if strings.Contains(allNodes[i].Name, "-md-") {
+			workerNode = allNodes[i]
+		}
+	}
+	if err != nil {
+		e.T.Fatalf("Error getting nodes: %v", err)
+	}
+
+	got, _ = workerNode.Status.Capacity.Pods().AsInt64()
+	if got != int64(maxPod60) {
+		e.T.Fatalf("Node capacity for worker node pods not equal to %v", maxPod60)
+	}
+
+	e.T.Log("Successfully verified Kubelet Configuration for worker nodes")
+}
diff --git a/test/framework/network.go b/test/framework/network.go
index 6fd0d20b9584..b28fa2add74a 100644
--- a/test/framework/network.go
+++ b/test/framework/network.go
@@ -54,7 +54,7 @@ func GetIP(cidr, ipEnvVar string) (string, error) {
 	} else {
 		ip, err = GenerateUniqueIp(cidr)
 		if err != nil {
-			return "", fmt.Errorf("failed to generate ip for cidr %s: %v", cidr, err)
+			return "", fmt.Errorf("GenerateUniqueIp() failed to generate ip for cidr %s: %v", cidr, err)
 		}
 	}
 	return ip, nil
diff --git a/test/framework/vsphere.go b/test/framework/vsphere.go
index 1b124d7abcb7..3328c5a0d378 100644
--- a/test/framework/vsphere.go
+++ b/test/framework/vsphere.go
@@ -413,6 +413,12 @@ func (v *VSphere) WithUbuntu128() api.ClusterConfigFiller {
 	return v.WithKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, nil)
 }
 
+// WithUbuntu129 returns a cluster config filler that sets the kubernetes version of the cluster to 1.29
+// as well as the right ubuntu template and osFamily for all VSphereMachineConfigs.
+func (v *VSphere) WithUbuntu129() api.ClusterConfigFiller {
+	return v.WithKubeVersionAndOS(anywherev1.Kube129, Ubuntu2004, nil)
+}
+
 // WithUbuntu130 returns a cluster config filler that sets the kubernetes version of the cluster to 1.30
 // as well as the right ubuntu template and osFamily for all VSphereMachineConfigs.
 func (v *VSphere) WithUbuntu130() api.ClusterConfigFiller {

From c014c30250b176dd0ee1ce3cee8ed0eead8b08b9 Mon Sep 17 00:00:00 2001
From: Rahul Ganesh <31204974+rahulbabu95@users.noreply.github.com>
Date: Sat, 22 Jun 2024 22:09:57 -0700
Subject: [PATCH 12/32] trigger staging bundles for v0.20.0 (#8358)

Signed-off-by: Rahul Ganesh <rahulgab@amazon.com>
Co-authored-by: Rahul Ganesh <rahulgab@amazon.com>
---
 release/triggers/bundle-release/development/BUNDLE_NUMBER   | 2 +-
 release/triggers/bundle-release/development/CLI_MAX_VERSION | 2 +-
 release/triggers/bundle-release/development/CLI_MIN_VERSION | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/release/triggers/bundle-release/development/BUNDLE_NUMBER b/release/triggers/bundle-release/development/BUNDLE_NUMBER
index c5b431b6cba2..3d9aebb2cce8 100644
--- a/release/triggers/bundle-release/development/BUNDLE_NUMBER
+++ b/release/triggers/bundle-release/development/BUNDLE_NUMBER
@@ -1 +1 @@
-50
\ No newline at end of file
+68
\ No newline at end of file
diff --git a/release/triggers/bundle-release/development/CLI_MAX_VERSION b/release/triggers/bundle-release/development/CLI_MAX_VERSION
index ade995fb77a3..8c20c5272156 100644
--- a/release/triggers/bundle-release/development/CLI_MAX_VERSION
+++ b/release/triggers/bundle-release/development/CLI_MAX_VERSION
@@ -1 +1 @@
-v0.18.0
\ No newline at end of file
+v0.20.0
\ No newline at end of file
diff --git a/release/triggers/bundle-release/development/CLI_MIN_VERSION b/release/triggers/bundle-release/development/CLI_MIN_VERSION
index ade995fb77a3..8c20c5272156 100644
--- a/release/triggers/bundle-release/development/CLI_MIN_VERSION
+++ b/release/triggers/bundle-release/development/CLI_MIN_VERSION
@@ -1 +1 @@
-v0.18.0
\ No newline at end of file
+v0.20.0
\ No newline at end of file

From a15067ddd5610d3b728aaa14e954238548b7dcf2 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 24 Jun 2024 10:18:58 -0700
Subject: [PATCH 13/32] Handle nil commandOut (#8367)

Co-authored-by: Xu Deng <xudeng@amazon.com>
---
 internal/pkg/ssm/run_output.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/internal/pkg/ssm/run_output.go b/internal/pkg/ssm/run_output.go
index d6f3be8ee7f2..a8a4c509cd4e 100644
--- a/internal/pkg/ssm/run_output.go
+++ b/internal/pkg/ssm/run_output.go
@@ -23,5 +23,9 @@ func (r *RunOutput) Successful() bool {
 
 // StatusDetails returns the status details of the ssm command.
 func (r *RunOutput) StatusDetails() string {
+	// handle nil pointer
+	if r.commandOut == nil {
+		return "NoCommandOutput"
+	}
 	return *r.commandOut.StatusDetails
 }

From 4ad96668f37407fa24425a7d5c7419d77656fa0e Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 24 Jun 2024 11:06:58 -0700
Subject: [PATCH 14/32] Change delete timeout to 30 mins (#8368)

Co-authored-by: ahreehong <46465244+ahreehong@users.noreply.github.com>
---
 pkg/dependencies/factory.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/dependencies/factory.go b/pkg/dependencies/factory.go
index 60f855589f86..6921ae1cfc86 100644
--- a/pkg/dependencies/factory.go
+++ b/pkg/dependencies/factory.go
@@ -1210,7 +1210,7 @@ func (f *Factory) WithClusterDeleter() *Factory {
 	f.buildSteps = append(f.buildSteps, func(ctx context.Context) error {
 		var opts []clustermanager.DeleterOpt
 		if f.config.noTimeouts {
-			opts = append(opts, clustermanager.WithDeleterApplyClusterTimeout(time.Hour))
+			opts = append(opts, clustermanager.WithDeleterApplyClusterTimeout(30*time.Minute))
 		}
 
 		f.dependencies.ClusterDeleter = clustermanager.NewDeleter(

From d2be185a81a0852e4f726ff0d5555badb25510b6 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 24 Jun 2024 11:51:58 -0700
Subject: [PATCH 15/32] Add descriptive error logs for download, rename and
 upload operations (#8370)

Co-authored-by: Abhay Krishna Arunachalam <arnchlm@amazon.com>
---
 release/cli/pkg/assets/archives/archives.go | 1 +
 release/cli/pkg/operations/download.go      | 6 +++---
 release/cli/pkg/operations/rename.go        | 6 +++---
 release/cli/pkg/operations/upload.go        | 9 ++++-----
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/release/cli/pkg/assets/archives/archives.go b/release/cli/pkg/assets/archives/archives.go
index 4044c5c49615..5e2fe49d2010 100644
--- a/release/cli/pkg/assets/archives/archives.go
+++ b/release/cli/pkg/assets/archives/archives.go
@@ -150,6 +150,7 @@ func RTOSArtifactPathGetter(rc *releasetypes.ReleaseConfig, archive *assettypes.
 	var sourceS3Prefix string
 	var releaseS3Path string
 	var releaseName string
+	eksDReleaseChannel = "1-29" // hardcoding because we vend RTOS artifacts only for 1-29 release branch
 
 	imageExtensions := map[string]string{
 		"ami": "gz",
diff --git a/release/cli/pkg/operations/download.go b/release/cli/pkg/operations/download.go
index f3c87ed7eb62..29cc0c0cfc62 100644
--- a/release/cli/pkg/operations/download.go
+++ b/release/cli/pkg/operations/download.go
@@ -126,7 +126,7 @@ func handleArchiveDownload(_ context.Context, r *releasetypes.ReleaseConfig, art
 
 		err = s3.DownloadFile(objectLocalFilePath, r.SourceBucket, objectKey, s3Downloader, artifact.Archive.Private)
 		if err != nil {
-			return errors.Cause(err)
+			return fmt.Errorf("downloading archive file [%s] from S3: %v", objectKey, err)
 		}
 
 		// Download checksum files for the archive
@@ -180,7 +180,7 @@ func handleArchiveDownload(_ context.Context, r *releasetypes.ReleaseConfig, art
 
 			err = s3.DownloadFile(objectShasumFileLocalFilePath, r.SourceBucket, objectShasumFileKey, s3Downloader, artifact.Archive.Private)
 			if err != nil {
-				return errors.Cause(err)
+				return fmt.Errorf("downloading checksum file [%s] from S3: %v", objectShasumFileKey, err)
 			}
 		}
 	}
@@ -223,7 +223,7 @@ func handleManifestDownload(_ context.Context, r *releasetypes.ReleaseConfig, ar
 
 	err = s3.DownloadFile(objectLocalFilePath, r.SourceBucket, objectKey, s3Downloader, artifact.Manifest.Private)
 	if err != nil {
-		return errors.Cause(err)
+		return fmt.Errorf("downloading manifest file [%s] from S3: %v", objectKey, err)
 	}
 
 	return nil
diff --git a/release/cli/pkg/operations/rename.go b/release/cli/pkg/operations/rename.go
index baf9764969f5..4304aa719920 100644
--- a/release/cli/pkg/operations/rename.go
+++ b/release/cli/pkg/operations/rename.go
@@ -73,7 +73,7 @@ func handleArchiveRename(_ context.Context, r *releasetypes.ReleaseConfig, artif
 		fmt.Printf("Renaming archive - %s\n", newArtifactFile)
 		err := os.Rename(oldArtifactFile, newArtifactFile)
 		if err != nil {
-			return errors.Cause(err)
+			return fmt.Errorf("renaming archive file [%s] to [%s]: %v", oldArtifactFile, newArtifactFile, err)
 		}
 
 		// Change the names of the checksum files
@@ -92,7 +92,7 @@ func handleArchiveRename(_ context.Context, r *releasetypes.ReleaseConfig, artif
 			fmt.Printf("Renaming checksum file - %s\n", newChecksumFile)
 			err = os.Rename(oldChecksumFile, newChecksumFile)
 			if err != nil {
-				return errors.Cause(err)
+				return fmt.Errorf("renaming checksum file [%s] to [%s]: %v", oldChecksumFile, newChecksumFile, err)
 			}
 		}
 	}
@@ -107,7 +107,7 @@ func handleManifestRename(_ context.Context, r *releasetypes.ReleaseConfig, arti
 	fmt.Printf("Renaming manifest - %s\n", newArtifactFile)
 	err := os.Rename(oldArtifactFile, newArtifactFile)
 	if err != nil {
-		return errors.Cause(err)
+		return fmt.Errorf("renaming manifest file [%s] to [%s]: %v", oldArtifactFile, newArtifactFile, err)
 	}
 
 	for _, imageTagOverride := range manifestArtifact.ImageTagOverrides {
diff --git a/release/cli/pkg/operations/upload.go b/release/cli/pkg/operations/upload.go
index 06fb98f99c70..b81f7207d4bd 100644
--- a/release/cli/pkg/operations/upload.go
+++ b/release/cli/pkg/operations/upload.go
@@ -22,7 +22,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	docker "github.com/fsouza/go-dockerclient"
-	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/aws/eks-anywhere/release/cli/pkg/aws/s3"
@@ -94,7 +93,7 @@ func handleArchiveUpload(_ context.Context, r *releasetypes.ReleaseConfig, artif
 	key := filepath.Join(artifact.Archive.ReleaseS3Path, artifact.Archive.ReleaseName)
 	err := s3.UploadFile(archiveFile, aws.String(r.ReleaseBucket), aws.String(key), r.ReleaseClients.S3.Uploader, artifact.Archive.Private)
 	if err != nil {
-		return errors.Cause(err)
+		return fmt.Errorf("uploading archive file [%s] to S3: %v", key, err)
 	}
 
 	checksumExtensions := []string{".sha256", ".sha512"}
@@ -111,7 +110,7 @@ func handleArchiveUpload(_ context.Context, r *releasetypes.ReleaseConfig, artif
 		key := filepath.Join(artifact.Archive.ReleaseS3Path, artifact.Archive.ReleaseName) + extension
 		err := s3.UploadFile(checksumFile, aws.String(r.ReleaseBucket), aws.String(key), r.ReleaseClients.S3.Uploader, artifact.Archive.Private)
 		if err != nil {
-			return errors.Cause(err)
+			return fmt.Errorf("uploading checksum file [%s] to S3: %v", key, err)
 		}
 	}
 
@@ -124,7 +123,7 @@ func handleManifestUpload(_ context.Context, r *releasetypes.ReleaseConfig, arti
 	key := filepath.Join(artifact.Manifest.ReleaseS3Path, artifact.Manifest.ReleaseName)
 	err := s3.UploadFile(manifestFile, aws.String(r.ReleaseBucket), aws.String(key), r.ReleaseClients.S3.Uploader, artifact.Manifest.Private)
 	if err != nil {
-		return errors.Cause(err)
+		return fmt.Errorf("uploading manifest file [%s] to S3: %v", key, err)
 	}
 
 	return nil
@@ -161,7 +160,7 @@ func handleImageUpload(_ context.Context, r *releasetypes.ReleaseConfig, package
 		fmt.Printf("Destination Image - %s\n", releaseImageUri)
 		err := images.CopyToDestination(sourceEcrAuthConfig, releaseEcrAuthConfig, sourceImageUri, releaseImageUri)
 		if err != nil {
-			return fmt.Errorf("copying image from source to destination: %v", err)
+			return fmt.Errorf("copying image from source [%s] to destination [%s]: %v", sourceImageUri, releaseImageUri, err)
 		}
 	}
 

From 3d6c158ee254d26268e6e78c7482043f14b5832c Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 24 Jun 2024 12:41:10 -0700
Subject: [PATCH 16/32] Create S3 downloader using source session instead of
 release session (#8375)

Co-authored-by: Abhay Krishna Arunachalam <arnchlm@amazon.com>
---
 release/cli/pkg/clients/clients.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/cli/pkg/clients/clients.go b/release/cli/pkg/clients/clients.go
index 88cbeca53476..d9deb99939d9 100644
--- a/release/cli/pkg/clients/clients.go
+++ b/release/cli/pkg/clients/clients.go
@@ -165,7 +165,7 @@ func CreateStagingReleaseClients() (*SourceClients, *ReleaseClients, error) {
 
 	// Release S3 client and uploader
 	releaseS3Client := s3.New(releaseSession)
-	downloader := s3manager.NewDownloader(releaseSession)
+	downloader := s3manager.NewDownloader(sourceSession)
 	uploader := s3manager.NewUploader(releaseSession)
 
 	// Get source ECR auth config
@@ -242,7 +242,7 @@ func CreateProdReleaseClients() (*SourceClients, *ReleaseClients, error) {
 
 	// Release S3 client and uploader
 	releaseS3Client := s3.New(releaseSession)
-	downloader := s3manager.NewDownloader(releaseSession)
+	downloader := s3manager.NewDownloader(sourceSession)
 	uploader := s3manager.NewUploader(releaseSession)
 
 	// Get source ECR Public auth config

From ed17522fe59d8850ffb165dcbeb812ef1920b404 Mon Sep 17 00:00:00 2001
From: Rahul Ganesh <31204974+rahulbabu95@users.noreply.github.com>
Date: Mon, 24 Jun 2024 15:10:15 -0700
Subject: [PATCH 17/32] Trigger staging CLI for release 0.20 (#8376)

Signed-off-by: Rahul Ganesh <rahulgab@amazon.com>
Co-authored-by: Rahul Ganesh <rahulgab@amazon.com>
---
 release/triggers/eks-a-release/development/RELEASE_NUMBER  | 2 +-
 release/triggers/eks-a-release/development/RELEASE_VERSION | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/triggers/eks-a-release/development/RELEASE_NUMBER b/release/triggers/eks-a-release/development/RELEASE_NUMBER
index a2720097dccb..38b10c1b2bad 100644
--- a/release/triggers/eks-a-release/development/RELEASE_NUMBER
+++ b/release/triggers/eks-a-release/development/RELEASE_NUMBER
@@ -1 +1 @@
-39
+68
diff --git a/release/triggers/eks-a-release/development/RELEASE_VERSION b/release/triggers/eks-a-release/development/RELEASE_VERSION
index 49dd460a9f88..8c20c5272156 100644
--- a/release/triggers/eks-a-release/development/RELEASE_VERSION
+++ b/release/triggers/eks-a-release/development/RELEASE_VERSION
@@ -1 +1 @@
-v0.16.0
\ No newline at end of file
+v0.20.0
\ No newline at end of file

From 49f99859ac4aee368f0d255e37c1869321ee39dd Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 24 Jun 2024 15:45:58 -0700
Subject: [PATCH 18/32] [release-0.20] Kubelet config e2e tests for Nutanix,
 Cloudstack and Tinkerbell (#8377)

* Kubelet config e2e tests for Nutanix, Cloudstack and Tinkerbell

* m

---------

Co-authored-by: Mitali Paygude <mitalipaygude@gmail.com>
---
 pkg/providers/tinkerbell/upgrade.go     |  1 -
 test/e2e/TINKERBELL_HARDWARE_COUNT.yaml |  4 +++-
 test/e2e/cloudstack_test.go             | 21 +++++++++++++++++++++
 test/e2e/nutanix_test.go                | 21 +++++++++++++++++++++
 test/e2e/tinkerbell_test.go             | 21 +++++++++++++++++++++
 5 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/pkg/providers/tinkerbell/upgrade.go b/pkg/providers/tinkerbell/upgrade.go
index 178f0f644893..7d17fc2c13bc 100644
--- a/pkg/providers/tinkerbell/upgrade.go
+++ b/pkg/providers/tinkerbell/upgrade.go
@@ -123,7 +123,6 @@ func (p *Provider) SetupAndValidateUpgradeCluster(ctx context.Context, cluster *
 	}
 
 	if p.clusterConfig.IsManaged() {
-
 		// Update stack helm enviorment variable NO_PROXY value and append management cluster's Control plane Endpoint IP in case of workload cluster upgrade
 		if clusterSpec.Cluster.Spec.ProxyConfiguration != nil {
 			managementCluster, err := p.providerKubectlClient.GetEksaCluster(ctx, clusterSpec.ManagementCluster, clusterSpec.Cluster.Spec.ManagementCluster.Name)
diff --git a/test/e2e/TINKERBELL_HARDWARE_COUNT.yaml b/test/e2e/TINKERBELL_HARDWARE_COUNT.yaml
index ff49aa6df877..9f0516e6dd22 100644
--- a/test/e2e/TINKERBELL_HARDWARE_COUNT.yaml
+++ b/test/e2e/TINKERBELL_HARDWARE_COUNT.yaml
@@ -118,4 +118,6 @@ TestTinkerbellKubernetes128UbuntuTo129SingleNodeInPlaceUpgrade: 1
 TestTinkerbellKubernetes129UbuntuTo130SingleNodeInPlaceUpgrade: 1
 TestTinkerbellKubernetes128UpgradeManagementComponents: 2
 TestTinkerbellKubernetes126UbuntuTo130MultipleUpgrade: 4
-TestTinkerbellKubernetes130UbuntuAddWorkerNodeGroupWithAPI: 2
\ No newline at end of file
+TestTinkerbellKubernetes130UbuntuAddWorkerNodeGroupWithAPI: 2
+TestTinkerbellKubernetes129KubeletConfigurationSimpleFlow: 2
+TestTinkerbellKubernetes130KubeletConfigurationSimpleFlow: 2
\ No newline at end of file
diff --git a/test/e2e/cloudstack_test.go b/test/e2e/cloudstack_test.go
index b48ab6870042..275b6a077aa9 100644
--- a/test/e2e/cloudstack_test.go
+++ b/test/e2e/cloudstack_test.go
@@ -4204,3 +4204,24 @@ func TestCloudStackKubernetes127to128EtcdScaleDown(t *testing.T) {
 		provider.WithProviderUpgrade(provider.Redhat9Kubernetes128Template()),
 	)
 }
+
+// Kubelet Configuration e2e tests
+func TestCloudStackKubernetes129KubeletConfigurationSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewCloudStack(t, framework.WithCloudStackRedhat129()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}
+
+func TestCloudStackKubernetes130KubeletConfigurationSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewCloudStack(t, framework.WithCloudStackRedhat130()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube130)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}
diff --git a/test/e2e/nutanix_test.go b/test/e2e/nutanix_test.go
index 6dcc5c200e94..bbcaa37e4a3c 100644
--- a/test/e2e/nutanix_test.go
+++ b/test/e2e/nutanix_test.go
@@ -1359,3 +1359,24 @@ func TestNutanixKubernetes130UbuntuManagementCPUpgradeAPI(t *testing.T) {
 		),
 	)
 }
+
+// Kubelet Configuration e2e tests
+func TestNutanixKubernetes129KubeletConfigurationSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewNutanix(t, framework.WithUbuntu129Nutanix()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}
+
+func TestNutanixKubernetes130KubeletConfigurationSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewNutanix(t, framework.WithUbuntu130Nutanix()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube130)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}
diff --git a/test/e2e/tinkerbell_test.go b/test/e2e/tinkerbell_test.go
index b2ab843eb729..4cb6bf07400a 100644
--- a/test/e2e/tinkerbell_test.go
+++ b/test/e2e/tinkerbell_test.go
@@ -1718,3 +1718,24 @@ func TestTinkerbellKubernetes126UbuntuTo130MultipleUpgrade(t *testing.T) {
 		kube130clusterOpts,
 	)
 }
+
+// Kubelet Configuration e2e tests
+func TestTinkerbellKubernetes129KubeletConfigurationSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewTinkerbell(t, framework.WithUbuntu129Tinkerbell()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}
+
+func TestTinkerbellKubernetes130KubeletConfigurationSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewTinkerbell(t, framework.WithUbuntu130Tinkerbell()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube130)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}

From a9428e99be8aed77fbdf7806d75d2a486f720410 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 24 Jun 2024 16:54:56 -0700
Subject: [PATCH 19/32] Revert "ensure release branch is included in teest
 cluster name (#8338)" (#8381)

Manually merging in as all the tests passed in the PR on main and this is just a cherrypick.

This reverts commit 804eb68cb367abb0b128c719409ba56bd94586fa.

Co-authored-by: Cavaughn Browne <113555337+cxbrowne1207@users.noreply.github.com>
---
 internal/test/e2e/setup.go | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/internal/test/e2e/setup.go b/internal/test/e2e/setup.go
index fee99bb0e8e9..ee66c8bcd3c8 100644
--- a/internal/test/e2e/setup.go
+++ b/internal/test/e2e/setup.go
@@ -285,6 +285,10 @@ func clusterPrefix(branch, instanceId string) (clusterPrefix string) {
 		sanitizedBranch = strings.ReplaceAll(sanitizedBranch, char, "-")
 	}
 
+	if len(sanitizedBranch) > 7 {
+		sanitizedBranch = sanitizedBranch[:7]
+	}
+
 	if len(instanceId) > 7 {
 		instanceId = instanceId[:7]
 	}
@@ -295,9 +299,9 @@ func clusterPrefix(branch, instanceId string) (clusterPrefix string) {
 
 func (e *E2ESession) clusterName(branch, instanceId, testName string) (clusterName string) {
 	clusterName = fmt.Sprintf("%s-%s", clusterPrefix(branch, instanceId), e2etests.GetTestNameHash(testName))
-	if len(clusterName) > 35 {
-		e.logger.Info("Cluster name is longer than 35 characters; truncating to 35 characters.", "original cluster name", clusterName, "truncated cluster name", clusterName[:35])
-		clusterName = clusterName[:35]
+	if len(clusterName) > 63 {
+		e.logger.Info("Cluster name is longer than 63 characters; truncating to 63 characters.", "original cluster name", clusterName, "truncated cluster name", clusterName[:63])
+		clusterName = clusterName[:63]
 	}
 	return clusterName
 }

From 305485f2fc9bc6854726176def20059199049bab Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 24 Jun 2024 17:21:14 -0700
Subject: [PATCH 20/32] lower concurrent vsphere e2e run count (#8382)

Skip E2E presubmit as this is just an infra level change and has nothing to do with the code.

Co-authored-by: Cavaughn Browne <cxbrowne@amazon.com>
---
 .../build/buildspecs/vsphere-test-eks-a-cli.yml                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/integration_test/build/buildspecs/vsphere-test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/vsphere-test-eks-a-cli.yml
index 6758074837a0..2406befb455c 100644
--- a/cmd/integration_test/build/buildspecs/vsphere-test-eks-a-cli.yml
+++ b/cmd/integration_test/build/buildspecs/vsphere-test-eks-a-cli.yml
@@ -3,7 +3,7 @@ version: 0.2
 env:
   variables:
     INTEGRATION_TEST_MAX_EC2_COUNT: 180
-    INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 100
+    INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 80
     EKSA_GIT_KNOWN_HOSTS: "/tmp/known_hosts"
     EKSA_GIT_PRIVATE_KEY: "/tmp/private-key"
     INTEGRATION_TEST_INFRA_CONFIG: "/tmp/test-infra.yml"

From 43a446237e381cf62cbf3e72a92d64e0995083b9 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 24 Jun 2024 20:14:58 -0700
Subject: [PATCH 21/32] Upload RTOS artifacts to separate bucket (#8384)

Co-authored-by: Abhay Krishna Arunachalam <arnchlm@amazon.com>
---
 release/cli/pkg/assets/archives/archives.go   | 29 ++++++++++---------
 .../cli/pkg/assets/config/bundle_release.go   |  1 +
 release/cli/pkg/assets/types/types.go         |  1 +
 release/cli/pkg/constants/constants.go        |  3 +-
 release/cli/pkg/operations/upload.go          |  9 ++++--
 release/cli/pkg/types/types.go                | 29 ++++++++++---------
 6 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/release/cli/pkg/assets/archives/archives.go b/release/cli/pkg/assets/archives/archives.go
index 5e2fe49d2010..57390a425efc 100644
--- a/release/cli/pkg/assets/archives/archives.go
+++ b/release/cli/pkg/assets/archives/archives.go
@@ -223,20 +223,21 @@ func GetArchiveAssets(rc *releasetypes.ReleaseConfig, archive *assettypes.Archiv
 	}
 
 	archiveArtifact := &releasetypes.ArchiveArtifact{
-		SourceS3Key:       sourceS3Key,
-		SourceS3Prefix:    sourceS3Prefix,
-		ArtifactPath:      filepath.Join(rc.ArtifactDir, fmt.Sprintf("%s-%s", archive.Name, archive.Format), eksDReleaseChannel, rc.BuildRepoHead),
-		ReleaseName:       releaseName,
-		ReleaseS3Path:     releaseS3Path,
-		ReleaseCdnURI:     cdnURI,
-		OS:                os,
-		OSName:            archive.OSName,
-		Arch:              []string{arch},
-		GitTag:            gitTag,
-		ProjectPath:       projectPath,
-		SourcedFromBranch: sourcedFromBranch,
-		ImageFormat:       archive.Format,
-		Private:           archive.Private,
+		SourceS3Key:        sourceS3Key,
+		SourceS3Prefix:     sourceS3Prefix,
+		ArtifactPath:       filepath.Join(rc.ArtifactDir, fmt.Sprintf("%s-%s", archive.Name, archive.Format), eksDReleaseChannel, rc.BuildRepoHead),
+		ReleaseName:        releaseName,
+		ReleaseS3Path:      releaseS3Path,
+		ReleaseCdnURI:      cdnURI,
+		OS:                 os,
+		OSName:             archive.OSName,
+		Arch:               []string{arch},
+		GitTag:             gitTag,
+		ProjectPath:        projectPath,
+		SourcedFromBranch:  sourcedFromBranch,
+		ImageFormat:        archive.Format,
+		Private:            archive.Private,
+		UploadToRTOSBucket: archive.UploadToRTOSBucket,
 	}
 
 	return archiveArtifact, nil
diff --git a/release/cli/pkg/assets/config/bundle_release.go b/release/cli/pkg/assets/config/bundle_release.go
index f933db401f6e..a82c2e17180c 100644
--- a/release/cli/pkg/assets/config/bundle_release.go
+++ b/release/cli/pkg/assets/config/bundle_release.go
@@ -79,6 +79,7 @@ var bundleReleaseAssetsConfigMap = []assettypes.AssetConfig{
 				OSVersion:           "22.04",
 				ArchiveS3PathGetter: archives.RTOSArtifactPathGetter,
 				Private:             true,
+				UploadToRTOSBucket:  true,
 			},
 		},
 	},
diff --git a/release/cli/pkg/assets/types/types.go b/release/cli/pkg/assets/types/types.go
index cf34c6756a9d..3e9783896534 100644
--- a/release/cli/pkg/assets/types/types.go
+++ b/release/cli/pkg/assets/types/types.go
@@ -49,6 +49,7 @@ type Archive struct {
 	ArchitectureOverride string
 	ArchiveS3PathGetter  ArchiveS3PathGenerator
 	Private              bool
+	UploadToRTOSBucket   bool
 }
 
 type AssetConfig struct {
diff --git a/release/cli/pkg/constants/constants.go b/release/cli/pkg/constants/constants.go
index 431e6329c531..dc5b4320ae19 100644
--- a/release/cli/pkg/constants/constants.go
+++ b/release/cli/pkg/constants/constants.go
@@ -52,5 +52,6 @@ const (
 	//	01/02 03:04:05PM '06 -0700
 	//
 	// (January 2, 15:04:05, 2006, in time zone seven hours west of GMT).
-	YYYYMMDD = "2006-01-02"
+	YYYYMMDD                  = "2006-01-02"
+	RTOSArtifactsBucketEnvvar = "RTOS_ARTIFACTS_BUCKET"
 )
diff --git a/release/cli/pkg/operations/upload.go b/release/cli/pkg/operations/upload.go
index b81f7207d4bd..1c6e201bb4f6 100644
--- a/release/cli/pkg/operations/upload.go
+++ b/release/cli/pkg/operations/upload.go
@@ -17,6 +17,7 @@ package operations
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -88,10 +89,14 @@ func UploadArtifacts(ctx context.Context, r *releasetypes.ReleaseConfig, eksaArt
 }
 
 func handleArchiveUpload(_ context.Context, r *releasetypes.ReleaseConfig, artifact releasetypes.Artifact) error {
+	releaseBucket := r.ReleaseBucket
+	if artifact.Archive.UploadToRTOSBucket && r.ReleaseEnvironment == "production" {
+		releaseBucket = os.Getenv(constants.RTOSArtifactsBucketEnvvar)
+	}
 	archiveFile := filepath.Join(artifact.Archive.ArtifactPath, artifact.Archive.ReleaseName)
 	fmt.Printf("Archive - %s\n", archiveFile)
 	key := filepath.Join(artifact.Archive.ReleaseS3Path, artifact.Archive.ReleaseName)
-	err := s3.UploadFile(archiveFile, aws.String(r.ReleaseBucket), aws.String(key), r.ReleaseClients.S3.Uploader, artifact.Archive.Private)
+	err := s3.UploadFile(archiveFile, aws.String(releaseBucket), aws.String(key), r.ReleaseClients.S3.Uploader, artifact.Archive.Private)
 	if err != nil {
 		return fmt.Errorf("uploading archive file [%s] to S3: %v", key, err)
 	}
@@ -108,7 +113,7 @@ func handleArchiveUpload(_ context.Context, r *releasetypes.ReleaseConfig, artif
 		checksumFile := filepath.Join(artifact.Archive.ArtifactPath, artifact.Archive.ReleaseName) + extension
 		fmt.Printf("Checksum - %s\n", checksumFile)
 		key := filepath.Join(artifact.Archive.ReleaseS3Path, artifact.Archive.ReleaseName) + extension
-		err := s3.UploadFile(checksumFile, aws.String(r.ReleaseBucket), aws.String(key), r.ReleaseClients.S3.Uploader, artifact.Archive.Private)
+		err := s3.UploadFile(checksumFile, aws.String(releaseBucket), aws.String(key), r.ReleaseClients.S3.Uploader, artifact.Archive.Private)
 		if err != nil {
 			return fmt.Errorf("uploading checksum file [%s] to S3: %v", key, err)
 		}
diff --git a/release/cli/pkg/types/types.go b/release/cli/pkg/types/types.go
index 245ed7feb1c3..daae917d9fe5 100644
--- a/release/cli/pkg/types/types.go
+++ b/release/cli/pkg/types/types.go
@@ -64,20 +64,21 @@ type ImageTagOverride struct {
 }
 
 type ArchiveArtifact struct {
-	SourceS3Key       string
-	SourceS3Prefix    string
-	ArtifactPath      string
-	ReleaseName       string
-	ReleaseS3Path     string
-	ReleaseCdnURI     string
-	OS                string
-	OSName            string
-	Arch              []string
-	GitTag            string
-	ProjectPath       string
-	SourcedFromBranch string
-	ImageFormat       string
-	Private           bool
+	SourceS3Key        string
+	SourceS3Prefix     string
+	ArtifactPath       string
+	ReleaseName        string
+	ReleaseS3Path      string
+	ReleaseCdnURI      string
+	OS                 string
+	OSName             string
+	Arch               []string
+	GitTag             string
+	ProjectPath        string
+	SourcedFromBranch  string
+	ImageFormat        string
+	Private            bool
+	UploadToRTOSBucket bool
 }
 
 type ImageArtifact struct {

From 7573aac82a9ba78d96b863cfe15b334b6876a39d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 13 Jun 2024 16:14:16 -0700
Subject: [PATCH 22/32] Bump braces from 3.0.2 to 3.0.3 in /docs (#8297)

Bumps [braces](https://github.com/micromatch/braces) from 3.0.2 to 3.0.3.
- [Changelog](https://github.com/micromatch/braces/blob/master/CHANGELOG.md)
- [Commits](https://github.com/micromatch/braces/compare/3.0.2...3.0.3)

---
updated-dependencies:
- dependency-name: braces
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 docs/package-lock.json | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/docs/package-lock.json b/docs/package-lock.json
index 5e9d64186a54..33af14d6884e 100644
--- a/docs/package-lock.json
+++ b/docs/package-lock.json
@@ -130,11 +130,11 @@
       }
     },
     "node_modules/braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
       "dependencies": {
-        "fill-range": "^7.0.1"
+        "fill-range": "^7.1.1"
       },
       "engines": {
         "node": ">=8"
@@ -278,9 +278,9 @@
       }
     },
     "node_modules/fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
       "dependencies": {
         "to-regex-range": "^5.0.1"
       },
@@ -934,11 +934,11 @@
       "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA=="
     },
     "braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
       "requires": {
-        "fill-range": "^7.0.1"
+        "fill-range": "^7.1.1"
       }
     },
     "browserslist": {
@@ -1045,9 +1045,9 @@
       }
     },
     "fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
       "requires": {
         "to-regex-range": "^5.0.1"
       }

From 833270cbbed03e904dcf0c07fcea2d3a020689b7 Mon Sep 17 00:00:00 2001
From: Saurabh Parekh <sjparekh@amazon.com>
Date: Fri, 14 Jun 2024 12:38:17 -0700
Subject: [PATCH 23/32] Add Changelog for v0.19.7 (#8322)

---
 docs/content/en/docs/whatsnew/changelog.md | 30 ++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/docs/content/en/docs/whatsnew/changelog.md b/docs/content/en/docs/whatsnew/changelog.md
index ee9542bd1824..c9c8f7672f1b 100644
--- a/docs/content/en/docs/whatsnew/changelog.md
+++ b/docs/content/en/docs/whatsnew/changelog.md
@@ -31,6 +31,36 @@ description: >
 * When upgrading to a new minor version, a new OS image must be created using the new image-builder CLI pertaining to that release.
 {{% /alert %}}
 
+## [v0.19.7](https://github.com/aws/eks-anywhere/releases/tag/v0.19.7)
+### Supported OS version details
+|                     | vSphere | Bare Metal | Nutanix | CloudStack | Snow |
+|:-------------------:|:-------:|:----------:|:-------:|:----------:|:----:|
+|    Ubuntu 20.04     |    ✔    |     ✔      |    ✔    |     —      |  ✔   |
+|    Ubuntu 22.04     |    ✔    |     ✔      |    ✔    |     —      |  —   |
+| Bottlerocket 1.19.2 |    ✔    |     \*      |    —    |     —      |  —   |
+|      RHEL 8.x       |    ✔    |     ✔      |    ✔    |     ✔      |  —   |
+|      RHEL 9.x       |    —    |     —      |    ✔    |     ✔      |  —   |
+
+\* [EKS Anywhere issue regarding deprecation of Bottlerocket bare metal variants](https://github.com/aws/eks-anywhere/issues/7754)
+
+### Upgraded
+- Cluster API Provider Nutanix: `v1.3.3` to `v1.3.5`
+- Image Builder: `v0.1.24` to `v0.1.26`
+- EKS Distro:
+  - `v1-25-eks-39` to [`v1-25-eks-40`](https://distro.eks.amazonaws.com/releases/1-25/40/)
+  - `v1-26-eks-35` to [`v1-26-eks-38`](https://distro.eks.amazonaws.com/releases/1-26/38/)
+  - `v1-27-eks-29` to [`v1-27-eks-32`](https://distro.eks.amazonaws.com/releases/1-27/32/)
+  - `v1-28-eks-22` to [`v1-28-eks-25`](https://distro.eks.amazonaws.com/releases/1-28/25/)
+  - `v1-29-eks-11` to [`v1-29-eks-14`](https://distro.eks.amazonaws.com/releases/1-29/14/)
+
+### Changed
+- Updated cluster status reconciliation logic for worker node groups with [autoscaling](https://anywhere.eks.amazonaws.com/docs/getting-started/optional/autoscaling/) configuration [#8254](https://github.com/aws/eks-anywhere/pull/8254)
+- Added logic to apply new hardware on baremetal cluster upgrades [#8288](https://github.com/aws/eks-anywhere/pull/8288)
+
+### Fixed
+- Fixed bug when installer does not create CCM secret for Nutanix workload cluster [#8191](https://github.com/aws/eks-anywhere/pull/8191)
+- Fixed upgrade workflow for registry mirror certificates in EKS Anywhere packages [#7114](https://github.com/aws/eks-anywhere/issues/7114)
+
 ## [v0.19.6](https://github.com/aws/eks-anywhere/releases/tag/v0.19.6)
 ### Supported OS version details
 |                     | vSphere | Bare Metal | Nutanix | CloudStack | Snow |

From 26d8b0e4b0e1763995f63aacd59298bfcd794d35 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Fri, 14 Jun 2024 13:48:28 -0700
Subject: [PATCH 24/32] [PR BOT] Generate release testdata files (#8328)

---
 .../testdata/main-bundle-release.yaml         | 40 +++++++++----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/release/cli/pkg/operations/testdata/main-bundle-release.yaml b/release/cli/pkg/operations/testdata/main-bundle-release.yaml
index c7f1b0b58c20..48b78fd97099 100644
--- a/release/cli/pkg/operations/testdata/main-bundle-release.yaml
+++ b/release/cli/pkg/operations/testdata/main-bundle-release.yaml
@@ -306,7 +306,7 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-cli-tools
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-cli-tools:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-cli-tools:v0.19.7-eks-a-v0.0.0-dev-build.1
       clusterController:
         arch:
         - amd64
@@ -315,9 +315,9 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-cluster-controller
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-cluster-controller:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-cluster-controller:v0.19.7-eks-a-v0.0.0-dev-build.1
       components:
-        uri: https://release-bucket/artifacts/v0.0.0-dev-build.0/eks-anywhere/manifests/cluster-controller/v0.19.6/eksa-components.yaml
+        uri: https://release-bucket/artifacts/v0.0.0-dev-build.0/eks-anywhere/manifests/cluster-controller/v0.19.7/eksa-components.yaml
       diagnosticCollector:
         arch:
         - amd64
@@ -326,7 +326,7 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-diagnostic-collector
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-diagnostic-collector:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-diagnostic-collector:v0.19.7-eks-a-v0.0.0-dev-build.1
       version: v0.0.0-dev+build.0+abcdef1
     etcdadmBootstrap:
       components:
@@ -1103,7 +1103,7 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-cli-tools
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-cli-tools:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-cli-tools:v0.19.7-eks-a-v0.0.0-dev-build.1
       clusterController:
         arch:
         - amd64
@@ -1112,9 +1112,9 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-cluster-controller
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-cluster-controller:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-cluster-controller:v0.19.7-eks-a-v0.0.0-dev-build.1
       components:
-        uri: https://release-bucket/artifacts/v0.0.0-dev-build.0/eks-anywhere/manifests/cluster-controller/v0.19.6/eksa-components.yaml
+        uri: https://release-bucket/artifacts/v0.0.0-dev-build.0/eks-anywhere/manifests/cluster-controller/v0.19.7/eksa-components.yaml
       diagnosticCollector:
         arch:
         - amd64
@@ -1123,7 +1123,7 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-diagnostic-collector
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-diagnostic-collector:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-diagnostic-collector:v0.19.7-eks-a-v0.0.0-dev-build.1
       version: v0.0.0-dev+build.0+abcdef1
     etcdadmBootstrap:
       components:
@@ -1900,7 +1900,7 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-cli-tools
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-cli-tools:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-cli-tools:v0.19.7-eks-a-v0.0.0-dev-build.1
       clusterController:
         arch:
         - amd64
@@ -1909,9 +1909,9 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-cluster-controller
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-cluster-controller:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-cluster-controller:v0.19.7-eks-a-v0.0.0-dev-build.1
       components:
-        uri: https://release-bucket/artifacts/v0.0.0-dev-build.0/eks-anywhere/manifests/cluster-controller/v0.19.6/eksa-components.yaml
+        uri: https://release-bucket/artifacts/v0.0.0-dev-build.0/eks-anywhere/manifests/cluster-controller/v0.19.7/eksa-components.yaml
       diagnosticCollector:
         arch:
         - amd64
@@ -1920,7 +1920,7 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-diagnostic-collector
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-diagnostic-collector:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-diagnostic-collector:v0.19.7-eks-a-v0.0.0-dev-build.1
       version: v0.0.0-dev+build.0+abcdef1
     etcdadmBootstrap:
       components:
@@ -2697,7 +2697,7 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-cli-tools
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-cli-tools:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-cli-tools:v0.19.7-eks-a-v0.0.0-dev-build.1
       clusterController:
         arch:
         - amd64
@@ -2706,9 +2706,9 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-cluster-controller
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-cluster-controller:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-cluster-controller:v0.19.7-eks-a-v0.0.0-dev-build.1
       components:
-        uri: https://release-bucket/artifacts/v0.0.0-dev-build.0/eks-anywhere/manifests/cluster-controller/v0.19.6/eksa-components.yaml
+        uri: https://release-bucket/artifacts/v0.0.0-dev-build.0/eks-anywhere/manifests/cluster-controller/v0.19.7/eksa-components.yaml
       diagnosticCollector:
         arch:
         - amd64
@@ -2717,7 +2717,7 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-diagnostic-collector
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-diagnostic-collector:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-diagnostic-collector:v0.19.7-eks-a-v0.0.0-dev-build.1
       version: v0.0.0-dev+build.0+abcdef1
     etcdadmBootstrap:
       components:
@@ -3494,7 +3494,7 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-cli-tools
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-cli-tools:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-cli-tools:v0.19.7-eks-a-v0.0.0-dev-build.1
       clusterController:
         arch:
         - amd64
@@ -3503,9 +3503,9 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-cluster-controller
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-cluster-controller:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-cluster-controller:v0.19.7-eks-a-v0.0.0-dev-build.1
       components:
-        uri: https://release-bucket/artifacts/v0.0.0-dev-build.0/eks-anywhere/manifests/cluster-controller/v0.19.6/eksa-components.yaml
+        uri: https://release-bucket/artifacts/v0.0.0-dev-build.0/eks-anywhere/manifests/cluster-controller/v0.19.7/eksa-components.yaml
       diagnosticCollector:
         arch:
         - amd64
@@ -3514,7 +3514,7 @@ spec:
         imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
         name: eks-anywhere-diagnostic-collector
         os: linux
-        uri: public.ecr.aws/release-container-registry/eks-anywhere-diagnostic-collector:v0.19.6-eks-a-v0.0.0-dev-build.1
+        uri: public.ecr.aws/release-container-registry/eks-anywhere-diagnostic-collector:v0.19.7-eks-a-v0.0.0-dev-build.1
       version: v0.0.0-dev+build.0+abcdef1
     etcdadmBootstrap:
       components:

From d73edc7f485112d1bca3fb3b5ba243591aba9d48 Mon Sep 17 00:00:00 2001
From: Saurabh Parekh <sjparekh@amazon.com>
Date: Fri, 14 Jun 2024 14:05:17 -0700
Subject: [PATCH 25/32] Update brew to v0.19.7 (#8327)

---
 release/triggers/brew-version-release/CLI_RELEASE_VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/triggers/brew-version-release/CLI_RELEASE_VERSION b/release/triggers/brew-version-release/CLI_RELEASE_VERSION
index b3f4adba7b0f..8a77b417780a 100644
--- a/release/triggers/brew-version-release/CLI_RELEASE_VERSION
+++ b/release/triggers/brew-version-release/CLI_RELEASE_VERSION
@@ -1 +1 @@
-v0.19.3
\ No newline at end of file
+v0.19.7

From 3907dabf50b1113901ccb484578d1b0aecbf040f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Jun 2024 16:54:25 -0700
Subject: [PATCH 26/32] Bump codecov/codecov-action from 4.4.1 to 4.5.0 (#8334)

Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4.4.1 to 4.5.0.
- [Release notes](https://github.com/codecov/codecov-action/releases)
- [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/codecov/codecov-action/compare/v4.4.1...v4.5.0)

---
updated-dependencies:
- dependency-name: codecov/codecov-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/go-coverage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/go-coverage.yml b/.github/workflows/go-coverage.yml
index 4dbae47db862..f18a61009f7e 100644
--- a/.github/workflows/go-coverage.yml
+++ b/.github/workflows/go-coverage.yml
@@ -22,7 +22,7 @@ jobs:
       - name: Run go test with coverage
         run: COVER_PROFILE=coverage.txt make coverage-unit-test
       - name: Codecov upload
-        uses: codecov/codecov-action@v4.4.1
+        uses: codecov/codecov-action@v4.5.0
         with:
           files: ./coverage.txt
           token: ${{ secrets.CODECOV_TOKEN }}

From cb5b0aa7940e99ea1037e9ab664a04719376c5c7 Mon Sep 17 00:00:00 2001
From: EKS Distro PR Bot <75336432+eks-distro-pr-bot@users.noreply.github.com>
Date: Mon, 17 Jun 2024 20:44:21 -0700
Subject: [PATCH 27/32] [PR BOT] Generate release testdata files (#8340)

---
 .../pkg/operations/testdata/main-bundle-release.yaml   | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/release/cli/pkg/operations/testdata/main-bundle-release.yaml b/release/cli/pkg/operations/testdata/main-bundle-release.yaml
index 48b78fd97099..f4b3cbeed637 100644
--- a/release/cli/pkg/operations/testdata/main-bundle-release.yaml
+++ b/release/cli/pkg/operations/testdata/main-bundle-release.yaml
@@ -744,7 +744,7 @@ spec:
           description: Helm chart for tinkerbell-chart
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
           name: tinkerbell-chart
-          uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.6-eks-a-v0.0.0-dev-build.1
+          uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.7-eks-a-v0.0.0-dev-build.1
         tinkerbellCrds:
           description: Helm chart for tinkerbell-crds
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
@@ -1541,7 +1541,7 @@ spec:
           description: Helm chart for tinkerbell-chart
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
           name: tinkerbell-chart
-          uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.6-eks-a-v0.0.0-dev-build.1
+          uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.7-eks-a-v0.0.0-dev-build.1
         tinkerbellCrds:
           description: Helm chart for tinkerbell-crds
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
@@ -2338,7 +2338,7 @@ spec:
           description: Helm chart for tinkerbell-chart
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
           name: tinkerbell-chart
-          uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.6-eks-a-v0.0.0-dev-build.1
+          uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.7-eks-a-v0.0.0-dev-build.1
         tinkerbellCrds:
           description: Helm chart for tinkerbell-crds
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
@@ -3135,7 +3135,7 @@ spec:
           description: Helm chart for tinkerbell-chart
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
           name: tinkerbell-chart
-          uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.6-eks-a-v0.0.0-dev-build.1
+          uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.7-eks-a-v0.0.0-dev-build.1
         tinkerbellCrds:
           description: Helm chart for tinkerbell-crds
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
@@ -3932,7 +3932,7 @@ spec:
           description: Helm chart for tinkerbell-chart
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
           name: tinkerbell-chart
-          uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.6-eks-a-v0.0.0-dev-build.1
+          uri: public.ecr.aws/release-container-registry/tinkerbell/tinkerbell-chart:0.2.7-eks-a-v0.0.0-dev-build.1
         tinkerbellCrds:
           description: Helm chart for tinkerbell-crds
           imageDigest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef

From 62bfa7200990a4c96afbfb6e8b0b532856e0419c Mon Sep 17 00:00:00 2001
From: Abhay Krishna <arnchlm@amazon.com>
Date: Tue, 18 Jun 2024 12:43:22 -0700
Subject: [PATCH 28/32] Skip multi-protocol Service test in conformance flow
 (#8349)

---
 test/framework/conformance.go | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/test/framework/conformance.go b/test/framework/conformance.go
index 329a705de507..0f6ef8a37138 100644
--- a/test/framework/conformance.go
+++ b/test/framework/conformance.go
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"strings"
 
+	versionutil "k8s.io/apimachinery/pkg/util/version"
+
 	"github.com/aws/eks-anywhere/internal/pkg/conformance"
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
 	"github.com/aws/eks-anywhere/pkg/manifests"
@@ -12,7 +14,11 @@ import (
 	"github.com/aws/eks-anywhere/pkg/version"
 )
 
-const kubeConformanceImage = "registry.k8s.io/conformance"
+const (
+	kubeConformanceImage                  = "registry.k8s.io/conformance"
+	minKubernetesVersionRequiringTestSkip = "v1.29.0"
+	skippedTestName                       = "Services should serve endpoints on same port and different protocols"
+)
 
 func (e *ClusterE2ETest) RunConformanceTests() {
 	ctx := context.Background()
@@ -28,6 +34,17 @@ func (e *ClusterE2ETest) RunConformanceTests() {
 		e.T.Errorf("Error getting EKS-D release KubeVersion from bundle: %v", err)
 		return
 	}
+	kubeVersionSemver, err := versionutil.ParseSemantic(kubeVersion)
+	if err != nil {
+		e.T.Errorf("Error getting semver for Kubernetes version %s: %v", kubeVersion, err)
+		return
+	}
+	k8s129Compare, err := kubeVersionSemver.Compare(minKubernetesVersionRequiringTestSkip)
+	if err != nil {
+		e.T.Errorf("Error comparing cluster Kubernetes version with %s", minKubernetesVersionRequiringTestSkip)
+		return
+	}
+
 	e.T.Log("Downloading Sonobuoy binary for testing")
 	err = conformance.Download()
 	if err != nil {
@@ -36,6 +53,15 @@ func (e *ClusterE2ETest) RunConformanceTests() {
 	}
 	kubeConformanceImageTagged := fmt.Sprintf("%s:%s", kubeConformanceImage, kubeVersion)
 	args := []string{"--kube-conformance-image", kubeConformanceImageTagged}
+	// If running conformance tests for Kubernetes 1.29 or above, skip this particular test
+	// because it will not pass with our deployment of Cilium.
+	// References:
+	// 1. https://github.com/kubernetes/kubernetes/pull/120069
+	// 2. https://github.com/cilium/cilium/issues/29913
+	// 3. https://github.com/cncf/k8s-conformance/pull/3049
+	if k8s129Compare != -1 {
+		args = append(args, fmt.Sprintf("--e2e-skip='%s'", skippedTestName))
+	}
 	e.T.Logf("Running k8s conformance tests with Image: %s", kubeConformanceImageTagged)
 	output, err := conformance.RunTests(ctx, contextName, args...)
 	if err != nil {

From 2bcca0f8cbe539bf6bc7e0cc17691fefa2a3d777 Mon Sep 17 00:00:00 2001
From: Abhay Krishna <arnchlm@amazon.com>
Date: Thu, 20 Jun 2024 12:02:55 -0700
Subject: [PATCH 29/32] Fetch latest E2E test AMI ID dynamically (#8238)

---
 .../build/script/create_infra_config.sh       | 18 ++++++++++-
 scripts/e2e_test_docker.sh                    | 30 ++++++++++++++-----
 test/e2e/E2E_AMI_FILTER_VARS                  |  3 ++
 3 files changed, 42 insertions(+), 9 deletions(-)
 create mode 100644 test/e2e/E2E_AMI_FILTER_VARS

diff --git a/cmd/integration_test/build/script/create_infra_config.sh b/cmd/integration_test/build/script/create_infra_config.sh
index 438735076e30..f5a01cf6f643 100755
--- a/cmd/integration_test/build/script/create_infra_config.sh
+++ b/cmd/integration_test/build/script/create_infra_config.sh
@@ -17,11 +17,27 @@ set -e
 set -x
 set -o pipefail
 
+REPO_ROOT=$(git rev-parse --show-toplevel)
+source $REPO_ROOT/test/e2e/E2E_AMI_FILTER_VARS
+
+INTEGRATION_TEST_AMI_ID=$(aws ec2 describe-images \
+  --profile ${AWS_PROFILE} \
+  --owners ${AMI_OWNER_ID_FILTER} \
+  --filters "Name=name,Values=${AMI_NAME_FILTER}" "Name=description,Values=${AMI_DESCRIPTION_FILTER}" \
+  --query 'sort_by(Images, &CreationDate)[-1].[ImageId]' \
+  --output text
+)
+
+if [ -z "$INTEGRATION_TEST_AMI_ID" ]; then
+  echo "INTEGRATION_TEST_AMI_ID cannot be empty. Exiting"
+  exit 1
+fi
+
 cat << EOF > ${INTEGRATION_TEST_INFRA_CONFIG}
 ---
 
 ec2:
-  amiId: ${INTEGRATION_TEST_AL2_AMI_ID}
+  amiId: ${INTEGRATION_TEST_AMI_ID}
   subnetId: ${INTEGRATION_TEST_SUBNET_ID}
 
 vSphere:
diff --git a/scripts/e2e_test_docker.sh b/scripts/e2e_test_docker.sh
index 278a15872126..d96c00b3a21f 100755
--- a/scripts/e2e_test_docker.sh
+++ b/scripts/e2e_test_docker.sh
@@ -38,37 +38,51 @@ REPO_ROOT=$(git rev-parse --show-toplevel)
 BIN_FOLDER=$REPO_ROOT/bin
 TEST_REGEX="${1:-TestDockerKubernetes130SimpleFlow}"
 BRANCH_NAME="${2:-main}"
+source $REPO_ROOT/test/e2e/E2E_AMI_FILTER_VARS
 
+export AWS_SDK_LOAD_CONFIG=true
+export AWS_CONFIG_FILE=$(pwd)/config_file
+export AWS_PROFILE=e2e-docker-test
 
-cat << EOF > config_file
+cat << EOF > ${AWS_CONFIG_FILE}
 [default]
 output=json
 region=${AWS_REGION:-${AWS_DEFAULT_REGION:-us-west-2}}
 role_arn=$AWS_ROLE_ARN
 web_identity_token_file=/var/run/secrets/eks.amazonaws.com/serviceaccount/token
-[profile e2e-docker-test]
+[profile ${AWS_PROFILE}]
 role_arn=$TEST_ROLE_ARN
 region=${AWS_REGION:-${AWS_DEFAULT_REGION:-us-west-2}}
 source_profile=default
 EOF
 
+unset AWS_ROLE_ARN AWS_WEB_IDENTITY_TOKEN_FILE
+
 INTEGRATION_TEST_INFRA_CONFIG="/tmp/test-infra.yml"
 export T_TINKERBELL_S3_INVENTORY_CSV_KEY="inventory/den80/den80-hardware.csv"
 
+INTEGRATION_TEST_AMI_ID=$(aws ec2 describe-images \
+  --profile ${AWS_PROFILE} \
+  --owners ${AMI_OWNER_ID_FILTER} \
+  --filters "Name=name,Values=${AMI_NAME_FILTER}" "Name=description,Values=${AMI_DESCRIPTION_FILTER}" \
+  --query 'sort_by(Images, &CreationDate)[-1].[ImageId]' \
+  --output text
+)
+
+if [ -z $INTEGRATION_TEST_AMI_ID ]; then
+  echo "INTEGRATION_TEST_AMI_ID cannot be empty. Exiting"
+  exit 1
+fi
+
 cat << EOF > ${INTEGRATION_TEST_INFRA_CONFIG}
 ---
 
 ec2:
-  amiId: ${INTEGRATION_TEST_AL2_AMI_ID}
+  amiId: ${INTEGRATION_TEST_AMI_ID}
   subnetId:
 
 EOF
 
-export AWS_SDK_LOAD_CONFIG=true
-export AWS_CONFIG_FILE=$(pwd)/config_file
-export AWS_PROFILE=e2e-docker-test
-unset AWS_ROLE_ARN AWS_WEB_IDENTITY_TOKEN_FILE
-
 BUNDLES_OVERRIDE=false
 if [ -f "$BIN_FOLDER/local-bundle-release.yaml" ]; then
     BUNDLES_OVERRIDE=true
diff --git a/test/e2e/E2E_AMI_FILTER_VARS b/test/e2e/E2E_AMI_FILTER_VARS
new file mode 100644
index 000000000000..c78543c92150
--- /dev/null
+++ b/test/e2e/E2E_AMI_FILTER_VARS
@@ -0,0 +1,3 @@
+AMI_OWNER_ID_FILTER=857151390494
+AMI_NAME_FILTER=eksa-integration-test-AL2-*
+AMI_DESCRIPTION_FILTER="*Kernel version 5.X*"

From 654ea0b352d7422b0753af8579e2a7c2150ba1e5 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jacobweinstock@users.noreply.github.com>
Date: Thu, 20 Jun 2024 15:16:55 -0600
Subject: [PATCH 30/32] Update the HookOS customization doc: (#8181)

There is a new and much simpler process now.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 .../baremetal/customize/bare-custom-hookos.md | 191 +++---------------
 1 file changed, 31 insertions(+), 160 deletions(-)

diff --git a/docs/content/en/docs/getting-started/baremetal/customize/bare-custom-hookos.md b/docs/content/en/docs/getting-started/baremetal/customize/bare-custom-hookos.md
index 0e6a44c174dc..bbbe505153da 100644
--- a/docs/content/en/docs/getting-started/baremetal/customize/bare-custom-hookos.md
+++ b/docs/content/en/docs/getting-started/baremetal/customize/bare-custom-hookos.md
@@ -8,191 +8,62 @@ description: >
   Customizing HookOS for EKS Anywhere on Bare Metal
 ---
 
-To initially network boot bare metal machines used in EKS Anywhere clusters, Tinkerbell acquires a kernel and initial ramdisk that is referred to as the HookOS.
+To network boot bare metal machines in EKS Anywhere clusters, machines acquire a kernel and initial ramdisk that is referred to as HookOS.
 A default HookOS is provided when you create an EKS Anywhere cluster.
-However, there may be cases where you want to override the default HookOS, such as to add drivers required to boot your particular type of hardware.
+However, there may be cases where you want and/or need to customize the default HookOS, such as to add drivers required to boot your particular type of hardware.
 
-The following procedure describes how to get the Tinkerbell stack’s Hook/Linuxkit OS built locally.
-For more information on Tinkerbell’s Hook Installation Environment, see the [Tinkerbell Hook repo](https://github.com/tinkerbell/hook).
+The following procedure describes how to customize and build HookOS.
+For more information on Tinkerbell’s HookOS Installation Environment, see the [Tinkerbell Hook repo](https://github.com/tinkerbell/hook).
 
-1. Clone the hook repo or your fork of that repo:
-
-    ```bash
-    git clone https://github.com/tinkerbell/hook.git
-    cd hook/
-    ```
-
-1. Pull down the commit that EKS Anywhere is tracking for Hook:
-
-    ```bash
-    git checkout -b <new-branch> 03a67729d895635fe3c612e4feca3400b9336cc9
-    ```
-
-    >**_NOTE_**: This commit number can be obtained from the [EKS-A build tooling repo](https://github.com/aws/eks-anywhere-build-tooling/blob/main/projects/tinkerbell/hook/GIT_TAG).
-    >
-
-1. Make changes shown in the following `diff` in the `Makefile` located in the root of the repo using your favorite editor. 
-
-    ```bash
-    diff --git a/Makefile b/Makefile
-    index e7fd844..8e87c78 100644
-    --- a/Makefile
-    +++ b/Makefile
-    @@ -2,7 +2,7 @@
-     ### !!NOTE!!
-     # If this is changed then a fresh output dir is required (`git clean -fxd` or just `rm -rf out`)
-     # Handling this better shows some of make's suckiness compared to newer build tools (redo, tup ...) where the command lines to tools invoked isn't tracked by make
-    -ORG := quay.io/tinkerbell
-    +ORG := localhost:5000/tinkerbell
-     # makes sure there's no trailing / so we can just add them in the recipes which looks nicer
-     ORG := $(shell echo "${ORG}" | sed 's|/*$$||')
-
-     ```
-
-    Changes above change the ORG variable to use a local registry (`localhost:5000`) 
-
-1. Make changes shown in the following `diff` in the `rules.mk` located in the root of the repo using your favorite editor.
-
-    ```bash
-    diff --git a/rules.mk b/rules.mk
-    index b2c5133..64e32b1 100644
-    --- a/rules.mk
-    +++ b/rules.mk
-    @@ -22,7 +22,7 @@ ifeq ($(ARCH),aarch64)
-     ARCH = arm64
-     endif
- 
-    -arches := amd64 arm64
-    +arches := amd64
-     modes := rel dbg
- 
-     hook-bootkit-deps := $(wildcard hook-bootkit/*)
-    @@ -87,13 +87,12 @@ push-hook-bootkit push-hook-docker:
-            docker buildx build --platform $$platforms --push -t $(ORG)/$(container):$T $(container)
- 
-     .PHONY: dist
-    -dist: out/$T/rel/amd64/hook.tar out/$T/rel/arm64/hook.tar ## Build tarballs for distribution
-    +dist: out/$T/rel/amd64/hook.tar ## Build tarballs for distribution
-     dbg-dist: out/$T/dbg/$(ARCH)/hook.tar ## Build debug enabled tarball
-     dist dbg-dist:
-            for f in $^; do
-            case $$f in
-            *amd64*) arch=x86_64 ;;
-     -      *arm64*) arch=aarch64 ;;
-            *) echo unknown arch && exit 1;;
-            esac
-            d=$$(dirname $$(dirname $$f))
-
-    ```
-
-    Above changes are for the `docker build` command to only build for the immediately required platform (amd64 in this case) to save time.
+## System requirements
 
+- `>= 2G memory`
+- `>= 4 CPU cores` # the more cores the better for kernel building.
+- `>= 20G disk space`
 
-1. Modify the `hook.yaml` file located in the root of the repo with the following changes:
+## Dependencies
 
-    ```bash
-    diff --git a/hook.yaml b/hook.yaml
-    
-     index 0c5d789..b51b35e 100644
-    
-     net: host
-    --- a/hook.yaml
-    +++ b/hook.yaml
-    @@ -1,5 +1,5 @@
-     kernel:
-    - image: quay.io/tinkerbell/hook-kernel:5.10.85 (http://quay.io/tinkerbell/hook-kernel:5.10.85)
-    + image: localhost:5000/tinkerbell/hook-kernel:5.10.85
-     cmdline: "console=tty0 console=ttyS0 console=ttyAMA0 console=ttysclp0"
-     init:
-     - linuxkit/init:v0.8
-    @@ -42,7 +42,7 @@ services:
-     binds:
-     - /var/run:/var/run
-     - name: docker
-    - image: quay.io/tinkerbell/hook-docker:0.0 (http://quay.io/tinkerbell/hook-docker:0.0)
-    + image: localhost:5000/tinkerbell/hook-docker:0.0
-     capabilities:
-     - all
-     net: host
-    @@ -64,7 +64,7 @@ services:
-     - /var/run/docker
-     - /var/run/worker
-     - name: bootkit
-    - image: quay.io/tinkerbell/hook-bootkit:0.0 (http://quay.io/tinkerbell/hook-bootkit:0.0)
-    + image: localhost:5000/tinkerbell/hook-bootkit:0.0
-     capabilities:
-     - all
-    ```
-
-    The changes above are for using local registry (localhost:5000) for hook-docker, hook-bootkit, and hook-kernel. 
+Be sure to install all the following dependencies.
 
-    >**_NOTE_**: You may also need to modify the `hook.yaml` file if you want to add or change components that are used to build up the image. So far, for example, we have needed to change versions of `init` and `getty` and inject SSH keys. Take a look at the [LinuxKit Examples](https://github.com/linuxkit/linuxkit/tree/master/examples) site for examples.
-    >
+- `jq`
+- `envsubst`
+- `pigz`
+- `docker`
+- `curl`
+- `bash` >= 4.4
+- `git`
+- `findutils`
 
-1. Make any planned custom modifications to the files under `hook`, if you are only making changes to `bootkit` or `tink-docker`.
-    
-    
-1. If you are modifying the kernel, such as to change kernel config parameters to add or modify drivers, follow these steps:
-
-    * Change into kernel directory and make a local image for amd64 architecture:
+1. Clone the Hook repo or your fork of that repo:
 
     ```bash
-    cd kernel; make kconfig_amd64
-    ```
-
-    * Run the image
-
-    ```bash
-    docker run --rm -ti -v $(pwd):/src:z quay.io/tinkerbell/kconfig
-    ```
-
-    * You can now navigate to the source code and run the UI for configuring the kernel:
-
-    ```bash
-    cd linux-5-10
-    make menuconfig
-    ```
-
-    * Once you have changed the necessary kernel configuration parameters, copy the new configuration:
-
-    ```bash
-    cp .config /src/config-5.10.x-x86_64
+    git clone https://github.com/tinkerbell/hook.git
+    cd hook/
     ```
 
-    Exit out of container into the repo’s kernel directory and run make:
+1. Run the Linux kernel [menuconfig](https://en.wikipedia.org/wiki/Menuconfig) TUI and configuring the kernel as needed. Be sure to save the config before exiting. The result of this step will be a modified kernel configuration file (`./kernel/configs/generic-6.6.y-x86_64`).
 
     ```bash
-    /linux-5.10.85 # exit
-    user1 % make
+    ./build.sh kernel-config hook-latest-lts-amd64
     ```
 
-1. Install Linuxkit based on instructions from the [LinuxKit](https://github.com/linuxkit/linuxkit) page.
-    
-    
-1. Ensure that the `linuxkit` tool is in your PATH:
+1. Build the kernel container image. The result of this step will be a container image. Use `docker images quay.io/tinkerbell/hook-kernel` to see it.
 
     ```bash
-    export PATH=$PATH:/home/tink/linuxkit/bin
+    ./build.sh kernel hook-latest-lts-amd64
     ```
 
-1. Start a local registry:
+1. Build the HookOS kernel and initramfs artifacts. The result of this step will be the kernel and initramfs. These files are located at `./out/hook/vmlinuz-latest-lts-x86_64` and `./out/hook/initramfs-latest-lts-x86_64` respectively.
 
     ```bash
-    docker run -d -p 5000:5000 --name registry registry:2
+    ./build.sh linuxkit hook-latest-lts-amd64 
     ```
 
-1. Compile by running the following in the root of the repo:
-
-    ```bash
-    make dist  
-    ```
-1. Artifacts will be put under the `dist` directory in the repo’s root:
+1. Rename the kernel and initramfs files to `vmlinuz-x86_64` and `initramfs-x86_64` respectively.
 
     ```bash
-    ./initramfs-aarch64
-    ./initramfs-x86_64
-    ./vmlinuz-aarch64
-    ./vmlinuz-x86_64
+    mv ./out/hook/vmlinuz-latest-lts-x86_64 ./out/hook/vmlinuz-x86_64
+    mv ./out/hook/initramfs-latest-lts-x86_64 ./out/hook/initramfs-x86_64
     ```
 
-1. To use the kernel (`vmlinuz`) and initial ram disk (`initramfs`) when you build your cluster, see the description of the `hookImagesURLPath` field in your Bare Metal configuration file.
+1. To use the kernel (`vmlinuz-x86_64`) and initial ram disk (`initramfs-x86_64`) when you build your EKS Anywhere cluster, see the description of the [`hookImagesURLPath`]({{< relref "../bare-spec#hookimagesurlpath-optional" >}}) field in your cluster configuration file.

From 64180bc2356e20c76a505e3084602e60f36f24de Mon Sep 17 00:00:00 2001
From: ahreehong <46465244+ahreehong@users.noreply.github.com>
Date: Mon, 24 Jun 2024 09:17:02 -0700
Subject: [PATCH 31/32] Add comments about why cloudstack tests are skipped
 (#8364)

---
 test/e2e/SKIPPED_TESTS.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/e2e/SKIPPED_TESTS.yaml b/test/e2e/SKIPPED_TESTS.yaml
index b43bc6812410..09d7bc3f9dfc 100644
--- a/test/e2e/SKIPPED_TESTS.yaml
+++ b/test/e2e/SKIPPED_TESTS.yaml
@@ -1,11 +1,12 @@
 skipped_tests:
 
 # CloudStack
-#Airgapped tests
+# Airgapped tests due to an airgapped network not being setup properly on the cloudstack CI env.
 - TestCloudStackKubernetes126RedhatAirgappedRegistryMirror
 - TestCloudStackKubernetes128RedhatAirgappedProxy
 
-# Proxy tests
+# Proxy API tests skipped due to being unable to run in cloudstack CI env. We should probably revisit these and see if we can unskip them now
+# as we have a proxy env set up in the cloudstack CI env now.
 - TestCloudStackKubernetes126RedhatProxyConfigAPI
 - TestCloudStackKubernetes127RedhatProxyConfigAPI
 - TestCloudStackKubernetes128RedhatProxyConfigAPI

From 6e93d6938d1a99f4c5eabdcb27eba66bf22c1ee6 Mon Sep 17 00:00:00 2001
From: Mitali Paygude <mitalipaygude@gmail.com>
Date: Mon, 24 Jun 2024 22:11:35 -0700
Subject: [PATCH 32/32] Kubelet config e2e tests for vsphere Bottlerocket

---
 test/e2e/vsphere_test.go | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/test/e2e/vsphere_test.go b/test/e2e/vsphere_test.go
index 8eb3b0399792..af2799129e0b 100644
--- a/test/e2e/vsphere_test.go
+++ b/test/e2e/vsphere_test.go
@@ -5995,7 +5995,7 @@ func TestVSphereKubernetes129to130UbuntuEtcdScaleDown(t *testing.T) {
 }
 
 // Kubelet Configuration e2e tests
-func TestVSphereKubernetes129KubeletConfigurationSimpleFlow(t *testing.T) {
+func TestVSphereKubernetes129UbuntuKubeletConfiguration(t *testing.T) {
 	test := framework.NewClusterE2ETest(
 		t,
 		framework.NewVSphere(t, framework.WithUbuntu129()),
@@ -6005,7 +6005,7 @@ func TestVSphereKubernetes129KubeletConfigurationSimpleFlow(t *testing.T) {
 	runKubeletConfigurationFlow(test)
 }
 
-func TestVSphereKubernetes130KubeletConfigurationSimpleFlow(t *testing.T) {
+func TestVSphereKubernetes130UbuntuKubeletConfiguration(t *testing.T) {
 	test := framework.NewClusterE2ETest(
 		t,
 		framework.NewVSphere(t, framework.WithUbuntu130()),
@@ -6014,3 +6014,23 @@ func TestVSphereKubernetes130KubeletConfigurationSimpleFlow(t *testing.T) {
 	)
 	runKubeletConfigurationFlow(test)
 }
+
+func TestVSphereKubernetes129BottlerocketKubeletConfiguration(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewVSphere(t, framework.WithBottleRocket129()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}
+
+func TestVSphereKubernetes130BottlerocketKubeletConfiguration(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewVSphere(t, framework.WithBottleRocket130()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube130)),
+		framework.WithKubeletConfig(),
+	)
+	runKubeletConfigurationFlow(test)
+}