Implement adopted cluster e2e tests

test/e2e/clusterdeployment/aws/aws.go

@@ -17,7 +17,11 @@
 package aws
 
 import (
+	"bufio"
 	"context"
+	"fmt"
+	"gopkg.in/yaml.v3"
+	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -52,22 +56,37 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient,
 	Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster subnets")
 	Expect(found).To(BeTrue(), "AWS cluster has no subnets")
 
-	subnet, ok := subnets[0].(map[string]any)
-	Expect(ok).To(BeTrue(), "failed to cast subnet to map")
-
-	subnetID, ok := subnet["resourceID"].(string)
-	Expect(ok).To(BeTrue(), "failed to cast subnet ID to string")
-
-	subnetAZ, ok := subnet["availabilityZone"].(string)
-	Expect(ok).To(BeTrue(), "failed to cast subnet availability zone to string")
+	type awsSubnetMaps []map[string]any
+	subnetMaps := make(awsSubnetMaps, len(subnets))
+	for i, s := range subnets {
+		subnet, ok := s.(map[string]any)
+		Expect(ok).To(BeTrue(), "failed to cast subnet to map")
+		subnetMaps[i] = map[string]any{
+			"isPublic":         subnet["isPublic"],
+			"availabilityZone": subnet["availabilityZone"],
+			"id":               subnet["resourceID"],
+			"routeTableId":     subnet["routeTableId"],
+			"zoneType":         "availability-zone",
+		}
 
+		if natGatewayID, exists := subnet["natGatewayId"]; exists && natGatewayID != "" {
+			subnetMaps[i]["natGatewayId"] = natGatewayID
+		}
+	}
+	var subnetsFormatted string
+	encodedYaml, err := yaml.Marshal(subnetMaps)
+	Expect(err).NotTo(HaveOccurred(), "failed to get marshall subnet maps")
+	scanner := bufio.NewScanner(strings.NewReader(string(encodedYaml)))
+	for scanner.Scan() {
+		subnetsFormatted += fmt.Sprintf("    %s\n", scanner.Text())
+	}
+	GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnets, subnetsFormatted)
 	securityGroupID, found, err := unstructured.NestedString(
 		awsCluster.Object, "status", "networkStatus", "securityGroups", "node", "id")
 	Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster security group ID")
 	Expect(found).To(BeTrue(), "AWS cluster has no security group ID")
 
 	GinkgoT().Setenv(clusterdeployment.EnvVarAWSVPCID, vpcID)
-	GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnetID, subnetID)
-	GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnetAvailabilityZone, subnetAZ)
 	GinkgoT().Setenv(clusterdeployment.EnvVarAWSSecurityGroupID, securityGroupID)
+	GinkgoT().Setenv(clusterdeployment.EnvVarManagementClusterName, clusterName)
 }

test/e2e/clusterdeployment/clusterdeployment.go

@@ -15,6 +15,7 @@
 package clusterdeployment
 
 import (
+	"context"
 	_ "embed"
 	"fmt"
 	"os"
@@ -27,6 +28,7 @@ import (
 	"gopkg.in/yaml.v3"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
+	"github.com/K0rdent/kcm/test/e2e/kubeclient"
 	"github.com/K0rdent/kcm/test/utils"
 )
 
@@ -37,8 +39,8 @@ const (
 	ProviderAWS     ProviderType = "infrastructure-aws"
 	ProviderAzure   ProviderType = "infrastructure-azure"
 	ProviderVSphere ProviderType = "infrastructure-vsphere"
-
-	providerLabel = "cluster.x-k8s.io/provider"
+	ProviderAdopted ProviderType = "infrastructure-internal"
+	providerLabel                = "cluster.x-k8s.io/provider"
 )
 
 type Template string
@@ -50,6 +52,7 @@ const (
 	TemplateAzureStandaloneCP   Template = "azure-standalone-cp"
 	TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp"
 	TemplateVSphereHostedCP     Template = "vsphere-hosted-cp"
+	TemplateAdoptedCluster      Template = "adopted-cluster"
 )
 
 //go:embed resources/aws-standalone-cp.yaml.tpl
@@ -70,6 +73,9 @@ var vsphereStandaloneCPClusterDeploymentTemplateBytes []byte
 //go:embed resources/vsphere-hosted-cp.yaml.tpl
 var vsphereHostedCPClusterDeploymentTemplateBytes []byte
 
+//go:embed resources/adopted-cluster.yaml.tpl
+var adoptedClusterDeploymentTemplateBytes []byte
+
 func FilterAllProviders() []string {
 	return []string{
 		utils.HMCControllerLabel,
@@ -121,8 +127,7 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured {
 		// since we populate the vars from standalone prior to this step.
 		ValidateDeploymentVars([]string{
 			EnvVarAWSVPCID,
-			EnvVarAWSSubnetID,
-			EnvVarAWSSubnetAvailabilityZone,
+			EnvVarAWSSubnets,
 			EnvVarAWSSecurityGroupID,
 		})
 		clusterDeploymentTemplateBytes = awsHostedCPClusterDeploymentTemplateBytes
@@ -134,6 +139,8 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured {
 		clusterDeploymentTemplateBytes = azureHostedCPClusterDeploymentTemplateBytes
 	case TemplateAzureStandaloneCP:
 		clusterDeploymentTemplateBytes = azureStandaloneCPClusterDeploymentTemplateBytes
+	case TemplateAdoptedCluster:
+		clusterDeploymentTemplateBytes = adoptedClusterDeploymentTemplateBytes
 	default:
 		Fail(fmt.Sprintf("Unsupported template: %s", templateName))
 	}
@@ -156,3 +163,27 @@ func ValidateDeploymentVars(v []string) {
 		Expect(os.Getenv(envVar)).NotTo(BeEmpty(), envVar+" must be set")
 	}
 }
+
+func ValidateClusterTemplates(ctx context.Context, client *kubeclient.KubeClient) error {
+	templates, err := client.ListClusterTemplates(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to list cluster templates: %w", err)
+	}
+
+	for _, template := range templates {
+		valid, found, err := unstructured.NestedBool(template.Object, "status", "valid")
+		if err != nil {
+			return fmt.Errorf("failed to get valid flag for template %s: %w", template.GetName(), err)
+		}
+
+		if !found {
+			return fmt.Errorf("valid flag for template %s not found", template.GetName())
+		}
+
+		if !valid {
+			return fmt.Errorf("template %s is still invalid", template.GetName())
+		}
+	}
+
+	return nil
+}

test/e2e/clusterdeployment/clusteridentity/clusteridentity.go

@@ -40,6 +40,7 @@ type ClusterIdentity struct {
 	SecretData           map[string]string
 	Spec                 map[string]any
 	Namespaced           bool
+	CredentialName       string
 }
 
 // New creates a ClusterIdentity resource, credential and associated secret for
@@ -59,15 +60,30 @@ func New(kc *kubeclient.KubeClient, provider clusterdeployment.ProviderType) *Cl
 
 	secretName := fmt.Sprintf("%s-cluster-identity-secret", provider)
 	identityName := fmt.Sprintf("%s-cluster-identity", provider)
+	group := "infrastructure.cluster.x-k8s.io"
 
 	switch provider {
+	case clusterdeployment.ProviderAdopted:
+		kubeCfgBytes, err := os.ReadFile(os.Getenv(clusterdeployment.EnvVarAdoptedKubeconfigPath))
+		Expect(err).NotTo(HaveOccurred())
+
+		kind = "Secret"
+		version = "v1"
+		group = ""
+		identityName = secretName
+
+		secretStringData = map[string]string{
+			"Value": string(kubeCfgBytes),
+		}
+
 	case clusterdeployment.ProviderAWS:
 		resource = "awsclusterstaticidentities"
 		kind = "AWSClusterStaticIdentity"
 		version = "v1beta2"
 		secretStringData = map[string]string{
 			"AccessKeyID":     os.Getenv(clusterdeployment.EnvVarAWSAccessKeyID),
 			"SecretAccessKey": os.Getenv(clusterdeployment.EnvVarAWSSecretAccessKey),
+			"SessionToken":    os.Getenv("AWS_SESSION_TOKEN"),
 		}
 		spec = map[string]any{
 			"secretRef": secretName,
@@ -117,22 +133,26 @@ func New(kc *kubeclient.KubeClient, provider clusterdeployment.ProviderType) *Cl
 
 	ci := ClusterIdentity{
 		GroupVersionResource: schema.GroupVersionResource{
-			Group:    "infrastructure.cluster.x-k8s.io",
+			Group:    group,
 			Version:  version,
 			Resource: resource,
 		},
-		Kind:         kind,
-		SecretName:   secretName,
-		IdentityName: identityName,
-		SecretData:   secretStringData,
-		Spec:         spec,
-		Namespaced:   namespaced,
+		Kind:           kind,
+		SecretName:     secretName,
+		IdentityName:   identityName,
+		SecretData:     secretStringData,
+		Spec:           spec,
+		Namespaced:     namespaced,
+		CredentialName: fmt.Sprintf("%s-cred", identityName),
 	}
 
 	validateSecretDataPopulated(secretStringData)
-	ci.waitForResourceCRD(kc)
 	ci.createSecret(kc)
-	ci.createClusterIdentity(kc)
+
+	if provider != clusterdeployment.ProviderAdopted {
+		ci.waitForResourceCRD(kc)
+		ci.createClusterIdentity(kc)
+	}
 	ci.createCredential(kc)
 
 	return &ci
@@ -203,20 +223,19 @@ func (ci *ClusterIdentity) createSecret(kc *kubeclient.KubeClient) {
 func (ci *ClusterIdentity) createCredential(kc *kubeclient.KubeClient) {
 	GinkgoHelper()
 
-	credName := fmt.Sprintf("%s-cred", ci.IdentityName)
-	By(fmt.Sprintf("creating Credential: %s", credName))
+	By(fmt.Sprintf("creating Credential: %s", ci.CredentialName))
 
 	cred := &unstructured.Unstructured{
 		Object: map[string]any{
 			"apiVersion": "hmc.mirantis.com/v1alpha1",
 			"kind":       "Credential",
 			"metadata": map[string]any{
-				"name":      credName,
+				"name":      ci.CredentialName,
 				"namespace": kc.Namespace,
 			},
 			"spec": map[string]any{
 				"identityRef": map[string]any{
-					"apiVersion": ci.GroupVersionResource.Group + "/" + ci.GroupVersionResource.Version,
+					"apiVersion": ci.GroupVersionResource.GroupVersion().String(),
 					"kind":       ci.Kind,
 					"name":       ci.IdentityName,
 					"namespace":  kc.Namespace,
@@ -252,3 +271,28 @@ func (ci *ClusterIdentity) createClusterIdentity(kc *kubeclient.KubeClient) {
 
 	kc.CreateOrUpdateUnstructuredObject(ci.GroupVersionResource, id, ci.Namespaced)
 }
+
+func (ci *ClusterIdentity) WaitForValidCredential(kc *kubeclient.KubeClient) {
+	GinkgoHelper()
+
+	By(fmt.Sprintf("waiting for %s credential to be ready", ci.CredentialName))
+
+	ctx := context.Background()
+
+	Eventually(func() error {
+		cred, err := kc.GetCredential(ctx, ci.CredentialName)
+		if err != nil {
+			return fmt.Errorf("failed to get credntial: %w", err)
+		}
+
+		ready, found, err := unstructured.NestedBool(cred.Object, "status", "ready")
+		if !found {
+			return fmt.Errorf("failed to get ready status: %w", err)
+		}
+		if !ready {
+			_, _ = fmt.Fprintf(GinkgoWriter, "credential is not ready, retrying...\n")
+			return fmt.Errorf("credential is not ready: %s", ci.GroupVersionResource.String())
+		}
+		return nil
+	}).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(Succeed())
+}

test/e2e/clusterdeployment/constants.go

@@ -22,18 +22,17 @@ const (
 	EnvVarNamespace             = "NAMESPACE"
 	// EnvVarNoCleanup disables After* cleanup in provider specs to allow for
 	// debugging of test failures.
-	EnvVarNoCleanup = "NO_CLEANUP"
-
+	EnvVarNoCleanup             = "NO_CLEANUP"
+	EnvVarManagementClusterName = "MANAGEMENT_CLUSTER_NAME"
 	// AWS
-	EnvVarAWSAccessKeyID            = "AWS_ACCESS_KEY_ID"
-	EnvVarAWSSecretAccessKey        = "AWS_SECRET_ACCESS_KEY"
-	EnvVarAWSVPCID                  = "AWS_VPC_ID"
-	EnvVarAWSSubnetID               = "AWS_SUBNET_ID"
-	EnvVarAWSSubnetAvailabilityZone = "AWS_SUBNET_AVAILABILITY_ZONE"
-	EnvVarAWSInstanceType           = "AWS_INSTANCE_TYPE"
-	EnvVarAWSSecurityGroupID        = "AWS_SG_ID"
-	EnvVarAWSClusterIdentity        = "AWS_CLUSTER_IDENTITY"
-	EnvVarPublicIP                  = "AWS_PUBLIC_IP"
+	EnvVarAWSAccessKeyID     = "AWS_ACCESS_KEY_ID"
+	EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY"
+	EnvVarAWSVPCID           = "AWS_VPC_ID"
+	EnvVarAWSSubnets         = "AWS_SUBNETS"
+	EnvVarAWSInstanceType    = "AWS_INSTANCE_TYPE"
+	EnvVarAWSSecurityGroupID = "AWS_SG_ID"
+	EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY"
+	EnvVarPublicIP           = "AWS_PUBLIC_IP"
 
 	// VSphere
 	EnvVarVSphereUser            = "VSPHERE_USER"
@@ -47,4 +46,8 @@ const (
 	EnvVarAzureSubscription    = "AZURE_SUBSCRIPTION"
 	EnvVarAzureClusterIdentity = "AZURE_CLUSTER_IDENTITY"
 	EnvVarAzureRegion          = "AZURE_REGION"
+
+	// Adopted
+	EnvVarAdoptedKubeconfigPath = "KUBECONFIG_DATA_PATH"
+	EnvVarAdoptedCredential     = "ADOPTED_CREDENTIAL"
 )

test/e2e/clusterdeployment/providervalidator.go

@@ -65,15 +65,20 @@ func NewProviderValidator(template Template, clusterName string, action Validati
 		case TemplateAWSStandaloneCP, TemplateAWSHostedCP:
 			resourcesToValidate["ccm"] = validateCCM
 			resourceOrder = append(resourceOrder, "ccm")
-		case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP:
+		case TemplateAzureStandaloneCP, TemplateAzureHostedCP, TemplateVSphereStandaloneCP:
 			delete(resourcesToValidate, "csi-driver")
+		case TemplateAdoptedCluster:
+			resourcesToValidate = map[string]resourceValidationFunc{
+				"sveltoscluster": validateSveltosCluster,
+			}
 		}
 	} else {
 		resourcesToValidate = map[string]resourceValidationFunc{
 			"clusters":           validateClusterDeleted,
 			"machinedeployments": validateMachineDeploymentsDeleted,
 			"control-planes":     validateK0sControlPlanesDeleted,
 		}
+
 		resourceOrder = []string{"clusters", "machinedeployments", "control-planes"}
 	}
 

test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl

@@ -0,0 +1,16 @@
+apiVersion: hmc.mirantis.com/v1alpha1
+kind: ClusterDeployment
+metadata:
+  name: ${CLUSTER_DEPLOYMENT_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  template: adopted-cluster-0-0-1
+  credential: ${ADOPTED_CREDENTIAL}
+  config: {}
+  services:
+    - template: kyverno-3-2-6
+      name: kyverno
+      namespace: kyverno
+    - template: ingress-nginx-4-11-0
+      name: ingress-nginx
+      namespace: ingress-nginx

test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl

@@ -12,8 +12,11 @@ spec:
     vpcID: ${AWS_VPC_ID}
     region: ${AWS_REGION}
     subnets:
-      - id: ${AWS_SUBNET_ID}
-        availabilityZone: ${AWS_SUBNET_AVAILABILITY_ZONE}
+${AWS_SUBNETS}
     instanceType: ${AWS_INSTANCE_TYPE:=t3.medium}
     securityGroupIDs:
       - ${AWS_SG_ID}
+    managementClusterName: ${MANAGEMENT_CLUSTER_NAME}
+    controlPlane:
+       rootVolumeSize: 30
+    rootVolumeSize: 30

test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl

@@ -15,5 +15,7 @@ spec:
     workersNumber: ${WORKERS_NUMBER:=1}
     controlPlane:
       instanceType: ${AWS_INSTANCE_TYPE:=t3.small}
+      rootVolumeSize: 30
     worker:
       instanceType: ${AWS_INSTANCE_TYPE:=t3.small}
+      rootVolumeSize: 30