diff --git a/config/dev/adopted-clusterdeployment.yaml b/config/dev/adopted-clusterdeployment.yaml index a1b1c911..1bc2a251 100644 --- a/config/dev/adopted-clusterdeployment.yaml +++ b/config/dev/adopted-clusterdeployment.yaml @@ -7,7 +7,7 @@ spec: template: adopted-cluster-0-0-2 credential: adopted-cluster-cred config: {} - services: + serviceSpec: - template: kyverno-3-2-6 name: kyverno namespace: kyverno diff --git a/test/e2e/clusterdeployment/aws/aws.go b/test/e2e/clusterdeployment/aws/aws.go index 61a2a0a2..79dd7649 100644 --- a/test/e2e/clusterdeployment/aws/aws.go +++ b/test/e2e/clusterdeployment/aws/aws.go @@ -17,10 +17,14 @@ package aws import ( + "bufio" "context" + "fmt" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -52,22 +56,37 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster subnets") Expect(found).To(BeTrue(), "AWS cluster has no subnets") - subnet, ok := subnets[0].(map[string]any) - Expect(ok).To(BeTrue(), "failed to cast subnet to map") - - subnetID, ok := subnet["resourceID"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet ID to string") - - subnetAZ, ok := subnet["availabilityZone"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet availability zone to string") + type awsSubnetMaps []map[string]any + subnetMaps := make(awsSubnetMaps, len(subnets)) + for i, s := range subnets { + subnet, ok := s.(map[string]any) + Expect(ok).To(BeTrue(), "failed to cast subnet to map") + subnetMaps[i] = map[string]any{ + "isPublic": subnet["isPublic"], + "availabilityZone": subnet["availabilityZone"], + "id": subnet["resourceID"], + "routeTableId": subnet["routeTableId"], + "zoneType": "availability-zone", + } + if natGatewayID, exists := subnet["natGatewayId"]; exists && natGatewayID != "" { + subnetMaps[i]["natGatewayId"] = natGatewayID + } + } + var subnetsFormatted string + encodedYaml, err := yaml.Marshal(subnetMaps) + Expect(err).NotTo(HaveOccurred(), "failed to get marshall subnet maps") + scanner := bufio.NewScanner(strings.NewReader(string(encodedYaml))) + for scanner.Scan() { + subnetsFormatted += fmt.Sprintf(" %s\n", scanner.Text()) + } + GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnets, subnetsFormatted) securityGroupID, found, err := unstructured.NestedString( awsCluster.Object, "status", "networkStatus", "securityGroups", "node", "id") Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster security group ID") Expect(found).To(BeTrue(), "AWS cluster has no security group ID") GinkgoT().Setenv(clusterdeployment.EnvVarAWSVPCID, vpcID) - GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnetID, subnetID) - GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnetAvailabilityZone, subnetAZ) GinkgoT().Setenv(clusterdeployment.EnvVarAWSSecurityGroupID, securityGroupID) + GinkgoT().Setenv(clusterdeployment.EnvVarManagementClusterName, clusterName) } diff --git a/test/e2e/clusterdeployment/clusterdeployment.go b/test/e2e/clusterdeployment/clusterdeployment.go index bd563fe8..17bb10e6 100644 --- a/test/e2e/clusterdeployment/clusterdeployment.go +++ b/test/e2e/clusterdeployment/clusterdeployment.go @@ -15,6 +15,7 @@ package clusterdeployment import ( + "context" _ "embed" "fmt" "os" @@ -27,6 +28,7 @@ import ( "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "github.com/K0rdent/kcm/test/e2e/kubeclient" "github.com/K0rdent/kcm/test/utils" ) @@ -37,8 +39,8 @@ const ( ProviderAWS ProviderType = "infrastructure-aws" ProviderAzure ProviderType = "infrastructure-azure" ProviderVSphere ProviderType = "infrastructure-vsphere" - - providerLabel = "cluster.x-k8s.io/provider" + ProviderAdopted ProviderType = "infrastructure-internal" + providerLabel = "cluster.x-k8s.io/provider" ) type Template string @@ -50,6 +52,7 @@ const ( TemplateAzureStandaloneCP Template = "azure-standalone-cp" TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp" TemplateVSphereHostedCP Template = "vsphere-hosted-cp" + TemplateAdoptedCluster Template = "adopted-cluster" ) //go:embed resources/aws-standalone-cp.yaml.tpl @@ -70,6 +73,9 @@ var vsphereStandaloneCPClusterDeploymentTemplateBytes []byte //go:embed resources/vsphere-hosted-cp.yaml.tpl var vsphereHostedCPClusterDeploymentTemplateBytes []byte +//go:embed resources/adopted-cluster.yaml.tpl +var adoptedClusterDeploymentTemplateBytes []byte + func FilterAllProviders() []string { return []string{ utils.KCMControllerLabel, @@ -121,8 +127,7 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { // since we populate the vars from standalone prior to this step. ValidateDeploymentVars([]string{ EnvVarAWSVPCID, - EnvVarAWSSubnetID, - EnvVarAWSSubnetAvailabilityZone, + EnvVarAWSSubnets, EnvVarAWSSecurityGroupID, }) clusterDeploymentTemplateBytes = awsHostedCPClusterDeploymentTemplateBytes @@ -134,6 +139,8 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { clusterDeploymentTemplateBytes = azureHostedCPClusterDeploymentTemplateBytes case TemplateAzureStandaloneCP: clusterDeploymentTemplateBytes = azureStandaloneCPClusterDeploymentTemplateBytes + case TemplateAdoptedCluster: + clusterDeploymentTemplateBytes = adoptedClusterDeploymentTemplateBytes default: Fail(fmt.Sprintf("Unsupported template: %s", templateName)) } @@ -156,3 +163,27 @@ func ValidateDeploymentVars(v []string) { Expect(os.Getenv(envVar)).NotTo(BeEmpty(), envVar+" must be set") } } + +func ValidateClusterTemplates(ctx context.Context, client *kubeclient.KubeClient) error { + templates, err := client.ListClusterTemplates(ctx) + if err != nil { + return fmt.Errorf("failed to list cluster templates: %w", err) + } + + for _, template := range templates { + valid, found, err := unstructured.NestedBool(template.Object, "status", "valid") + if err != nil { + return fmt.Errorf("failed to get valid flag for template %s: %w", template.GetName(), err) + } + + if !found { + return fmt.Errorf("valid flag for template %s not found", template.GetName()) + } + + if !valid { + return fmt.Errorf("template %s is still invalid", template.GetName()) + } + } + + return nil +} diff --git a/test/e2e/clusterdeployment/clusteridentity/clusteridentity.go b/test/e2e/clusterdeployment/clusteridentity/clusteridentity.go index 4e9d3b7b..a9b15eca 100644 --- a/test/e2e/clusterdeployment/clusteridentity/clusteridentity.go +++ b/test/e2e/clusterdeployment/clusteridentity/clusteridentity.go @@ -40,6 +40,7 @@ type ClusterIdentity struct { SecretData map[string]string Spec map[string]any Namespaced bool + CredentialName string } // New creates a ClusterIdentity resource, credential and associated secret for @@ -59,8 +60,22 @@ func New(kc *kubeclient.KubeClient, provider clusterdeployment.ProviderType) *Cl secretName := fmt.Sprintf("%s-cluster-identity-secret", provider) identityName := fmt.Sprintf("%s-cluster-identity", provider) + group := "infrastructure.cluster.x-k8s.io" switch provider { + case clusterdeployment.ProviderAdopted: + kubeCfgBytes, err := os.ReadFile(os.Getenv(clusterdeployment.EnvVarAdoptedKubeconfigPath)) + Expect(err).NotTo(HaveOccurred()) + + kind = "Secret" + version = "v1" + group = "" + identityName = secretName + + secretStringData = map[string]string{ + "Value": string(kubeCfgBytes), + } + case clusterdeployment.ProviderAWS: resource = "awsclusterstaticidentities" kind = "AWSClusterStaticIdentity" @@ -68,6 +83,7 @@ func New(kc *kubeclient.KubeClient, provider clusterdeployment.ProviderType) *Cl secretStringData = map[string]string{ "AccessKeyID": os.Getenv(clusterdeployment.EnvVarAWSAccessKeyID), "SecretAccessKey": os.Getenv(clusterdeployment.EnvVarAWSSecretAccessKey), + "SessionToken": os.Getenv("AWS_SESSION_TOKEN"), } spec = map[string]any{ "secretRef": secretName, @@ -117,22 +133,26 @@ func New(kc *kubeclient.KubeClient, provider clusterdeployment.ProviderType) *Cl ci := ClusterIdentity{ GroupVersionResource: schema.GroupVersionResource{ - Group: "infrastructure.cluster.x-k8s.io", + Group: group, Version: version, Resource: resource, }, - Kind: kind, - SecretName: secretName, - IdentityName: identityName, - SecretData: secretStringData, - Spec: spec, - Namespaced: namespaced, + Kind: kind, + SecretName: secretName, + IdentityName: identityName, + SecretData: secretStringData, + Spec: spec, + Namespaced: namespaced, + CredentialName: fmt.Sprintf("%s-cred", identityName), } validateSecretDataPopulated(secretStringData) - ci.waitForResourceCRD(kc) ci.createSecret(kc) - ci.createClusterIdentity(kc) + + if provider != clusterdeployment.ProviderAdopted { + ci.waitForResourceCRD(kc) + ci.createClusterIdentity(kc) + } ci.createCredential(kc) return &ci @@ -203,20 +223,19 @@ func (ci *ClusterIdentity) createSecret(kc *kubeclient.KubeClient) { func (ci *ClusterIdentity) createCredential(kc *kubeclient.KubeClient) { GinkgoHelper() - credName := fmt.Sprintf("%s-cred", ci.IdentityName) - By(fmt.Sprintf("creating Credential: %s", credName)) + By(fmt.Sprintf("creating Credential: %s", ci.CredentialName)) cred := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "k0rdent.mirantis.com/v1alpha1", "kind": "Credential", "metadata": map[string]any{ - "name": credName, + "name": ci.CredentialName, "namespace": kc.Namespace, }, "spec": map[string]any{ "identityRef": map[string]any{ - "apiVersion": ci.GroupVersionResource.Group + "/" + ci.GroupVersionResource.Version, + "apiVersion": ci.GroupVersionResource.GroupVersion().String(), "kind": ci.Kind, "name": ci.IdentityName, "namespace": kc.Namespace, @@ -252,3 +271,28 @@ func (ci *ClusterIdentity) createClusterIdentity(kc *kubeclient.KubeClient) { kc.CreateOrUpdateUnstructuredObject(ci.GroupVersionResource, id, ci.Namespaced) } + +func (ci *ClusterIdentity) WaitForValidCredential(kc *kubeclient.KubeClient) { + GinkgoHelper() + + By(fmt.Sprintf("waiting for %s credential to be ready", ci.CredentialName)) + + ctx := context.Background() + + Eventually(func() error { + cred, err := kc.GetCredential(ctx, ci.CredentialName) + if err != nil { + return fmt.Errorf("failed to get credntial: %w", err) + } + + ready, found, err := unstructured.NestedBool(cred.Object, "status", "ready") + if !found { + return fmt.Errorf("failed to get ready status: %w", err) + } + if !ready { + _, _ = fmt.Fprintf(GinkgoWriter, "credential is not ready, retrying...\n") + return fmt.Errorf("credential is not ready: %s", ci.GroupVersionResource.String()) + } + return nil + }).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(Succeed()) +} diff --git a/test/e2e/clusterdeployment/constants.go b/test/e2e/clusterdeployment/constants.go index f5a1e203..17c19630 100644 --- a/test/e2e/clusterdeployment/constants.go +++ b/test/e2e/clusterdeployment/constants.go @@ -22,18 +22,17 @@ const ( EnvVarNamespace = "NAMESPACE" // EnvVarNoCleanup disables After* cleanup in provider specs to allow for // debugging of test failures. - EnvVarNoCleanup = "NO_CLEANUP" - + EnvVarNoCleanup = "NO_CLEANUP" + EnvVarManagementClusterName = "MANAGEMENT_CLUSTER_NAME" // AWS - EnvVarAWSAccessKeyID = "AWS_ACCESS_KEY_ID" - EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY" - EnvVarAWSVPCID = "AWS_VPC_ID" - EnvVarAWSSubnetID = "AWS_SUBNET_ID" - EnvVarAWSSubnetAvailabilityZone = "AWS_SUBNET_AVAILABILITY_ZONE" - EnvVarAWSInstanceType = "AWS_INSTANCE_TYPE" - EnvVarAWSSecurityGroupID = "AWS_SG_ID" - EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY" - EnvVarPublicIP = "AWS_PUBLIC_IP" + EnvVarAWSAccessKeyID = "AWS_ACCESS_KEY_ID" + EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + EnvVarAWSVPCID = "AWS_VPC_ID" + EnvVarAWSSubnets = "AWS_SUBNETS" + EnvVarAWSInstanceType = "AWS_INSTANCE_TYPE" + EnvVarAWSSecurityGroupID = "AWS_SG_ID" + EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY" + EnvVarPublicIP = "AWS_PUBLIC_IP" // VSphere EnvVarVSphereUser = "VSPHERE_USER" @@ -47,4 +46,8 @@ const ( EnvVarAzureSubscription = "AZURE_SUBSCRIPTION" EnvVarAzureClusterIdentity = "AZURE_CLUSTER_IDENTITY" EnvVarAzureRegion = "AZURE_REGION" + + // Adopted + EnvVarAdoptedKubeconfigPath = "KUBECONFIG_DATA_PATH" + EnvVarAdoptedCredential = "ADOPTED_CREDENTIAL" ) diff --git a/test/e2e/clusterdeployment/providervalidator.go b/test/e2e/clusterdeployment/providervalidator.go index 9d4ee121..0e45f465 100644 --- a/test/e2e/clusterdeployment/providervalidator.go +++ b/test/e2e/clusterdeployment/providervalidator.go @@ -65,8 +65,12 @@ func NewProviderValidator(template Template, clusterName string, action Validati case TemplateAWSStandaloneCP, TemplateAWSHostedCP: resourcesToValidate["ccm"] = validateCCM resourceOrder = append(resourceOrder, "ccm") - case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP: + case TemplateAzureStandaloneCP, TemplateAzureHostedCP, TemplateVSphereStandaloneCP: delete(resourcesToValidate, "csi-driver") + case TemplateAdoptedCluster: + resourcesToValidate = map[string]resourceValidationFunc{ + "sveltoscluster": validateSveltosCluster, + } } } else { resourcesToValidate = map[string]resourceValidationFunc{ @@ -74,6 +78,7 @@ func NewProviderValidator(template Template, clusterName string, action Validati "machinedeployments": validateMachineDeploymentsDeleted, "control-planes": validateK0sControlPlanesDeleted, } + resourceOrder = []string{"clusters", "machinedeployments", "control-planes"} } diff --git a/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl b/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl new file mode 100644 index 00000000..812aa4c3 --- /dev/null +++ b/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl @@ -0,0 +1,16 @@ +apiVersion: hmc.mirantis.com/v1alpha1 +kind: ClusterDeployment +metadata: + name: ${CLUSTER_DEPLOYMENT_NAME} + namespace: ${NAMESPACE} +spec: + template: adopted-cluster-0-0-2 + credential: ${ADOPTED_CREDENTIAL} + config: {} + serviceSpec: + - template: kyverno-3-2-6 + name: kyverno + namespace: kyverno + - template: ingress-nginx-4-11-0 + name: ingress-nginx + namespace: ingress-nginx \ No newline at end of file diff --git a/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl b/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl index 0e527a03..8ba130a3 100644 --- a/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl +++ b/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl @@ -12,8 +12,11 @@ spec: vpcID: ${AWS_VPC_ID} region: ${AWS_REGION} subnets: - - id: ${AWS_SUBNET_ID} - availabilityZone: ${AWS_SUBNET_AVAILABILITY_ZONE} +${AWS_SUBNETS} instanceType: ${AWS_INSTANCE_TYPE:=t3.medium} securityGroupIDs: - ${AWS_SG_ID} + managementClusterName: ${MANAGEMENT_CLUSTER_NAME} + controlPlane: + rootVolumeSize: 30 + rootVolumeSize: 30 \ No newline at end of file diff --git a/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl b/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl index 39eee3f7..b2c10edb 100644 --- a/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl +++ b/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl @@ -15,5 +15,7 @@ spec: workersNumber: ${WORKERS_NUMBER:=1} controlPlane: instanceType: ${AWS_INSTANCE_TYPE:=t3.small} + rootVolumeSize: 30 worker: instanceType: ${AWS_INSTANCE_TYPE:=t3.small} + rootVolumeSize: 30 \ No newline at end of file diff --git a/test/e2e/clusterdeployment/validate_deployed.go b/test/e2e/clusterdeployment/validate_deployed.go index 1dbcd788..5b76b94e 100644 --- a/test/e2e/clusterdeployment/validate_deployed.go +++ b/test/e2e/clusterdeployment/validate_deployed.go @@ -273,3 +273,22 @@ func validateCCM(ctx context.Context, kc *kubeclient.KubeClient, clusterName str return fmt.Errorf("%s Service does not yet have an external hostname", service.Name) } + +// validateSveltosCluster validates that the sveltos cluster is ready +func validateSveltosCluster(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { + sveltosCluster, err := kc.GetSveltosCluster(ctx, clusterName) + if err != nil { + return fmt.Errorf("error getting sveltos cluster: %v", err) + } + + ready, found, err := unstructured.NestedBool(sveltosCluster.Object, "status", "ready") + if err != nil { + return fmt.Errorf("error checking sveltos cluster ready: %v", err) + } + + if !found || !ready { + return fmt.Errorf("sveltos cluster %s is not ready", clusterName) + } + + return nil +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index f7d73a6e..4dc188e5 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -47,7 +47,6 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { GinkgoT().Setenv(clusterdeployment.EnvVarNamespace, internalutils.DefaultSystemNamespace) - By("building and deploying the controller-manager") cmd := exec.Command("make", "kind-deploy") _, err := utils.Run(cmd) @@ -66,6 +65,15 @@ var _ = BeforeSuite(func() { } return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + Eventually(func() error { + err = clusterdeployment.ValidateClusterTemplates(context.Background(), kc) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) }) var _ = AfterSuite(func() { @@ -176,6 +184,7 @@ func collectLogArtifacts(kc *kubeclient.KubeClient, clusterName string, provider } } + filterLabels = append(filterLabels, "app=source-controller") for _, label := range filterLabels { pods, _ := kc.Client.CoreV1().Pods(kc.Namespace).List(context.Background(), metav1.ListOptions{ LabelSelector: label, diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go index a13f403c..62c04aa0 100644 --- a/test/e2e/kubeclient/kubeclient.go +++ b/test/e2e/kubeclient/kubeclient.go @@ -33,6 +33,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "github.com/K0rdent/kcm/api/v1alpha1" "github.com/K0rdent/kcm/internal/utils/status" ) @@ -56,7 +57,7 @@ func NewFromLocal(namespace string) *KubeClient { // the kubeconfig from secret it needs an existing kubeclient. func (kc *KubeClient) NewFromCluster(ctx context.Context, namespace, clusterName string) *KubeClient { GinkgoHelper() - return newKubeClient(kc.getKubeconfigSecretData(ctx, clusterName), namespace) + return newKubeClient(kc.GetKubeconfigSecretData(ctx, clusterName), namespace) } // WriteKubeconfig writes the kubeconfig for the given clusterName to the @@ -65,7 +66,7 @@ func (kc *KubeClient) NewFromCluster(ctx context.Context, namespace, clusterName func (kc *KubeClient) WriteKubeconfig(ctx context.Context, clusterName string) (string, func() error) { GinkgoHelper() - secretData := kc.getKubeconfigSecretData(ctx, clusterName) + secretData := kc.GetKubeconfigSecretData(ctx, clusterName) dir, err := os.Getwd() Expect(err).NotTo(HaveOccurred()) @@ -89,7 +90,7 @@ func (kc *KubeClient) WriteKubeconfig(ctx context.Context, clusterName string) ( return path, deleteFunc } -func (kc *KubeClient) getKubeconfigSecretData(ctx context.Context, clusterName string) []byte { +func (kc *KubeClient) GetKubeconfigSecretData(ctx context.Context, clusterName string) []byte { GinkgoHelper() secret, err := kc.Client.CoreV1().Secrets(kc.Namespace).Get(ctx, clusterName+"-kubeconfig", metav1.GetOptions{}) @@ -279,3 +280,47 @@ func (kc *KubeClient) ListK0sControlPlanes( Resource: "k0scontrolplanes", }, clusterName) } + +func (kc *KubeClient) ListClusterTemplates(ctx context.Context) ([]unstructured.Unstructured, error) { + client := kc.GetDynamicClient(schema.GroupVersionResource{ + Group: v1alpha1.GroupVersion.Group, + Version: v1alpha1.GroupVersion.Version, + Resource: "clustertemplates", + }, true) + + resources, err := client.List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list cluster templates") + } + + return resources.Items, nil +} + +func (kc *KubeClient) GetCredential(ctx context.Context, name string) (*unstructured.Unstructured, error) { + client := kc.GetDynamicClient(schema.GroupVersionResource{ + Group: v1alpha1.GroupVersion.Group, + Version: v1alpha1.GroupVersion.Version, + Resource: "credentials", + }, true) + credential, err := client.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get credential %s: %w", name, err) + } + + return credential, nil +} + +func (kc *KubeClient) GetSveltosCluster(ctx context.Context, name string) (*unstructured.Unstructured, error) { + client := kc.GetDynamicClient(schema.GroupVersionResource{ + Group: "lib.projectsveltos.io", + Version: "v1beta1", + Resource: "sveltosclusters", + }, true) + + sveltosCluster, err := client.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get sveltos cluster %s: %w", name, err) + } + + return sveltosCluster, nil +} diff --git a/test/e2e/provider_adopted_test.go b/test/e2e/provider_adopted_test.go new file mode 100644 index 00000000..3ad47eb1 --- /dev/null +++ b/test/e2e/provider_adopted_test.go @@ -0,0 +1,137 @@ +// Copyright 2024 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "os" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + internalutils "github.com/K0rdent/kcm/internal/utils" + "github.com/K0rdent/kcm/test/e2e/clusterdeployment" + "github.com/K0rdent/kcm/test/e2e/clusterdeployment/clusteridentity" + "github.com/K0rdent/kcm/test/e2e/kubeclient" +) + +var _ = Describe("Adopted Cluster Templates", Label("provider:cloud", "provider:adopted"), Ordered, func() { + var ( + kc *kubeclient.KubeClient + standaloneClient *kubeclient.KubeClient + clusterDeleteFunc func() error + adoptedDeleteFunc func() error + kubecfgDeleteFunc func() error + clusterName string + ) + + BeforeAll(func() { + By("providing cluster identity") + kc = kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace) + ci := clusteridentity.New(kc, clusterdeployment.ProviderAWS) + Expect(os.Setenv(clusterdeployment.EnvVarAWSClusterIdentity, ci.IdentityName)).Should(Succeed()) + ci.WaitForValidCredential(kc) + }) + + AfterAll(func() { + // If we failed collect logs from each of the affiliated controllers + // as well as the output of clusterctl to store as artifacts. + if CurrentSpecReport().Failed() && !noCleanup() { + if standaloneClient != nil { + By("collecting failure logs from hosted controllers") + collectLogArtifacts(standaloneClient, clusterName, clusterdeployment.ProviderAWS, clusterdeployment.ProviderCAPI) + } + } + + By("deleting resources") + for _, deleteFunc := range []func() error{ + kubecfgDeleteFunc, + adoptedDeleteFunc, + clusterDeleteFunc, + } { + if deleteFunc != nil { + err := deleteFunc() + Expect(err).NotTo(HaveOccurred()) + } + } + }) + + It("should work with an Adopted cluster provider", func() { + // Deploy a standalone cluster and verify it is running/ready. Then, delete the management cluster and + // recreate it. Next "adopt" the cluster we created and verify the services were deployed. Next we delete + // the adopted cluster and finally the management cluster (AWS standalone). + GinkgoT().Setenv(clusterdeployment.EnvVarAWSInstanceType, "t3.xlarge") + + templateBy(clusterdeployment.TemplateAWSStandaloneCP, "creating a ClusterDeployment") + sd := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAWSStandaloneCP) + clusterName = sd.GetName() + + clusterDeleteFunc = kc.CreateClusterDeployment(context.Background(), sd) + + templateBy(clusterdeployment.TemplateAWSStandaloneCP, "waiting for infrastructure to deploy successfully") + deploymentValidator := clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAWSStandaloneCP, + clusterName, + clusterdeployment.ValidationActionDeploy, + ) + + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), kc) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + // create the adopted cluster using the AWS standalone cluster + var kubeCfgFile string + kubeCfgFile, kubecfgDeleteFunc = kc.WriteKubeconfig(context.Background(), clusterName) + GinkgoT().Setenv(clusterdeployment.EnvVarAdoptedKubeconfigPath, kubeCfgFile) + ci := clusteridentity.New(kc, clusterdeployment.ProviderAdopted) + Expect(os.Setenv(clusterdeployment.EnvVarAdoptedCredential, ci.CredentialName)).Should(Succeed()) + + ci.WaitForValidCredential(kc) + + adoptedCluster := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAdoptedCluster) + adoptedClusterName := adoptedCluster.GetName() + adoptedDeleteFunc = kc.CreateClusterDeployment(context.Background(), adoptedCluster) + + // validate the adopted cluster + deploymentValidator = clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAdoptedCluster, + adoptedClusterName, + clusterdeployment.ValidationActionDeploy, + ) + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), kc) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + // delete the adopted cluster + err := adoptedDeleteFunc() + Expect(err).NotTo(HaveOccurred()) + adoptedDeleteFunc = nil + + // finally delete the aws standalone cluster and verify it's deleted correctly + err = clusterDeleteFunc() + Expect(err).NotTo(HaveOccurred()) + clusterDeleteFunc = nil + + deletionValidator := clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAWSStandaloneCP, + clusterName, + clusterdeployment.ValidationActionDelete, + ) + Eventually(func() error { + return deletionValidator.Validate(context.Background(), kc) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + }) +}) diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 5f11a106..589d6ac1 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -46,6 +46,7 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order By("providing cluster identity") kc = kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace) ci := clusteridentity.New(kc, clusterdeployment.ProviderAWS) + ci.WaitForValidCredential(kc) Expect(os.Setenv(clusterdeployment.EnvVarAWSClusterIdentity, ci.IdentityName)).Should(Succeed()) }) @@ -124,8 +125,18 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + Eventually(func() error { + err = clusterdeployment.ValidateClusterTemplates(context.Background(), standaloneClient) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + // Ensure AWS credentials are set in the standalone cluster. - clusteridentity.New(standaloneClient, clusterdeployment.ProviderAWS) + standaloneCi := clusteridentity.New(standaloneClient, clusterdeployment.ProviderAWS) + standaloneCi.WaitForValidCredential(standaloneClient) // Populate the environment variables required for the hosted // cluster. @@ -156,6 +167,7 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order templateBy(clusterdeployment.TemplateAWSHostedCP, "deleting the clusterdeployment") err = hostedDeleteFunc() Expect(err).NotTo(HaveOccurred()) + hostedDeleteFunc = nil deletionValidator := clusterdeployment.NewProviderValidator( clusterdeployment.TemplateAWSHostedCP, @@ -168,22 +180,20 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order // Now delete the standalone clusterdeployment and verify it is // removed, it is deleted last since it is the basis for the hosted // cluster. - /* - FIXME(#339): This is currently disabled as the deletion of the - standalone cluster is failing due to outstanding issues. - templateBy(clusterdeployment.TemplateAWSStandaloneCP, "deleting the clusterdeployment") - err = standaloneDeleteFunc() - Expect(err).NotTo(HaveOccurred()) - - deletionValidator = clusterdeployment.NewProviderValidator( - clusterdeployment.TemplateAWSStandaloneCP, - clusterName, - clusterdeployment.ValidationActionDelete, - ) - Eventually(func() error { - return deletionValidator.Validate(context.Background(), kc) - }).WithTimeout(10 * time.Minute).WithPolling(10 * - time.Second).Should(Succeed()) - */ + + templateBy(clusterdeployment.TemplateAWSStandaloneCP, "deleting the clusterdeployment") + err = standaloneDeleteFunc() + Expect(err).NotTo(HaveOccurred()) + + standaloneDeleteFunc = nil + deletionValidator = clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAWSStandaloneCP, + clusterName, + clusterdeployment.ValidationActionDelete, + ) + Eventually(func() error { + return deletionValidator.Validate(context.Background(), kc) + }).WithTimeout(10 * time.Minute).WithPolling(10 * + time.Second).Should(Succeed()) }) }) diff --git a/test/e2e/provider_azure_test.go b/test/e2e/provider_azure_test.go index 401aba31..a4f39ac2 100644 --- a/test/e2e/provider_azure_test.go +++ b/test/e2e/provider_azure_test.go @@ -47,6 +47,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or By("ensuring Azure credentials are set") kc = kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace) ci := clusteridentity.New(kc, clusterdeployment.ProviderAzure) + ci.WaitForValidCredential(kc) Expect(os.Setenv(clusterdeployment.EnvVarAzureClusterIdentity, ci.IdentityName)).Should(Succeed()) }) @@ -82,7 +83,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or sd := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAzureStandaloneCP) sdName = sd.GetName() - standaloneDeleteFunc := kc.CreateClusterDeployment(context.Background(), sd) + standaloneDeleteFunc = kc.CreateClusterDeployment(context.Background(), sd) // verify the standalone cluster is deployed correctly deploymentValidator := clusterdeployment.NewProviderValidator( @@ -123,8 +124,18 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + Eventually(func() error { + err = clusterdeployment.ValidateClusterTemplates(context.Background(), standaloneClient) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + By("Create azure credential secret") - clusteridentity.New(standaloneClient, clusterdeployment.ProviderAzure) + standaloneCi := clusteridentity.New(standaloneClient, clusterdeployment.ProviderAzure) + standaloneCi.WaitForValidCredential(standaloneClient) By("Create default storage class for azure-disk CSI driver") azure.CreateDefaultStorageClass(standaloneClient) @@ -149,10 +160,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or By("verify the deployment deletes successfully") err = hostedDeleteFunc() Expect(err).NotTo(HaveOccurred()) - - err = standaloneDeleteFunc() - Expect(err).NotTo(HaveOccurred()) - + hostedDeleteFunc = nil deploymentValidator = clusterdeployment.NewProviderValidator( clusterdeployment.TemplateAzureHostedCP, hdName, @@ -163,6 +171,10 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or return deploymentValidator.Validate(context.Background(), standaloneClient) }).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + err = standaloneDeleteFunc() + Expect(err).NotTo(HaveOccurred()) + standaloneDeleteFunc = nil + deploymentValidator = clusterdeployment.NewProviderValidator( clusterdeployment.TemplateAzureStandaloneCP, hdName, diff --git a/test/e2e/provider_vsphere_test.go b/test/e2e/provider_vsphere_test.go index a4f54b3a..d1b635f8 100644 --- a/test/e2e/provider_vsphere_test.go +++ b/test/e2e/provider_vsphere_test.go @@ -44,6 +44,7 @@ var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere" kc = kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace) By("providing cluster identity") ci := clusteridentity.New(kc, clusterdeployment.ProviderVSphere) + ci.WaitForValidCredential(kc) By("setting VSPHERE_CLUSTER_IDENTITY env variable") Expect(os.Setenv(clusterdeployment.EnvVarVSphereClusterIdentity, ci.IdentityName)).Should(Succeed()) })