From d380b26972899113f1c6da271270da5067ab251d Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Tue, 31 Dec 2024 14:25:45 -0800 Subject: [PATCH 1/2] Implement adopted cluster e2e tests --- .../clusterdeployment/clusterdeployment.go | 36 ++++- .../clusteridentity/clusteridentity.go | 70 +++++++-- test/e2e/clusterdeployment/constants.go | 4 + .../clusterdeployment/providervalidator.go | 6 + .../resources/adopted-cluster.yaml.tpl | 16 +++ .../clusterdeployment/validate_deployed.go | 19 +++ test/e2e/e2e_suite_test.go | 10 +- test/e2e/kubeclient/kubeclient.go | 51 ++++++- test/e2e/provider_adopted_test.go | 133 ++++++++++++++++++ test/e2e/provider_aws_test.go | 4 +- test/e2e/provider_azure_test.go | 4 +- test/e2e/provider_vsphere_test.go | 1 + 12 files changed, 333 insertions(+), 21 deletions(-) create mode 100644 test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl create mode 100644 test/e2e/provider_adopted_test.go diff --git a/test/e2e/clusterdeployment/clusterdeployment.go b/test/e2e/clusterdeployment/clusterdeployment.go index 75d151728..f25ef91f3 100644 --- a/test/e2e/clusterdeployment/clusterdeployment.go +++ b/test/e2e/clusterdeployment/clusterdeployment.go @@ -15,6 +15,7 @@ package clusterdeployment import ( + "context" _ "embed" "fmt" "os" @@ -27,6 +28,7 @@ import ( "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "github.com/K0rdent/kcm/test/e2e/kubeclient" "github.com/K0rdent/kcm/test/utils" ) @@ -37,8 +39,8 @@ const ( ProviderAWS ProviderType = "infrastructure-aws" ProviderAzure ProviderType = "infrastructure-azure" ProviderVSphere ProviderType = "infrastructure-vsphere" - - providerLabel = "cluster.x-k8s.io/provider" + ProviderAdopted ProviderType = "infrastructure-internal" + providerLabel = "cluster.x-k8s.io/provider" ) type Template string @@ -50,6 +52,7 @@ const ( TemplateAzureStandaloneCP Template = "azure-standalone-cp" TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp" TemplateVSphereHostedCP Template = "vsphere-hosted-cp" + TemplateAdoptedCluster Template = "adopted-cluster" ) //go:embed resources/aws-standalone-cp.yaml.tpl @@ -70,6 +73,9 @@ var vsphereStandaloneCPClusterDeploymentTemplateBytes []byte //go:embed resources/vsphere-hosted-cp.yaml.tpl var vsphereHostedCPClusterDeploymentTemplateBytes []byte +//go:embed resources/adopted-cluster.yaml.tpl +var adoptedClusterDeploymentTemplateBytes []byte + func FilterAllProviders() []string { return []string{ utils.HMCControllerLabel, @@ -134,6 +140,8 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { clusterDeploymentTemplateBytes = azureHostedCPClusterDeploymentTemplateBytes case TemplateAzureStandaloneCP: clusterDeploymentTemplateBytes = azureStandaloneCPClusterDeploymentTemplateBytes + case TemplateAdoptedCluster: + clusterDeploymentTemplateBytes = adoptedClusterDeploymentTemplateBytes default: Fail(fmt.Sprintf("Unsupported template: %s", templateName)) } @@ -156,3 +164,27 @@ func ValidateDeploymentVars(v []string) { Expect(os.Getenv(envVar)).NotTo(BeEmpty(), envVar+" must be set") } } + +func ValidateClusterTemplates(ctx context.Context, client *kubeclient.KubeClient) error { + templates, err := client.ListClusterTemplates(ctx) + if err != nil { + return fmt.Errorf("failed to list cluster templates: %w", err) + } + + for _, template := range templates { + valid, found, err := unstructured.NestedBool(template.Object, "status", "valid") + if err != nil { + return fmt.Errorf("failed to get valid flag for template %s: %w", template.GetName(), err) + } + + if !found { + return fmt.Errorf("valid flag for template %s not found", template.GetName()) + } + + if !valid { + return fmt.Errorf("template %s is still invalid", template.GetName()) + } + } + + return nil +} diff --git a/test/e2e/clusterdeployment/clusteridentity/clusteridentity.go b/test/e2e/clusterdeployment/clusteridentity/clusteridentity.go index 08068b15b..1f1fe4aea 100644 --- a/test/e2e/clusterdeployment/clusteridentity/clusteridentity.go +++ b/test/e2e/clusterdeployment/clusteridentity/clusteridentity.go @@ -40,6 +40,7 @@ type ClusterIdentity struct { SecretData map[string]string Spec map[string]any Namespaced bool + CredentialName string } // New creates a ClusterIdentity resource, credential and associated secret for @@ -59,8 +60,22 @@ func New(kc *kubeclient.KubeClient, provider clusterdeployment.ProviderType) *Cl secretName := fmt.Sprintf("%s-cluster-identity-secret", provider) identityName := fmt.Sprintf("%s-cluster-identity", provider) + group := "infrastructure.cluster.x-k8s.io" switch provider { + case clusterdeployment.ProviderAdopted: + kubeCfgBytes, err := os.ReadFile(os.Getenv(clusterdeployment.EnvVarAdoptedKubeconfigPath)) + Expect(err).NotTo(HaveOccurred()) + + kind = "Secret" + version = "v1" + group = "" + identityName = secretName + + secretStringData = map[string]string{ + "Value": string(kubeCfgBytes), + } + case clusterdeployment.ProviderAWS: resource = "awsclusterstaticidentities" kind = "AWSClusterStaticIdentity" @@ -68,6 +83,7 @@ func New(kc *kubeclient.KubeClient, provider clusterdeployment.ProviderType) *Cl secretStringData = map[string]string{ "AccessKeyID": os.Getenv(clusterdeployment.EnvVarAWSAccessKeyID), "SecretAccessKey": os.Getenv(clusterdeployment.EnvVarAWSSecretAccessKey), + "SessionToken": os.Getenv("AWS_SESSION_TOKEN"), } spec = map[string]any{ "secretRef": secretName, @@ -117,22 +133,26 @@ func New(kc *kubeclient.KubeClient, provider clusterdeployment.ProviderType) *Cl ci := ClusterIdentity{ GroupVersionResource: schema.GroupVersionResource{ - Group: "infrastructure.cluster.x-k8s.io", + Group: group, Version: version, Resource: resource, }, - Kind: kind, - SecretName: secretName, - IdentityName: identityName, - SecretData: secretStringData, - Spec: spec, - Namespaced: namespaced, + Kind: kind, + SecretName: secretName, + IdentityName: identityName, + SecretData: secretStringData, + Spec: spec, + Namespaced: namespaced, + CredentialName: fmt.Sprintf("%s-cred", identityName), } validateSecretDataPopulated(secretStringData) - ci.waitForResourceCRD(kc) ci.createSecret(kc) - ci.createClusterIdentity(kc) + + if provider != clusterdeployment.ProviderAdopted { + ci.waitForResourceCRD(kc) + ci.createClusterIdentity(kc) + } ci.createCredential(kc) return &ci @@ -203,20 +223,19 @@ func (ci *ClusterIdentity) createSecret(kc *kubeclient.KubeClient) { func (ci *ClusterIdentity) createCredential(kc *kubeclient.KubeClient) { GinkgoHelper() - credName := fmt.Sprintf("%s-cred", ci.IdentityName) - By(fmt.Sprintf("creating Credential: %s", credName)) + By(fmt.Sprintf("creating Credential: %s", ci.CredentialName)) cred := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "hmc.mirantis.com/v1alpha1", "kind": "Credential", "metadata": map[string]any{ - "name": credName, + "name": ci.CredentialName, "namespace": kc.Namespace, }, "spec": map[string]any{ "identityRef": map[string]any{ - "apiVersion": ci.GroupVersionResource.Group + "/" + ci.GroupVersionResource.Version, + "apiVersion": ci.GroupVersionResource.GroupVersion().String(), "kind": ci.Kind, "name": ci.IdentityName, "namespace": kc.Namespace, @@ -252,3 +271,28 @@ func (ci *ClusterIdentity) createClusterIdentity(kc *kubeclient.KubeClient) { kc.CreateOrUpdateUnstructuredObject(ci.GroupVersionResource, id, ci.Namespaced) } + +func (ci *ClusterIdentity) WaitForValidCredential(kc *kubeclient.KubeClient) { + GinkgoHelper() + + By(fmt.Sprintf("waiting for %s credential to be ready", ci.CredentialName)) + + ctx := context.Background() + + Eventually(func() error { + cred, err := kc.GetCredential(ctx, ci.CredentialName) + if err != nil { + return fmt.Errorf("failed to get credntial: %w", err) + } + + ready, found, err := unstructured.NestedBool(cred.Object, "status", "ready") + if !found { + return fmt.Errorf("failed to get ready status: %w", err) + } + if !ready { + _, _ = fmt.Fprintf(GinkgoWriter, "credential is not ready, retrying...\n") + return fmt.Errorf("credential is not ready: %s", ci.GroupVersionResource.String()) + } + return nil + }).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(Succeed()) +} diff --git a/test/e2e/clusterdeployment/constants.go b/test/e2e/clusterdeployment/constants.go index f5a1e2037..2a356fdd7 100644 --- a/test/e2e/clusterdeployment/constants.go +++ b/test/e2e/clusterdeployment/constants.go @@ -47,4 +47,8 @@ const ( EnvVarAzureSubscription = "AZURE_SUBSCRIPTION" EnvVarAzureClusterIdentity = "AZURE_CLUSTER_IDENTITY" EnvVarAzureRegion = "AZURE_REGION" + + // Adopted + EnvVarAdoptedKubeconfigPath = "KUBECONFIG_DATA_PATH" + EnvVarAdoptedCredential = "ADOPTED_CREDENTIAL" ) diff --git a/test/e2e/clusterdeployment/providervalidator.go b/test/e2e/clusterdeployment/providervalidator.go index 9d4ee121d..80d1cdaca 100644 --- a/test/e2e/clusterdeployment/providervalidator.go +++ b/test/e2e/clusterdeployment/providervalidator.go @@ -67,6 +67,11 @@ func NewProviderValidator(template Template, clusterName string, action Validati resourceOrder = append(resourceOrder, "ccm") case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP: delete(resourcesToValidate, "csi-driver") + + case TemplateAdoptedCluster: + resourcesToValidate = map[string]resourceValidationFunc{ + "sveltoscluster": validateSveltosCluster, + } } } else { resourcesToValidate = map[string]resourceValidationFunc{ @@ -74,6 +79,7 @@ func NewProviderValidator(template Template, clusterName string, action Validati "machinedeployments": validateMachineDeploymentsDeleted, "control-planes": validateK0sControlPlanesDeleted, } + resourceOrder = []string{"clusters", "machinedeployments", "control-planes"} } diff --git a/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl b/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl new file mode 100644 index 000000000..180926bae --- /dev/null +++ b/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl @@ -0,0 +1,16 @@ +apiVersion: hmc.mirantis.com/v1alpha1 +kind: ClusterDeployment +metadata: + name: ${CLUSTER_DEPLOYMENT_NAME} + namespace: ${NAMESPACE} +spec: + template: adopted-cluster-0-0-1 + credential: ${ADOPTED_CREDENTIAL} + config: {} + services: + - template: kyverno-3-2-6 + name: kyverno + namespace: kyverno + - template: ingress-nginx-4-11-0 + name: ingress-nginx + namespace: ingress-nginx \ No newline at end of file diff --git a/test/e2e/clusterdeployment/validate_deployed.go b/test/e2e/clusterdeployment/validate_deployed.go index 1dbcd7885..5b76b94e9 100644 --- a/test/e2e/clusterdeployment/validate_deployed.go +++ b/test/e2e/clusterdeployment/validate_deployed.go @@ -273,3 +273,22 @@ func validateCCM(ctx context.Context, kc *kubeclient.KubeClient, clusterName str return fmt.Errorf("%s Service does not yet have an external hostname", service.Name) } + +// validateSveltosCluster validates that the sveltos cluster is ready +func validateSveltosCluster(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { + sveltosCluster, err := kc.GetSveltosCluster(ctx, clusterName) + if err != nil { + return fmt.Errorf("error getting sveltos cluster: %v", err) + } + + ready, found, err := unstructured.NestedBool(sveltosCluster.Object, "status", "ready") + if err != nil { + return fmt.Errorf("error checking sveltos cluster ready: %v", err) + } + + if !found || !ready { + return fmt.Errorf("sveltos cluster %s is not ready", clusterName) + } + + return nil +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index cccd0e70f..25bcfa2cd 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -47,7 +47,6 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { GinkgoT().Setenv(clusterdeployment.EnvVarNamespace, internalutils.DefaultSystemNamespace) - By("building and deploying the controller-manager") cmd := exec.Command("make", "kind-deploy") _, err := utils.Run(cmd) @@ -66,6 +65,15 @@ var _ = BeforeSuite(func() { } return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + Eventually(func() error { + err = clusterdeployment.ValidateClusterTemplates(context.Background(), kc) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) }) var _ = AfterSuite(func() { diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go index 760426ae4..926dcb1c8 100644 --- a/test/e2e/kubeclient/kubeclient.go +++ b/test/e2e/kubeclient/kubeclient.go @@ -56,7 +56,7 @@ func NewFromLocal(namespace string) *KubeClient { // the kubeconfig from secret it needs an existing kubeclient. func (kc *KubeClient) NewFromCluster(ctx context.Context, namespace, clusterName string) *KubeClient { GinkgoHelper() - return newKubeClient(kc.getKubeconfigSecretData(ctx, clusterName), namespace) + return newKubeClient(kc.GetKubeconfigSecretData(ctx, clusterName), namespace) } // WriteKubeconfig writes the kubeconfig for the given clusterName to the @@ -65,7 +65,7 @@ func (kc *KubeClient) NewFromCluster(ctx context.Context, namespace, clusterName func (kc *KubeClient) WriteKubeconfig(ctx context.Context, clusterName string) (string, func() error) { GinkgoHelper() - secretData := kc.getKubeconfigSecretData(ctx, clusterName) + secretData := kc.GetKubeconfigSecretData(ctx, clusterName) dir, err := os.Getwd() Expect(err).NotTo(HaveOccurred()) @@ -89,7 +89,7 @@ func (kc *KubeClient) WriteKubeconfig(ctx context.Context, clusterName string) ( return path, deleteFunc } -func (kc *KubeClient) getKubeconfigSecretData(ctx context.Context, clusterName string) []byte { +func (kc *KubeClient) GetKubeconfigSecretData(ctx context.Context, clusterName string) []byte { GinkgoHelper() secret, err := kc.Client.CoreV1().Secrets(kc.Namespace).Get(ctx, clusterName+"-kubeconfig", metav1.GetOptions{}) @@ -279,3 +279,48 @@ func (kc *KubeClient) ListK0sControlPlanes( Resource: "k0scontrolplanes", }, clusterName) } + +func (kc *KubeClient) ListClusterTemplates(ctx context.Context) ([]unstructured.Unstructured, error) { + client := kc.GetDynamicClient(schema.GroupVersionResource{ + Group: "hmc.mirantis.com", + Version: "v1alpha1", + Resource: "clustertemplates", + }, true) + + resources, err := client.List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list cluster templates") + } + + return resources.Items, nil +} + +func (kc *KubeClient) GetCredential(ctx context.Context, name string) (*unstructured.Unstructured, error) { + client := kc.GetDynamicClient(schema.GroupVersionResource{ + Group: "hmc.mirantis.com", + Version: "v1alpha1", + Resource: "credentials", + }, true) + + credential, err := client.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get credential %s: %w", name, err) + } + + return credential, nil +} + +func (kc *KubeClient) GetSveltosCluster(ctx context.Context, name string) (*unstructured.Unstructured, error) { + client := kc.GetDynamicClient(schema.GroupVersionResource{ + Group: "lib.projectsveltos.io", + Version: "v1beta1", + Resource: "sveltosclusters", + }, true) + + sveltosCluster, err := client.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get sveltos cluster %s: %w", name, err) + } + + return sveltosCluster, nil +} diff --git a/test/e2e/provider_adopted_test.go b/test/e2e/provider_adopted_test.go new file mode 100644 index 000000000..4ed9a6647 --- /dev/null +++ b/test/e2e/provider_adopted_test.go @@ -0,0 +1,133 @@ +// Copyright 2024 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "os" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + internalutils "github.com/K0rdent/kcm/internal/utils" + "github.com/K0rdent/kcm/test/e2e/clusterdeployment" + "github.com/K0rdent/kcm/test/e2e/clusterdeployment/clusteridentity" + "github.com/K0rdent/kcm/test/e2e/kubeclient" +) + +var _ = Describe("Adopted Cluster Templates", Label("provider:cloud", "provider:adopted"), Ordered, func() { + var ( + kc *kubeclient.KubeClient + standaloneClient *kubeclient.KubeClient + clusterDeleteFunc func() error + adoptedDeleteFunc func() error + kubecfgDeleteFunc func() error + clusterName string + ) + + BeforeAll(func() { + By("providing cluster identity") + kc = kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace) + ci := clusteridentity.New(kc, clusterdeployment.ProviderAWS) + Expect(os.Setenv(clusterdeployment.EnvVarAWSClusterIdentity, ci.IdentityName)).Should(Succeed()) + ci.WaitForValidCredential(kc) + }) + + AfterAll(func() { + // If we failed collect logs from each of the affiliated controllers + // as well as the output of clusterctl to store as artifacts. + if CurrentSpecReport().Failed() && !noCleanup() { + if standaloneClient != nil { + By("collecting failure logs from hosted controllers") + collectLogArtifacts(standaloneClient, clusterName, clusterdeployment.ProviderAWS, clusterdeployment.ProviderCAPI) + } + } + + By("deleting resources") + for _, deleteFunc := range []func() error{ + kubecfgDeleteFunc, + adoptedDeleteFunc, + clusterDeleteFunc, + } { + if deleteFunc != nil { + err := deleteFunc() + Expect(err).NotTo(HaveOccurred()) + } + } + }) + + It("should work with an Adopted cluster provider", func() { + // Deploy a standalone cluster and verify it is running/ready. Then, delete the management cluster and + // recreate it. Next "adopt" the cluster we created and verify the services were deployed. + GinkgoT().Setenv(clusterdeployment.EnvVarAWSInstanceType, "t3.xlarge") + + templateBy(clusterdeployment.TemplateAWSStandaloneCP, "creating a ManagedCluster") + sd := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAWSStandaloneCP) + clusterName = sd.GetName() + + clusterDeleteFunc = kc.CreateClusterDeployment(context.Background(), sd) + + templateBy(clusterdeployment.TemplateAWSStandaloneCP, "waiting for infrastructure to deploy successfully") + deploymentValidator := clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAWSStandaloneCP, + clusterName, + clusterdeployment.ValidationActionDeploy, + ) + + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), kc) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + // create the adopted cluster using the AWS standalone cluster + var kubeCfgFile string + kubeCfgFile, kubecfgDeleteFunc = kc.WriteKubeconfig(context.Background(), clusterName) + GinkgoT().Setenv(clusterdeployment.EnvVarAdoptedKubeconfigPath, kubeCfgFile) + ci := clusteridentity.New(kc, clusterdeployment.ProviderAdopted) + Expect(os.Setenv(clusterdeployment.EnvVarAdoptedCredential, ci.CredentialName)).Should(Succeed()) + + ci.WaitForValidCredential(kc) + + adoptedCluster := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAdoptedCluster) + adoptedClusterName := adoptedCluster.GetName() + adoptedDeleteFunc = kc.CreateClusterDeployment(context.Background(), adoptedCluster) + + // validate the adopted cluster + deploymentValidator = clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAdoptedCluster, + adoptedClusterName, + clusterdeployment.ValidationActionDeploy, + ) + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), kc) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + err := adoptedDeleteFunc() + Expect(err).NotTo(HaveOccurred()) + + err = clusterDeleteFunc() + Expect(err).NotTo(HaveOccurred()) + + // finally delete the aws standalone clsuter + deletionValidator := clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAWSStandaloneCP, + clusterName, + clusterdeployment.ValidationActionDelete, + ) + Eventually(func() error { + return deletionValidator.Validate(context.Background(), kc) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + }) +}) diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 5f11a1061..7bc7f1fd1 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -46,6 +46,7 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order By("providing cluster identity") kc = kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace) ci := clusteridentity.New(kc, clusterdeployment.ProviderAWS) + ci.WaitForValidCredential(kc) Expect(os.Setenv(clusterdeployment.EnvVarAWSClusterIdentity, ci.IdentityName)).Should(Succeed()) }) @@ -125,7 +126,8 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) // Ensure AWS credentials are set in the standalone cluster. - clusteridentity.New(standaloneClient, clusterdeployment.ProviderAWS) + standaloneCi := clusteridentity.New(standaloneClient, clusterdeployment.ProviderAWS) + standaloneCi.WaitForValidCredential(standaloneClient) // Populate the environment variables required for the hosted // cluster. diff --git a/test/e2e/provider_azure_test.go b/test/e2e/provider_azure_test.go index 401aba31a..5ce409deb 100644 --- a/test/e2e/provider_azure_test.go +++ b/test/e2e/provider_azure_test.go @@ -47,6 +47,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or By("ensuring Azure credentials are set") kc = kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace) ci := clusteridentity.New(kc, clusterdeployment.ProviderAzure) + ci.WaitForValidCredential(kc) Expect(os.Setenv(clusterdeployment.EnvVarAzureClusterIdentity, ci.IdentityName)).Should(Succeed()) }) @@ -124,7 +125,8 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) By("Create azure credential secret") - clusteridentity.New(standaloneClient, clusterdeployment.ProviderAzure) + standaloneCi := clusteridentity.New(standaloneClient, clusterdeployment.ProviderAzure) + standaloneCi.WaitForValidCredential(standaloneClient) By("Create default storage class for azure-disk CSI driver") azure.CreateDefaultStorageClass(standaloneClient) diff --git a/test/e2e/provider_vsphere_test.go b/test/e2e/provider_vsphere_test.go index a4f54b3a6..d1b635f8d 100644 --- a/test/e2e/provider_vsphere_test.go +++ b/test/e2e/provider_vsphere_test.go @@ -44,6 +44,7 @@ var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere" kc = kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace) By("providing cluster identity") ci := clusteridentity.New(kc, clusterdeployment.ProviderVSphere) + ci.WaitForValidCredential(kc) By("setting VSPHERE_CLUSTER_IDENTITY env variable") Expect(os.Setenv(clusterdeployment.EnvVarVSphereClusterIdentity, ci.IdentityName)).Should(Succeed()) }) From 88ed748ee633d94cecebfd857914baa055febeb3 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Thu, 9 Jan 2025 09:20:31 -0800 Subject: [PATCH 2/2] Fix up AWS and Azure tests --- config/dev/adopted-clusterdeployment.yaml | 2 +- test/e2e/clusterdeployment/aws/aws.go | 39 ++++++++++++----- .../clusterdeployment/clusterdeployment.go | 3 +- test/e2e/clusterdeployment/constants.go | 21 +++++----- .../clusterdeployment/providervalidator.go | 3 +- .../resources/adopted-cluster.yaml.tpl | 2 +- .../resources/aws-hosted-cp.yaml.tpl | 7 +++- .../resources/aws-standalone-cp.yaml.tpl | 2 + test/e2e/provider_adopted_test.go | 5 ++- test/e2e/provider_aws_test.go | 42 +++++++++++-------- test/e2e/provider_azure_test.go | 20 ++++++--- 11 files changed, 94 insertions(+), 52 deletions(-) diff --git a/config/dev/adopted-clusterdeployment.yaml b/config/dev/adopted-clusterdeployment.yaml index c20e819df..b84bb6641 100644 --- a/config/dev/adopted-clusterdeployment.yaml +++ b/config/dev/adopted-clusterdeployment.yaml @@ -7,7 +7,7 @@ spec: template: adopted-cluster-0-0-1 credential: adopted-cluster-cred config: {} - services: + serviceSpec: - template: kyverno-3-2-6 name: kyverno namespace: kyverno diff --git a/test/e2e/clusterdeployment/aws/aws.go b/test/e2e/clusterdeployment/aws/aws.go index 61a2a0a21..79dd76496 100644 --- a/test/e2e/clusterdeployment/aws/aws.go +++ b/test/e2e/clusterdeployment/aws/aws.go @@ -17,10 +17,14 @@ package aws import ( + "bufio" "context" + "fmt" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -52,22 +56,37 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster subnets") Expect(found).To(BeTrue(), "AWS cluster has no subnets") - subnet, ok := subnets[0].(map[string]any) - Expect(ok).To(BeTrue(), "failed to cast subnet to map") - - subnetID, ok := subnet["resourceID"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet ID to string") - - subnetAZ, ok := subnet["availabilityZone"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet availability zone to string") + type awsSubnetMaps []map[string]any + subnetMaps := make(awsSubnetMaps, len(subnets)) + for i, s := range subnets { + subnet, ok := s.(map[string]any) + Expect(ok).To(BeTrue(), "failed to cast subnet to map") + subnetMaps[i] = map[string]any{ + "isPublic": subnet["isPublic"], + "availabilityZone": subnet["availabilityZone"], + "id": subnet["resourceID"], + "routeTableId": subnet["routeTableId"], + "zoneType": "availability-zone", + } + if natGatewayID, exists := subnet["natGatewayId"]; exists && natGatewayID != "" { + subnetMaps[i]["natGatewayId"] = natGatewayID + } + } + var subnetsFormatted string + encodedYaml, err := yaml.Marshal(subnetMaps) + Expect(err).NotTo(HaveOccurred(), "failed to get marshall subnet maps") + scanner := bufio.NewScanner(strings.NewReader(string(encodedYaml))) + for scanner.Scan() { + subnetsFormatted += fmt.Sprintf(" %s\n", scanner.Text()) + } + GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnets, subnetsFormatted) securityGroupID, found, err := unstructured.NestedString( awsCluster.Object, "status", "networkStatus", "securityGroups", "node", "id") Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster security group ID") Expect(found).To(BeTrue(), "AWS cluster has no security group ID") GinkgoT().Setenv(clusterdeployment.EnvVarAWSVPCID, vpcID) - GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnetID, subnetID) - GinkgoT().Setenv(clusterdeployment.EnvVarAWSSubnetAvailabilityZone, subnetAZ) GinkgoT().Setenv(clusterdeployment.EnvVarAWSSecurityGroupID, securityGroupID) + GinkgoT().Setenv(clusterdeployment.EnvVarManagementClusterName, clusterName) } diff --git a/test/e2e/clusterdeployment/clusterdeployment.go b/test/e2e/clusterdeployment/clusterdeployment.go index f25ef91f3..1179c37b9 100644 --- a/test/e2e/clusterdeployment/clusterdeployment.go +++ b/test/e2e/clusterdeployment/clusterdeployment.go @@ -127,8 +127,7 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { // since we populate the vars from standalone prior to this step. ValidateDeploymentVars([]string{ EnvVarAWSVPCID, - EnvVarAWSSubnetID, - EnvVarAWSSubnetAvailabilityZone, + EnvVarAWSSubnets, EnvVarAWSSecurityGroupID, }) clusterDeploymentTemplateBytes = awsHostedCPClusterDeploymentTemplateBytes diff --git a/test/e2e/clusterdeployment/constants.go b/test/e2e/clusterdeployment/constants.go index 2a356fdd7..17c196301 100644 --- a/test/e2e/clusterdeployment/constants.go +++ b/test/e2e/clusterdeployment/constants.go @@ -22,18 +22,17 @@ const ( EnvVarNamespace = "NAMESPACE" // EnvVarNoCleanup disables After* cleanup in provider specs to allow for // debugging of test failures. - EnvVarNoCleanup = "NO_CLEANUP" - + EnvVarNoCleanup = "NO_CLEANUP" + EnvVarManagementClusterName = "MANAGEMENT_CLUSTER_NAME" // AWS - EnvVarAWSAccessKeyID = "AWS_ACCESS_KEY_ID" - EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY" - EnvVarAWSVPCID = "AWS_VPC_ID" - EnvVarAWSSubnetID = "AWS_SUBNET_ID" - EnvVarAWSSubnetAvailabilityZone = "AWS_SUBNET_AVAILABILITY_ZONE" - EnvVarAWSInstanceType = "AWS_INSTANCE_TYPE" - EnvVarAWSSecurityGroupID = "AWS_SG_ID" - EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY" - EnvVarPublicIP = "AWS_PUBLIC_IP" + EnvVarAWSAccessKeyID = "AWS_ACCESS_KEY_ID" + EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + EnvVarAWSVPCID = "AWS_VPC_ID" + EnvVarAWSSubnets = "AWS_SUBNETS" + EnvVarAWSInstanceType = "AWS_INSTANCE_TYPE" + EnvVarAWSSecurityGroupID = "AWS_SG_ID" + EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY" + EnvVarPublicIP = "AWS_PUBLIC_IP" // VSphere EnvVarVSphereUser = "VSPHERE_USER" diff --git a/test/e2e/clusterdeployment/providervalidator.go b/test/e2e/clusterdeployment/providervalidator.go index 80d1cdaca..0e45f4650 100644 --- a/test/e2e/clusterdeployment/providervalidator.go +++ b/test/e2e/clusterdeployment/providervalidator.go @@ -65,9 +65,8 @@ func NewProviderValidator(template Template, clusterName string, action Validati case TemplateAWSStandaloneCP, TemplateAWSHostedCP: resourcesToValidate["ccm"] = validateCCM resourceOrder = append(resourceOrder, "ccm") - case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP: + case TemplateAzureStandaloneCP, TemplateAzureHostedCP, TemplateVSphereStandaloneCP: delete(resourcesToValidate, "csi-driver") - case TemplateAdoptedCluster: resourcesToValidate = map[string]resourceValidationFunc{ "sveltoscluster": validateSveltosCluster, diff --git a/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl b/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl index 180926bae..26e0af8e1 100644 --- a/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl +++ b/test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl @@ -7,7 +7,7 @@ spec: template: adopted-cluster-0-0-1 credential: ${ADOPTED_CREDENTIAL} config: {} - services: + serviceSpec: - template: kyverno-3-2-6 name: kyverno namespace: kyverno diff --git a/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl b/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl index 62fd8685e..3df4d2f0d 100644 --- a/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl +++ b/test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl @@ -12,8 +12,11 @@ spec: vpcID: ${AWS_VPC_ID} region: ${AWS_REGION} subnets: - - id: ${AWS_SUBNET_ID} - availabilityZone: ${AWS_SUBNET_AVAILABILITY_ZONE} +${AWS_SUBNETS} instanceType: ${AWS_INSTANCE_TYPE:=t3.medium} securityGroupIDs: - ${AWS_SG_ID} + managementClusterName: ${MANAGEMENT_CLUSTER_NAME} + controlPlane: + rootVolumeSize: 30 + rootVolumeSize: 30 \ No newline at end of file diff --git a/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl b/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl index 3a8252d9d..52187f079 100644 --- a/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl +++ b/test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl @@ -15,5 +15,7 @@ spec: workersNumber: ${WORKERS_NUMBER:=1} controlPlane: instanceType: ${AWS_INSTANCE_TYPE:=t3.small} + rootVolumeSize: 30 worker: instanceType: ${AWS_INSTANCE_TYPE:=t3.small} + rootVolumeSize: 30 \ No newline at end of file diff --git a/test/e2e/provider_adopted_test.go b/test/e2e/provider_adopted_test.go index 4ed9a6647..666595ea9 100644 --- a/test/e2e/provider_adopted_test.go +++ b/test/e2e/provider_adopted_test.go @@ -114,13 +114,16 @@ var _ = Describe("Adopted Cluster Templates", Label("provider:cloud", "provider: return deploymentValidator.Validate(context.Background(), kc) }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + // delete the adopted cluster err := adoptedDeleteFunc() Expect(err).NotTo(HaveOccurred()) + adoptedDeleteFunc = nil + // finally delete the aws standalone cluster err = clusterDeleteFunc() Expect(err).NotTo(HaveOccurred()) + clusterDeleteFunc = nil - // finally delete the aws standalone clsuter deletionValidator := clusterdeployment.NewProviderValidator( clusterdeployment.TemplateAWSStandaloneCP, clusterName, diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 7bc7f1fd1..589d6ac1d 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -125,6 +125,15 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + Eventually(func() error { + err = clusterdeployment.ValidateClusterTemplates(context.Background(), standaloneClient) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + // Ensure AWS credentials are set in the standalone cluster. standaloneCi := clusteridentity.New(standaloneClient, clusterdeployment.ProviderAWS) standaloneCi.WaitForValidCredential(standaloneClient) @@ -158,6 +167,7 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order templateBy(clusterdeployment.TemplateAWSHostedCP, "deleting the clusterdeployment") err = hostedDeleteFunc() Expect(err).NotTo(HaveOccurred()) + hostedDeleteFunc = nil deletionValidator := clusterdeployment.NewProviderValidator( clusterdeployment.TemplateAWSHostedCP, @@ -170,22 +180,20 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order // Now delete the standalone clusterdeployment and verify it is // removed, it is deleted last since it is the basis for the hosted // cluster. - /* - FIXME(#339): This is currently disabled as the deletion of the - standalone cluster is failing due to outstanding issues. - templateBy(clusterdeployment.TemplateAWSStandaloneCP, "deleting the clusterdeployment") - err = standaloneDeleteFunc() - Expect(err).NotTo(HaveOccurred()) - - deletionValidator = clusterdeployment.NewProviderValidator( - clusterdeployment.TemplateAWSStandaloneCP, - clusterName, - clusterdeployment.ValidationActionDelete, - ) - Eventually(func() error { - return deletionValidator.Validate(context.Background(), kc) - }).WithTimeout(10 * time.Minute).WithPolling(10 * - time.Second).Should(Succeed()) - */ + + templateBy(clusterdeployment.TemplateAWSStandaloneCP, "deleting the clusterdeployment") + err = standaloneDeleteFunc() + Expect(err).NotTo(HaveOccurred()) + + standaloneDeleteFunc = nil + deletionValidator = clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAWSStandaloneCP, + clusterName, + clusterdeployment.ValidationActionDelete, + ) + Eventually(func() error { + return deletionValidator.Validate(context.Background(), kc) + }).WithTimeout(10 * time.Minute).WithPolling(10 * + time.Second).Should(Succeed()) }) }) diff --git a/test/e2e/provider_azure_test.go b/test/e2e/provider_azure_test.go index 5ce409deb..a4f39ac2f 100644 --- a/test/e2e/provider_azure_test.go +++ b/test/e2e/provider_azure_test.go @@ -83,7 +83,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or sd := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAzureStandaloneCP) sdName = sd.GetName() - standaloneDeleteFunc := kc.CreateClusterDeployment(context.Background(), sd) + standaloneDeleteFunc = kc.CreateClusterDeployment(context.Background(), sd) // verify the standalone cluster is deployed correctly deploymentValidator := clusterdeployment.NewProviderValidator( @@ -124,6 +124,15 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + Eventually(func() error { + err = clusterdeployment.ValidateClusterTemplates(context.Background(), standaloneClient) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + By("Create azure credential secret") standaloneCi := clusteridentity.New(standaloneClient, clusterdeployment.ProviderAzure) standaloneCi.WaitForValidCredential(standaloneClient) @@ -151,10 +160,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or By("verify the deployment deletes successfully") err = hostedDeleteFunc() Expect(err).NotTo(HaveOccurred()) - - err = standaloneDeleteFunc() - Expect(err).NotTo(HaveOccurred()) - + hostedDeleteFunc = nil deploymentValidator = clusterdeployment.NewProviderValidator( clusterdeployment.TemplateAzureHostedCP, hdName, @@ -165,6 +171,10 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or return deploymentValidator.Validate(context.Background(), standaloneClient) }).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + err = standaloneDeleteFunc() + Expect(err).NotTo(HaveOccurred()) + standaloneDeleteFunc = nil + deploymentValidator = clusterdeployment.NewProviderValidator( clusterdeployment.TemplateAzureStandaloneCP, hdName,