From 00088a607f9537d239188b9e73a4a7c5a30a9140 Mon Sep 17 00:00:00 2001 From: Kyle Squizzato Date: Wed, 2 Oct 2024 12:16:59 -0700 Subject: [PATCH] Fix issues deploying vsphere-csi-driver Signed-off-by: Kyle Squizzato --- .../templates/k0scontrolplane.yaml | 2 + test/e2e/kubeclient/kubeclient.go | 30 ++++---- test/e2e/managedcluster/aws/aws.go | 2 +- test/e2e/managedcluster/azure/azure.go | 2 +- .../clusteridentity/clusteridentity.go | 77 ++++++++++--------- 5 files changed, 63 insertions(+), 50 deletions(-) diff --git a/templates/cluster/vsphere-standalone-cp/templates/k0scontrolplane.yaml b/templates/cluster/vsphere-standalone-cp/templates/k0scontrolplane.yaml index 5bf52722a..60aeb2cc2 100644 --- a/templates/cluster/vsphere-standalone-cp/templates/k0scontrolplane.yaml +++ b/templates/cluster/vsphere-standalone-cp/templates/k0scontrolplane.yaml @@ -114,8 +114,10 @@ spec: images: driver: tag: v3.1.2 + repo: "registry.k8s.io/csi-vsphere/driver" syncer: tag: v3.1.2 + repo: "registry.k8s.io/csi-vsphere/syncer" machineTemplate: infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go index 4b74c0359..55a244539 100644 --- a/test/e2e/kubeclient/kubeclient.go +++ b/test/e2e/kubeclient/kubeclient.go @@ -136,34 +136,38 @@ func newKubeClient(configBytes []byte, namespace string) *KubeClient { } // GetDynamicClient returns a dynamic client for the given GroupVersionResource. -func (kc *KubeClient) GetDynamicClient(gvr schema.GroupVersionResource) dynamic.ResourceInterface { +// +//nolint:revive +func (kc *KubeClient) GetDynamicClient(gvr schema.GroupVersionResource, namespaced bool) dynamic.ResourceInterface { GinkgoHelper() client, err := dynamic.NewForConfig(kc.Config) Expect(err).NotTo(HaveOccurred(), "failed to create dynamic client for resource: %s", gvr.String()) + if !namespaced { + return client.Resource(gvr) + } + return client.Resource(gvr).Namespace(kc.Namespace) } -func (kc *KubeClient) CreateOrUpdateUnstructuredObject(gvr schema.GroupVersionResource, obj *unstructured.Unstructured) { +func (kc *KubeClient) CreateOrUpdateUnstructuredObject(gvr schema.GroupVersionResource, obj *unstructured.Unstructured, namespaced bool) { GinkgoHelper() - client := kc.GetDynamicClient(gvr) + client := kc.GetDynamicClient(gvr, namespaced) kind, name := utils.ObjKindName(obj) - _, _ = fmt.Fprintf(GinkgoWriter, "Creating or updating %s: %s: %+v\n with kubeclient: %+v", kind, name, obj, *kc) - - _, err := client.Create(context.Background(), obj, metav1.CreateOptions{}) - if apierrors.IsAlreadyExists(err) { - resp, err := client.Get(context.Background(), name, metav1.GetOptions{}) + resp, err := client.Get(context.Background(), name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + _, err = client.Create(context.Background(), obj, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred(), "failed to create %s: %s", kind, name) + } else { Expect(err).NotTo(HaveOccurred(), "failed to get existing %s: %s", kind, name) obj.SetResourceVersion(resp.GetResourceVersion()) _, err = client.Update(context.Background(), obj, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred(), "failed to update existing %s: %s", kind, name) - } else { - Expect(err).NotTo(HaveOccurred(), "failed to create %s: %s", kind, name) } } @@ -182,7 +186,7 @@ func (kc *KubeClient) CreateManagedCluster( Group: "hmc.mirantis.com", Version: "v1alpha1", Resource: "managedclusters", - }) + }, true) _, err := client.Create(ctx, managedcluster, metav1.CreateOptions{}) if !apierrors.IsAlreadyExists(err) { @@ -206,7 +210,7 @@ func (kc *KubeClient) GetCluster(ctx context.Context, clusterName string) (*unst Resource: "clusters", } - client := kc.GetDynamicClient(gvr) + client := kc.GetDynamicClient(gvr, true) cluster, err := client.Get(ctx, clusterName, metav1.GetOptions{}) if err != nil { @@ -221,7 +225,7 @@ func (kc *KubeClient) GetCluster(ctx context.Context, clusterName string) (*unst func (kc *KubeClient) listResource( ctx context.Context, gvr schema.GroupVersionResource, clusterName string, ) ([]unstructured.Unstructured, error) { - client := kc.GetDynamicClient(gvr) + client := kc.GetDynamicClient(gvr, true) resources, err := client.List(ctx, metav1.ListOptions{ LabelSelector: "cluster.x-k8s.io/cluster-name=" + clusterName, diff --git a/test/e2e/managedcluster/aws/aws.go b/test/e2e/managedcluster/aws/aws.go index 7533b83bc..0c839941e 100644 --- a/test/e2e/managedcluster/aws/aws.go +++ b/test/e2e/managedcluster/aws/aws.go @@ -98,5 +98,5 @@ func getAWSClusterClient(kc *kubeclient.KubeClient) dynamic.ResourceInterface { Group: "infrastructure.cluster.x-k8s.io", Version: "v1beta2", Resource: "awsclusters", - }) + }, true) } diff --git a/test/e2e/managedcluster/azure/azure.go b/test/e2e/managedcluster/azure/azure.go index 104d55146..f08065527 100644 --- a/test/e2e/managedcluster/azure/azure.go +++ b/test/e2e/managedcluster/azure/azure.go @@ -37,7 +37,7 @@ func getAzureInfo(ctx context.Context, name string, kc *kubeclient.KubeClient) m Resource: "azureclusters", } - dc := kc.GetDynamicClient(resourceID) + dc := kc.GetDynamicClient(resourceID, true) list, err := dc.List(ctx, metav1.ListOptions{ LabelSelector: labels.SelectorFromSet(map[string]string{hmc.FluxHelmChartNameKey: name}).String(), }) diff --git a/test/e2e/managedcluster/clusteridentity/clusteridentity.go b/test/e2e/managedcluster/clusteridentity/clusteridentity.go index 453f078e6..520ad982d 100644 --- a/test/e2e/managedcluster/clusteridentity/clusteridentity.go +++ b/test/e2e/managedcluster/clusteridentity/clusteridentity.go @@ -37,6 +37,8 @@ type ClusterIdentity struct { SecretName string IdentityName string SecretData map[string]string + Spec map[string]any + Namespaced bool } // New creates a ClusterIdentity resource, credential and associated secret for @@ -50,8 +52,13 @@ func New(kc *kubeclient.KubeClient, provider managedcluster.ProviderType) *Clust kind string version string secretStringData map[string]string + spec map[string]any + namespaced bool ) + secretName := fmt.Sprintf("%s-cluster-identity-secret", provider) + identityName := fmt.Sprintf("%s-cluster-identity", provider) + switch provider { case managedcluster.ProviderAWS: resource = "awsclusterstaticidentities" @@ -61,6 +68,14 @@ func New(kc *kubeclient.KubeClient, provider managedcluster.ProviderType) *Clust "AccessKeyID": os.Getenv(managedcluster.EnvVarAWSAccessKeyID), "SecretAccessKey": os.Getenv(managedcluster.EnvVarAWSSecretAccessKey), } + spec = map[string]any{ + "secretRef": secretName, + "allowedNamespaces": map[string]any{ + "selector": map[string]any{ + "matchLabels": map[string]any{}, + }, + }, + } case managedcluster.ProviderAzure: resource = "azureclusteridentities" kind = "AzureClusterIdentity" @@ -68,6 +83,17 @@ func New(kc *kubeclient.KubeClient, provider managedcluster.ProviderType) *Clust secretStringData = map[string]string{ "clientSecret": os.Getenv(managedcluster.EnvVarAzureClientSecret), } + spec = map[string]any{ + "allowedNamespaces": map[string]any{}, + "clientID": os.Getenv(managedcluster.EnvVarAzureClientID), + "clientSecret": map[string]any{ + "name": secretName, + "namespace": kc.Namespace, + }, + "tenantID": os.Getenv(managedcluster.EnvVarAzureTenantID), + "type": "ServicePrincipal", + } + namespaced = true case managedcluster.ProviderVSphere: resource = "vsphereclusteridentities" kind = "VSphereClusterIdentity" @@ -76,12 +102,18 @@ func New(kc *kubeclient.KubeClient, provider managedcluster.ProviderType) *Clust "username": os.Getenv(managedcluster.EnvVarVSphereUser), "password": os.Getenv(managedcluster.EnvVarVSpherePassword), } + spec = map[string]any{ + "secretName": secretName, + "allowedNamespaces": map[string]any{ + "selector": map[string]any{ + "matchLabels": map[string]any{}, + }, + }, + } default: Fail(fmt.Sprintf("Unsupported provider: %s", provider)) } - validateSecretDataPopulated(secretStringData) - ci := ClusterIdentity{ GroupVersionResource: schema.GroupVersionResource{ Group: "infrastructure.cluster.x-k8s.io", @@ -89,11 +121,14 @@ func New(kc *kubeclient.KubeClient, provider managedcluster.ProviderType) *Clust Resource: resource, }, Kind: kind, - SecretName: fmt.Sprintf("%s-cluster-identity-secret", provider), - IdentityName: fmt.Sprintf("%s-cluster-identity", provider), + SecretName: secretName, + IdentityName: identityName, SecretData: secretStringData, + Spec: spec, + Namespaced: namespaced, } + validateSecretDataPopulated(secretStringData) ci.waitForResourceCRD(kc) ci.createSecret(kc) ci.createClusterIdentity(kc) @@ -193,7 +228,7 @@ func (ci *ClusterIdentity) createCredential(kc *kubeclient.KubeClient) { Group: "hmc.mirantis.com", Version: "v1alpha1", Resource: "credentials", - }, cred) + }, cred, true) } // createClusterIdentity creates a ClusterIdentity resource. @@ -210,37 +245,9 @@ func (ci *ClusterIdentity) createClusterIdentity(kc *kubeclient.KubeClient) { "name": ci.IdentityName, "namespace": kc.Namespace, }, - "spec": map[string]any{ - "secretRef": ci.SecretName, - "allowedNamespaces": map[string]any{ - "selector": map[string]any{ - "matchLabels": map[string]any{}, - }, - }, - }, + "spec": ci.Spec, }, } - if ci.GroupVersionResource.Resource == "azureclusteridentities" { - err := unstructured.SetNestedField(id.Object, os.Getenv(managedcluster.EnvVarAzureTenantID), "spec", "tenantID") - Expect(err).NotTo(HaveOccurred()) - - managedcluster.ValidateDeploymentVars([]string{ - managedcluster.EnvVarAzureClientID, - managedcluster.EnvVarAzureTenantID, - }) - - id.Object["spec"] = map[string]any{ - "allowedNamespaces": map[string]any{}, - "clientID": os.Getenv(managedcluster.EnvVarAzureClientID), - "clientSecret": map[string]any{ - "name": ci.SecretName, - "namespace": kc.Namespace, - }, - "tenantID": os.Getenv(managedcluster.EnvVarAzureTenantID), - "type": "ServicePrincipal", - } - } - - kc.CreateOrUpdateUnstructuredObject(ci.GroupVersionResource, id) + kc.CreateOrUpdateUnstructuredObject(ci.GroupVersionResource, id, ci.Namespaced) }