Fix various issues applying credentials within e2e

Signed-off-by: Kyle Squizzato <[email protected]>

.github/workflows/build_test.yml

@@ -21,7 +21,7 @@ jobs:
     concurrency:
       group: build-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
-    name: Build and Test
+    name: Build and Unit Test
     runs-on: ubuntu-latest
     outputs:
       version: ${{ steps.vars.outputs.version }}
@@ -93,7 +93,7 @@ jobs:
         uses: azure/setup-kubectl@v4
       - name: Run E2E tests
         env:
-          GINKGO_LABEL_FILTER: '!provider'
+          GINKGO_LABEL_FILTER: 'controller'
           MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }}
           IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
         run: |

Makefile

@@ -302,7 +302,7 @@ dev-push: docker-build helm-push
 
 .PHONY: dev-templates
 dev-templates: templates-generate
-	$(KUBECTL) -n $(NAMESPACE) apply -f $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/templates
+	$(KUBECTL) -n $(NAMESPACE) apply --force -f $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/templates
 
 .PHONY: dev-release
 dev-release:

test/e2e/managedcluster/clusteridentity/clusteridentity.go

@@ -76,6 +76,7 @@ func New(kc *kubeclient.KubeClient, provider managedcluster.ProviderType) *Clust
 		Fail(fmt.Sprintf("Unsupported provider: %s", provider))
 	}
 
+	waitForResourceCRD(kc, resource)
 	validateSecretDataPopulated(secretStringData)
 
 	ci := ClusterIdentity{
@@ -94,33 +95,57 @@ func New(kc *kubeclient.KubeClient, provider managedcluster.ProviderType) *Clust
 
 func validateSecretDataPopulated(secretData map[string]string) {
 	for key, value := range secretData {
-		Expect(value).ToNot(BeEmpty(), fmt.Sprintf("Secret data key %s is empty", key))
+		Expect(value).ToNot(BeEmpty(), fmt.Sprintf("Secret data key %s should not be empty", key))
 	}
 }
 
-// createSecret creates a secret affiliated with a ClusterIdentity.
-func (ci *ClusterIdentity) createSecret(kc *kubeclient.KubeClient) {
+// waitForResourceCRD ensures the CRD for the given resource is present by
+// trying to list the resources of the given type until it succeeds.
+func waitForResourceCRD(kc *kubeclient.KubeClient, resource string) {
 	GinkgoHelper()
 
 	ctx := context.Background()
+	client, err := dynamic.NewForConfig(kc.Config)
+	Expect(err).NotTo(HaveOccurred())
 
-	_, err := kc.Client.CoreV1().Secrets(kc.Namespace).
-		Get(ctx, ci.SecretName, metav1.GetOptions{})
-	if !apierrors.IsNotFound(err) {
-		Expect(err).NotTo(HaveOccurred(), "failed to get AWS credentials secret")
-		return
+	gvr := schema.GroupVersionResource{
+		Group:    "infrastructure.cluster.x-k8s.io",
+		Version:  "v1beta2",
+		Resource: resource,
 	}
 
-	// The secret does not yet exist, so create it.
-	_, err = kc.Client.CoreV1().Secrets(kc.Namespace).Create(ctx, &corev1.Secret{
+	Eventually(func() bool {
+		_, err := client.Resource(gvr).List(ctx, metav1.ListOptions{})
+		return err == nil
+	}, "1m", "5s").Should(BeTrue(), "failed to list %s resources", resource)
+}
+
+// createSecret creates a secret affiliated with a ClusterIdentity.
+func (ci *ClusterIdentity) createSecret(kc *kubeclient.KubeClient) {
+	GinkgoHelper()
+
+	ctx := context.Background()
+
+	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      ci.SecretName,
 			Namespace: kc.Namespace,
 		},
 		StringData: ci.SecretData,
 		Type:       corev1.SecretTypeOpaque,
-	}, metav1.CreateOptions{})
-	Expect(err).NotTo(HaveOccurred(), "failed to create AWS credentials secret")
+	}
+
+	_, err := kc.Client.CoreV1().Secrets(kc.Namespace).Create(ctx, secret, metav1.CreateOptions{})
+	if apierrors.IsAlreadyExists(err) {
+		resp, err := kc.Client.CoreV1().Secrets(kc.Namespace).Get(ctx, ci.SecretName, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+
+		secret.SetResourceVersion(resp.GetResourceVersion())
+		_, err = kc.Client.CoreV1().Secrets(kc.Namespace).Update(ctx, secret, metav1.UpdateOptions{})
+		Expect(err).NotTo(HaveOccurred())
+	} else {
+		Expect(err).NotTo(HaveOccurred())
+	}
 }
 
 // createClusterIdentity creates a ClusterIdentity resource.
@@ -178,5 +203,14 @@ func (ci *ClusterIdentity) createClusterIdentity(kc *kubeclient.KubeClient) {
 	}
 
 	_, err = client.Resource(gvr).Create(ctx, clusterIdentity, metav1.CreateOptions{})
-	Expect(err).NotTo(HaveOccurred(), "failed to create cluster identity")
+	if apierrors.IsAlreadyExists(err) {
+		resp, err := client.Resource(gvr).Get(ctx, ci.IdentityName, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+
+		clusterIdentity.SetResourceVersion(resp.GetResourceVersion())
+		_, err = client.Resource(gvr).Update(ctx, clusterIdentity, metav1.UpdateOptions{})
+		Expect(err).NotTo(HaveOccurred(), "failed to create cluster identity")
+	} else {
+		Expect(err).NotTo(HaveOccurred())
+	}
 }

test/e2e/managedcluster/constants.go

@@ -34,15 +34,18 @@ const (
 	EnvVarAWSSubnetAvailabilityZone = "AWS_SUBNET_AVAILABILITY_ZONE"
 	EnvVarAWSInstanceType           = "AWS_INSTANCE_TYPE"
 	EnvVarAWSSecurityGroupID        = "AWS_SG_ID"
+	EnvVarAWSClusterIdentity        = "AWS_CLUSTER_IDENTITY"
 	EnvVarPublicIP                  = "AWS_PUBLIC_IP"
 
 	// VSphere
-	EnvVarVSphereUser     = "VSPHERE_USER"
-	EnvVarVSpherePassword = "VSPHERE_PASSWORD"
+	EnvVarVSphereUser            = "VSPHERE_USER"
+	EnvVarVSpherePassword        = "VSPHERE_PASSWORD"
+	EnvVarVSphereClusterIdentity = "VSPHERE_CLUSTER_IDENTITY"
 
 	// Azure
-	EnvVarAzureClientSecret = "AZURE_CLIENT_SECRET"
-	EnvVarAzureClientID     = "AZURE_CLIENT_ID"
-	EnvVarAzureTenantID     = "AZURE_TENANT_ID"
-	EnvVarAzureSubscription = "AZURE_SUBSCRIPTION"
+	EnvVarAzureClientSecret    = "AZURE_CLIENT_SECRET"
+	EnvVarAzureClientID        = "AZURE_CLIENT_ID"
+	EnvVarAzureTenantID        = "AZURE_TENANT_ID"
+	EnvVarAzureSubscription    = "AZURE_SUBSCRIPTION"
+	EnvVarAzureClusterIdentity = "AZURE_CLUSTER_IDENTITY"
 )

test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl

@@ -2,12 +2,12 @@ apiVersion: hmc.mirantis.com/v1alpha1
 kind: ManagedCluster
 metadata:
   name: ${HOSTED_MANAGED_CLUSTER_NAME}-aws
-  name: ${NAMESPACE}
+  namespace: ${NAMESPACE}
 spec:
   template: aws-hosted-cp
   config:
     clusterIdentity:
-      name: aws-cluster-identity
+      name: ${AWS_CLUSTER_IDENTITY}
       namespace: ${NAMESPACE}
     vpcID: ${AWS_VPC_ID}
     region: ${AWS_REGION}

test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl

@@ -7,7 +7,7 @@ spec:
   template: aws-standalone-cp
   config:
     clusterIdentity:
-      name: aws-cluster-identity
+      name:  ${AWS_CLUSTER_IDENTITY}
       namespace: ${NAMESPACE}
     region: ${AWS_REGION}
     publicIP: ${AWS_PUBLIC_IP:=true}

test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl

@@ -10,7 +10,7 @@ spec:
     subscriptionID: "${AZURE_SUBSCRIPTION_ID}"
     vmSize: Standard_A4_v2
     clusterIdentity:
-      name: azure-cluster-identity
+      name: ${AZURE_CLUSTER_IDENTITY}
       namespace: hmc-system
     resourceGroup: "${AZURE_RESOURCE_GROUP}"
     network:

test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl

@@ -15,7 +15,7 @@ spec:
     worker:
       vmSize: Standard_A4_v2
     clusterIdentity:
-      name: azure-cluster-identity
+      name: ${AZURE_CLUSTER_IDENTITY}
       namespace: ${NAMESPACE}
     tenantID: "${AZURE_TENANT_ID}"
     clientID: "${AZURE_CLIENT_ID}"

test/e2e/provider_aws_test.go

@@ -42,9 +42,10 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order
 	)
 
 	BeforeAll(func() {
-		By("setting AWS credentials")
+		By("providing cluster identity")
 		kc = kubeclient.NewFromLocal(internalutils.DefaultSystemNamespace)
-		clusteridentity.New(kc, managedcluster.ProviderAWS)
+		ci := clusteridentity.New(kc, managedcluster.ProviderAWS)
+		Expect(os.Setenv(managedcluster.EnvVarAWSClusterIdentity, ci.IdentityName)).Should(Succeed())
 	})
 
 	AfterAll(func() {

test/e2e/provider_azure_test.go

@@ -23,8 +23,8 @@ import (
 
 	"github.com/Mirantis/hmc/test/e2e/kubeclient"
 	"github.com/Mirantis/hmc/test/e2e/managedcluster"
+	"github.com/Mirantis/hmc/test/e2e/managedcluster/azure"
 	"github.com/Mirantis/hmc/test/e2e/managedcluster/clusteridentity"
-	"github.com/Mirantis/hmc/test/managedcluster/azure"
 	"github.com/Mirantis/hmc/test/utils"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -43,7 +43,8 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or
 	BeforeAll(func() {
 		By("ensuring Azure credentials are set")
 		kc = kubeclient.NewFromLocal(namespace)
-		clusteridentity.New(kc, managedcluster.ProviderAzure)
+		ci := clusteridentity.New(kc, managedcluster.ProviderAzure)
+		Expect(os.Setenv(managedcluster.EnvVarAzureClusterIdentity, ci.IdentityName)).Should(Succeed())
 	})
 
 	AfterEach(func() {
@@ -177,11 +178,7 @@ func deployOnAzureCluster(kubeCfgPath string) {
 	_, err = utils.Run(cmd)
 	Expect(err).NotTo(HaveOccurred())
 
-	cmd = exec.Command("make", "dev-deploy")
-	_, err = utils.Run(cmd)
-	Expect(err).NotTo(HaveOccurred())
-
-	cmd = exec.Command("make", "dev-templates")
+	cmd = exec.Command("make", "test-apply")
 	_, err = utils.Run(cmd)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(os.Unsetenv("KUBECONFIG")).To(Succeed())

test/e2e/provider_vsphere_test.go

@@ -43,7 +43,7 @@ var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere"
 		By("providing cluster identity")
 		ci := clusteridentity.New(kc, managedcluster.ProviderVSphere)
 		By("setting VSPHERE_CLUSTER_IDENTITY env variable")
-		Expect(os.Setenv("VSPHERE_CLUSTER_IDENTITY", ci.IdentityName)).Should(Succeed())
+		Expect(os.Setenv(managedcluster.EnvVarVSphereClusterIdentity, ci.IdentityName)).Should(Succeed())
 	})
 
 	AfterEach(func() {