Implemented Azure e2e tests

.github/workflows/test.yml

@@ -18,6 +18,11 @@ env:
   AWS_REGION: us-west-2
   AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}
+  AZURE_SUBSCRIPTION_ID: ${{ secrets.CI_AZURE_SUBSCRIPTION_ID }}
+  AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }}
+  AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }}
+  AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }}
+  NAMESPACE: hmc-system
 
 jobs:
   e2etest:

Makefile

@@ -109,7 +109,7 @@ test: generate-all fmt vet envtest tidy external-crd ## Run tests.
 # compatibility with other vendors.
 .PHONY: test-e2e # Run the e2e tests using a Kind k8s instance as the management cluster.
 test-e2e: cli-install
-	 KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) go test ./test/e2e/ -v -ginkgo.v -timeout=2h
+	KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) go test ./test/e2e/ -v -ginkgo.v -timeout=3h
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter & yamllint

test/e2e/e2e_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/Mirantis/hmc/test/kubeclient"
 	"github.com/Mirantis/hmc/test/managedcluster"
 	"github.com/Mirantis/hmc/test/managedcluster/aws"
+	"github.com/Mirantis/hmc/test/managedcluster/azure"
 	"github.com/Mirantis/hmc/test/managedcluster/vsphere"
 	"github.com/Mirantis/hmc/test/utils"
 )
@@ -126,7 +127,7 @@ var _ = Describe("controller", Ordered, func() {
 			GinkgoT().Setenv(managedcluster.EnvVarInstallBeachHeadServices, "false")
 
 			templateBy(managedcluster.TemplateAWSStandaloneCP, "creating a ManagedCluster")
-			sd := managedcluster.GetUnstructured(managedcluster.TemplateAWSStandaloneCP)
+			sd := managedcluster.GetUnstructured(managedcluster.TemplateAWSStandaloneCP, managedcluster.ProviderAWS)
 			clusterName = sd.GetName()
 
 			standaloneDeleteFunc = kc.CreateManagedCluster(context.Background(), sd)
@@ -179,10 +180,10 @@ var _ = Describe("controller", Ordered, func() {
 
 			// Populate the environment variables required for the hosted
 			// cluster.
-			aws.PopulateHostedTemplateVars(context.Background(), kc)
+			aws.PopulateHostedTemplateVars(context.Background(), kc, clusterName)
 
 			templateBy(managedcluster.TemplateAWSHostedCP, "creating a ManagedCluster")
-			hd := managedcluster.GetUnstructured(managedcluster.TemplateAWSHostedCP)
+			hd := managedcluster.GetUnstructured(managedcluster.TemplateAWSHostedCP, managedcluster.ProviderAWS)
 			hdName := hd.GetName()
 
 			// Deploy the hosted cluster on top of the standalone cluster.
@@ -296,7 +297,7 @@ var _ = Describe("controller", Ordered, func() {
 
 		It("should deploy standalone managed cluster", func() {
 			By("creating a managed cluster")
-			d := managedcluster.GetUnstructured(managedcluster.TemplateVSphereStandaloneCP)
+			d := managedcluster.GetUnstructured(managedcluster.TemplateVSphereStandaloneCP, managedcluster.ProviderVSphere)
 			clusterName = d.GetName()
 
 			deleteFunc := kc.CreateManagedCluster(context.Background(), d)
@@ -325,8 +326,165 @@ var _ = Describe("controller", Ordered, func() {
 		})
 	})
 
+	Describe("Azure Templates", Label("provider"), func() {
+		var (
+			kc                   *kubeclient.KubeClient
+			standaloneClient     *kubeclient.KubeClient
+			standaloneDeleteFunc func() error
+			hostedDeleteFunc     func() error
+			kubecfgDeleteFunc    func() error
+			sdName               string
+		)
+
+		BeforeAll(func() {
+			By("ensuring Azure credentials are set")
+			kc = kubeclient.NewFromLocal(namespace)
+			azure.CreateCredentialSecret(context.Background(), kc)
+		})
+
+		AfterEach(func() {
+			// If we failed collect logs from each of the affiliated controllers
+			// as well as the output of clusterctl to store as artifacts.
+			if CurrentSpecReport().Failed() && !noCleanup() {
+				By("collecting failure logs from controllers")
+				if kc != nil {
+					collectLogArtifacts(kc, sdName, managedcluster.ProviderAzure, managedcluster.ProviderCAPI)
+				}
+				if standaloneClient != nil {
+					collectLogArtifacts(standaloneClient, sdName, managedcluster.ProviderAzure, managedcluster.ProviderCAPI)
+				}
+
+				By("deleting resources after failure")
+				for _, deleteFunc := range []func() error{
+					kubecfgDeleteFunc,
+					hostedDeleteFunc,
+					standaloneDeleteFunc,
+				} {
+					if deleteFunc != nil {
+						err := deleteFunc()
+						Expect(err).NotTo(HaveOccurred())
+					}
+				}
+			}
+		})
+
+		It("should work with an Azure provider", func() {
+			templateBy(managedcluster.TemplateAzureStandaloneCP, "creating a ManagedCluster")
+			sd := managedcluster.GetUnstructured(managedcluster.TemplateAzureStandaloneCP, managedcluster.ProviderAzure)
+			sdName = sd.GetName()
+
+			standaloneDeleteFunc := kc.CreateManagedCluster(context.Background(), sd)
+
+			// verify the standalone cluster is deployed correctly
+			deploymentValidator := managedcluster.NewProviderValidator(
+				managedcluster.TemplateAzureStandaloneCP,
+				sdName,
+				managedcluster.ValidationActionDeploy,
+			)
+
+			templateBy(managedcluster.TemplateAzureStandaloneCP, "waiting for infrastructure provider to deploy successfully")
+			Eventually(func() error {
+				return deploymentValidator.Validate(context.Background(), kc)
+			}).WithTimeout(90 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
+			// setup environment variables for deploying the hosted template (subnet name, etc)
+			azure.SetAzureEnvironmentVariables(sdName, kc)
+
+			hd := managedcluster.GetUnstructured(managedcluster.TemplateAzureHostedCP, managedcluster.ProviderAzure)
+			hdName := hd.GetName()
+
+			var kubeCfgPath string
+			kubeCfgPath, kubecfgDeleteFunc = kc.WriteKubeconfig(context.Background(), sdName)
+
+			By("Deploy onto standalone cluster")
+			deployOnAzureCluster(kubeCfgPath)
+
+			templateBy(managedcluster.TemplateAzureHostedCP, "creating a ManagedCluster")
+			standaloneClient = kc.NewFromCluster(context.Background(), namespace, sdName)
+			// verify the cluster is ready prior to creating credentials
+			Eventually(func() error {
+				err := verifyControllersUp(standaloneClient, managedcluster.ProviderAzure)
+				if err != nil {
+					_, _ = fmt.Fprintf(GinkgoWriter, "Controller validation failed: %v\n", err)
+					return err
+				}
+				return nil
+			}).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
+			By("Create azure credential secret")
+			azure.CreateCredentialSecret(context.Background(), standaloneClient)
+
+			templateBy(managedcluster.TemplateAzureHostedCP,
+				fmt.Sprintf("creating a Deployment using template %s", managedcluster.TemplateAzureHostedCP))
+			hostedDeleteFunc = standaloneClient.CreateManagedCluster(context.Background(), hd)
+
+			templateBy(managedcluster.TemplateAzureHostedCP, "waiting for infrastructure to deploy successfully")
+
+			deploymentValidator = managedcluster.NewProviderValidator(
+				managedcluster.TemplateAzureStandaloneCP,
+				hdName,
+				managedcluster.ValidationActionDeploy,
+			)
+
+			Eventually(func() error {
+				return deploymentValidator.Validate(context.Background(), standaloneClient)
+			}).WithTimeout(90 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
+			By("verify the deployment deletes successfully")
+			err := hostedDeleteFunc()
+			Expect(err).NotTo(HaveOccurred())
+
+			err = standaloneDeleteFunc()
+			Expect(err).NotTo(HaveOccurred())
+
+			deploymentValidator = managedcluster.NewProviderValidator(
+				managedcluster.TemplateAzureHostedCP,
+				hdName,
+				managedcluster.ValidationActionDelete,
+			)
+
+			Eventually(func() error {
+				return deploymentValidator.Validate(context.Background(), standaloneClient)
+			}).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
+			deploymentValidator = managedcluster.NewProviderValidator(
+				managedcluster.TemplateAzureStandaloneCP,
+				hdName,
+				managedcluster.ValidationActionDelete,
+			)
+
+			Eventually(func() error {
+				return deploymentValidator.Validate(context.Background(), kc)
+			}).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
+		})
+	})
 })
 
+func deployOnAzureCluster(kubeCfgPath string) {
+	GinkgoT().Helper()
+	GinkgoT().Setenv("KUBECONFIG", kubeCfgPath)
+	cmd := exec.Command("kubectl", "create", "-f",
+		"https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/deploy/example/"+
+			"storageclass-azuredisk-csi.yaml")
+	_, err := utils.Run(cmd)
+	Expect(err).NotTo(HaveOccurred())
+
+	cmd = exec.Command("kubectl", "patch", "storageclass", "managed-csi", "-p",
+		"{\"metadata\": {\"annotations\":{\"storageclass.kubernetes.io/is-default-class\":\"true\"}}}")
+	_, err = utils.Run(cmd)
+	Expect(err).NotTo(HaveOccurred())
+
+	cmd = exec.Command("make", "dev-deploy")
+	_, err = utils.Run(cmd)
+	Expect(err).NotTo(HaveOccurred())
+
+	cmd = exec.Command("make", "dev-templates")
+	_, err = utils.Run(cmd)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(os.Unsetenv("KUBECONFIG")).To(Succeed())
+}
+
 // templateBy wraps a Ginkgo By with a block describing the template being
 // tested.
 func templateBy(t managedcluster.Template, description string) {

test/managedcluster/aws/aws.go

@@ -19,7 +19,6 @@ package aws
 import (
 	"context"
 	"encoding/json"
-	"os"
 	"os/exec"
 
 	corev1 "k8s.io/api/core/v1"
@@ -70,11 +69,11 @@ func CreateCredentialSecret(ctx context.Context, kc *kubeclient.KubeClient) {
 // PopulateHostedTemplateVars populates the environment variables required for
 // the AWS hosted CP template by querying the standalone CP cluster with the
 // given kubeclient.
-func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient) {
+func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) {
 	GinkgoHelper()
 
 	c := getAWSClusterClient(kc)
-	awsCluster, err := c.Get(ctx, os.Getenv(managedcluster.EnvVarManagedClusterName), metav1.GetOptions{})
+	awsCluster, err := c.Get(ctx, clusterName, metav1.GetOptions{})
 	Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster")
 
 	vpcID, found, err := unstructured.NestedString(awsCluster.Object, "spec", "network", "vpc", "id")