e2e test of managed service deployment

test/e2e/clusterdeployment/constants.go

@@ -24,6 +24,7 @@ const (
 	// debugging of test failures.
 	EnvVarNoCleanup             = "NO_CLEANUP"
 	EnvVarManagementClusterName = "MANAGEMENT_CLUSTER_NAME"
+
 	// AWS
 	EnvVarAWSAccessKeyID     = "AWS_ACCESS_KEY_ID"
 	EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY"

test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl

@@ -8,9 +8,10 @@ spec:
   credential: ${ADOPTED_CREDENTIAL}
   config: {}
   serviceSpec:
-    - template: kyverno-3-2-6
-      name: kyverno
-      namespace: kyverno
-    - template: ingress-nginx-4-11-0
-      name: ingress-nginx
-      namespace: ingress-nginx
+    services:
+      - template: kyverno-3-2-6
+        name: kyverno
+        namespace: kyverno
+      - template: ingress-nginx-4-11-0
+        name: ingress-nginx
+        namespace: ingress-nginx

test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl

@@ -18,4 +18,4 @@ spec:
       rootVolumeSize: 30
     worker:
       instanceType: ${AWS_INSTANCE_TYPE:=t3.small}
-      rootVolumeSize: 30
+      rootVolumeSize: 30

test/e2e/clusterdeployment/servicevalidator.go

@@ -0,0 +1,82 @@
+// Copyright 2024
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clusterdeployment
+
+import (
+	"context"
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+
+	"github.com/K0rdent/kcm/test/e2e/kubeclient"
+)
+
+type ManagedServiceResource struct {
+	// ResourceNameSuffix is the suffix added to the resource name
+	ResourceNameSuffix string
+
+	// ValidationFunc is the validation function for the resource
+	ValidationFunc resourceValidationFunc
+}
+
+func (m ManagedServiceResource) fullName(serviceName string) string {
+	if m.ResourceNameSuffix == "" {
+		return serviceName
+	}
+	return fmt.Sprintf("%s-%s", serviceName, m.ResourceNameSuffix)
+}
+
+type ServiceValidator struct {
+	// clusterDeploymentName is the name of managed cluster
+	clusterDeploymentName string
+
+	// managedServiceName is the name of managed service
+	managedServiceName string
+
+	// namespace is a namespace of deployed service
+	namespace string
+
+	// resourcesToValidate is a map of resource names and corresponding resources definitions
+	resourcesToValidate map[string]ManagedServiceResource
+}
+
+func NewServiceValidator(clusterName, serviceName, namespace string) *ServiceValidator {
+	return &ServiceValidator{
+		clusterDeploymentName: clusterName,
+		managedServiceName:    serviceName,
+		namespace:             namespace,
+		resourcesToValidate:   make(map[string]ManagedServiceResource),
+	}
+}
+
+func (v *ServiceValidator) WithResourceValidation(resourceName string, resource ManagedServiceResource) *ServiceValidator {
+	v.resourcesToValidate[resourceName] = resource
+	return v
+}
+
+func (v *ServiceValidator) Validate(ctx context.Context, kc *kubeclient.KubeClient) error {
+	clusterKubeClient := kc.NewFromCluster(ctx, v.namespace, v.clusterDeploymentName)
+
+	for resourceName, resource := range v.resourcesToValidate {
+		resourceFullName := resource.fullName(v.managedServiceName)
+		err := resource.ValidationFunc(ctx, clusterKubeClient, resourceFullName)
+		if err != nil {
+			_, _ = fmt.Fprintf(GinkgoWriter, "[%s/%s] validation error: %v\n", resourceName, resourceFullName, err)
+			return err
+		}
+		_, _ = fmt.Fprintf(GinkgoWriter, "[%s/%s] validation succeeded\n", resourceName, resourceFullName)
+	}
+	return nil
+}

test/e2e/clusterdeployment/validate_deployed.go

@@ -18,9 +18,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -306,3 +308,48 @@ func validateSveltosCluster(ctx context.Context, kc *kubeclient.KubeClient, clus
 
 	return nil
 }
+
+func ValidateService(ctx context.Context, kc *kubeclient.KubeClient, name string) error {
+	_, err := kc.Client.CoreV1().Services(kc.Namespace).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func ValidateDeployment(ctx context.Context, kc *kubeclient.KubeClient, name string) error {
+	dep, err := kc.Client.AppsV1().Deployments(kc.Namespace).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if dep.Generation > dep.Status.ObservedGeneration {
+		return fmt.Errorf("deployment %s has observed generation equals to %d, expected %d", name, dep.Status.ObservedGeneration, dep.Generation)
+	}
+
+	const timedOutReason = "ProgressDeadlineExceeded" // avoid dependency
+
+	if slices.ContainsFunc(dep.Status.Conditions, func(c appsv1.DeploymentCondition) bool {
+		return c.Type == appsv1.DeploymentProgressing && c.Reason == timedOutReason
+	}) {
+		return fmt.Errorf("deployment %s has timed out", name)
+	}
+
+	if dep.Spec.Replicas != nil && dep.Status.UpdatedReplicas < *dep.Spec.Replicas {
+		return fmt.Errorf("deployment %s has %d updated replicas, desired replicas %d", name, dep.Status.UpdatedReplicas, *dep.Spec.Replicas)
+	}
+
+	if dep.Status.Replicas > dep.Status.UpdatedReplicas {
+		return fmt.Errorf("deployment %s has %d updated replicas, expected %d", name, dep.Status.UpdatedReplicas, dep.Status.Replicas)
+	}
+
+	if dep.Status.AvailableReplicas < dep.Status.UpdatedReplicas {
+		return fmt.Errorf("deployment %s has %d available replicas, expected %d", name, dep.Status.AvailableReplicas, dep.Status.UpdatedReplicas)
+	}
+
+	if *dep.Spec.Replicas != dep.Status.ReadyReplicas {
+		return fmt.Errorf("deployment %s has %d ready replicas, desired replicas %d", name, dep.Status.ReadyReplicas, *dep.Spec.Replicas)
+	}
+
+	return nil
+}

test/e2e/kubeclient/kubeclient.go

@@ -370,3 +370,32 @@ func (kc *KubeClient) GetSveltosCluster(ctx context.Context, name string) (*unst
 
 	return sveltosCluster, nil
 }
+
+func (kc *KubeClient) CreateMultiClusterService(
+	ctx context.Context,
+	multiClusterService *unstructured.Unstructured,
+) func() error {
+	GinkgoHelper()
+
+	kind := multiClusterService.GetKind()
+	Expect(kind).To(Equal("MultiClusterService"))
+
+	client := kc.GetDynamicClient(schema.GroupVersionResource{
+		Group:    "k0rdent.mirantis.com",
+		Version:  "v1alpha1",
+		Resource: "multiclusterservices",
+	}, false)
+
+	_, err := client.Create(ctx, multiClusterService, metav1.CreateOptions{})
+	if !apierrors.IsAlreadyExists(err) {
+		Expect(err).NotTo(HaveOccurred(), "failed to create %s", kind)
+	}
+
+	return func() error {
+		err := client.Delete(ctx, multiClusterService.GetName(), metav1.DeleteOptions{})
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+}