Add HA and deployment override configuration

Add HA configuration replicas for tektonPipeline. Add deploymet override configuration for tektonPipeline components for example resource env and args override. Signed-off-by: yuzhipeng <[email protected]>
yuzp1996 · Feb 19, 2023 · ee0a35a · ee0a35a
1 parent f96d242
commit ee0a35a
Show file tree

Hide file tree

Showing 4 changed files with 295 additions and 0 deletions.
diff --git a/pkg/apis/operator/v1alpha1/tektonconfig_types.go b/pkg/apis/operator/v1alpha1/tektonconfig_types.go
@@ -156,6 +156,49 @@ type Config struct {
 	// PriorityClassName holds the priority class to be set to pod template
 	// +optional
 	PriorityClassName string `json:"priorityClassName,omitempty"`
+	// HighAvailability allows specification of HA control plane.
+	// +optional
+	HighAvailability HighAvailability `json:"highAvailability,omitempty"`
+	// DeploymentOverride overrides Deployment configurations such as resource.
+	// +optional
+	DeploymentOverride []DeploymentOverride `json:"deployments,omitempty"`
+}
+
+// HighAvailability specifies options for deploying Tekton Pipeline Deployment
+type HighAvailability struct {
+	// Replicas is the number of replicas that HA parts of the control plane
+	// will be scaled to.
+	// +optional
+	Replicas *int32 `json:"replicas,omitempty"`
+}
+
+// DeploymentOverride specifies resource override for deployment
+type DeploymentOverride struct {
+	// Name is the name of the deployment to override.
+	Name string `json:"name"`
+	// Resources overrides resources for the containers.
+	// +optional
+	Containers []ContainerOverride `json:"containers,omitempty"`
+	// Replicas is the number of replicas that this deployment will scaled to.
+	// It has a higher priority than HighAvailability.Replicas
+	// +optional
+	Replicas *int32 `json:"replicas,omitempty"`
+}
+
+// ContainerOverride enables the user to override any container's
+// configuration specified in the embedded manifest
+type ContainerOverride struct {
+	// Name represent container name
+	Name string `json:"name"`
+	// Resource represent the desired ResourceRequirements
+	// +optional
+	Resource corev1.ResourceRequirements `json:"resource,omitempty"`
+	// Env represent the env that will replace the existing one or append if not existed
+	// +optional
+	Env []corev1.EnvVar `json:"env,omitempty"`
+	// Args represent the args will append to the existing args
+	// +optional
+	Args []string `json:"args,omitempty"`
 }
 
 type Platforms struct {

diff --git a/pkg/reconciler/common/transformers.go b/pkg/reconciler/common/transformers.go
@@ -29,6 +29,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"knative.dev/pkg/logging"
@@ -598,6 +599,150 @@ func AddConfiguration(config v1alpha1.Config) mf.Transformer {
 	}
 }
 
+// HighAvailabilityTransform mutates
+func HighAvailabilityTransform(ha v1alpha1.HighAvailability) mf.Transformer {
+	return func(u *unstructured.Unstructured) error {
+		if ha.Replicas == nil {
+			return nil
+		}
+		replicas := int64(*ha.Replicas)
+
+		// Transform deployments that support HA.
+		if u.GetKind() == "Deployment" {
+			if err := unstructured.SetNestedField(u.Object, replicas, "spec", "replicas"); err != nil {
+				return err
+			}
+		}
+
+		if u.GetKind() == "HorizontalPodAutoscaler" {
+			min, _, err := unstructured.NestedInt64(u.Object, "spec", "minReplicas")
+			if err != nil {
+				return err
+			}
+			// Do nothing if the HPA ships with even more replicas out of the box.
+			if min >= replicas {
+				return nil
+			}
+
+			if err := unstructured.SetNestedField(u.Object, replicas, "spec", "minReplicas"); err != nil {
+				return err
+			}
+
+			max, found, err := unstructured.NestedInt64(u.Object, "spec", "maxReplicas")
+			if err != nil {
+				return err
+			}
+
+			// Do nothing if maxReplicas is not defined.
+			if !found {
+				return nil
+			}
+
+			// Increase maxReplicas to the amount that we increased,
+			// because we need to avoid minReplicas > maxReplicas happenning.
+			if err := unstructured.SetNestedField(u.Object, max+(replicas-min), "spec", "maxReplicas"); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+}
+
+// DeploymentOverrideTransform configures the resource requests for
+// all containers within all deployments in the manifest
+func DeploymentOverrideTransform(deploymentOverRides []v1alpha1.DeploymentOverride) mf.Transformer {
+	return func(u *unstructured.Unstructured) error {
+		if u.GetKind() != "Deployment" {
+			return nil
+		}
+
+		var deploymentOverRide v1alpha1.DeploymentOverride
+		for _, deployment := range deploymentOverRides {
+			if deployment.Name == u.GetName() {
+				deploymentOverRide = deployment
+				break
+			}
+		}
+		if deploymentOverRide.Name == "" {
+			return nil
+		}
+
+		d := &appsv1.Deployment{}
+
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, d); err != nil {
+			return err
+		}
+		containers := d.Spec.Template.Spec.Containers
+		for i := range containers {
+			if override := find(deploymentOverRide.Containers, containers[i].Name); override != nil {
+				merge(&override.Resource.Limits, &containers[i].Resources.Limits)
+				merge(&override.Resource.Requests, &containers[i].Resources.Requests)
+
+				if len(override.Args) > 0 {
+					containers[i].Args = append(containers[i].Args, override.Args...)
+				}
+
+				if len(override.Env) > 0 {
+					containers[i].Env = upsertEnv(containers[i].Env, override.Env)
+				}
+			}
+		}
+		if deploymentOverRide.Replicas != nil {
+			d.Spec.Replicas = deploymentOverRide.Replicas
+		}
+
+		// Avoid superfluous updates from converted zero defaults
+		d.SetCreationTimestamp(metav1.Time{})
+
+		unstrObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(d)
+		if err != nil {
+			return err
+		}
+		u.SetUnstructuredContent(unstrObj)
+		return nil
+	}
+}
+
+func merge(src, tgt *corev1.ResourceList) {
+	if src == nil || tgt == nil {
+		return
+	}
+	if len(*tgt) > 0 {
+		for k, v := range *src {
+			(*tgt)[k] = v
+		}
+	} else {
+		*tgt = *src
+	}
+}
+
+func find(resources []v1alpha1.ContainerOverride, name string) *v1alpha1.ContainerOverride {
+	for _, override := range resources {
+		if override.Name == name {
+			return &override
+		}
+	}
+	return nil
+}
+
+func upsertEnv(exists, overrides []corev1.EnvVar) []corev1.EnvVar {
+	for _, override := range overrides {
+		var found bool
+		for i, exist := range exists {
+			if override.Name == exist.Name {
+				exists[i] = override
+				found = true
+				break
+			}
+		}
+		if !found {
+			exists = append(exists, override)
+		}
+	}
+	return exists
+}
+
 // AddDeploymentRestrictedPSA will add the default restricted spec on Deployment to remove errors/warning
 func AddDeploymentRestrictedPSA() mf.Transformer {
 	return func(u *unstructured.Unstructured) error {

diff --git a/pkg/reconciler/common/transformers_test.go b/pkg/reconciler/common/transformers_test.go
@@ -29,6 +29,8 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/api/apps/v1beta1"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -514,6 +516,109 @@ func TestAddConfiguration(t *testing.T) {
 	assert.Equal(t, d.Spec.Template.Spec.PriorityClassName, config.PriorityClassName)
 }
 
+func TestHighAvailabilityDeploymentResourceTransform(t *testing.T) {
+
+	testData := path.Join("testdata", "test-add-configurations.yaml")
+	manifest, err := mf.ManifestFrom(mf.Recursive(testData))
+	assertNoEror(t, err)
+
+	replicasNum := int32(3)
+	containerOverride := []v1alpha1.ContainerOverride{
+		{
+			Name: "controller-deployment",
+			Env: []corev1.EnvVar{
+				{
+					Name:  "KUBERNETES_MIN_VERSION",
+					Value: "v1.23.0",
+				},
+			},
+			Args: []string{
+				"-kube-api-qps", "50",
+			},
+			Resource: corev1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceCPU:    resource.MustParse("1"),
+					v1.ResourceMemory: resource.MustParse("2"),
+				},
+				Limits: v1.ResourceList{
+					v1.ResourceCPU:    resource.MustParse("2"),
+					v1.ResourceMemory: resource.MustParse("4"),
+				},
+			},
+		},
+	}
+
+	config := v1alpha1.Config{
+		HighAvailability: v1alpha1.HighAvailability{
+			Replicas: &replicasNum,
+		},
+		DeploymentOverride: []v1alpha1.DeploymentOverride{
+			{
+				Name:       "controller",
+				Containers: containerOverride,
+			},
+		},
+	}
+
+	manifest, err = manifest.Transform(HighAvailabilityTransform(config.HighAvailability))
+	assertNoEror(t, err)
+	manifest, err = manifest.Transform(DeploymentOverrideTransform(config.DeploymentOverride))
+	assertNoEror(t, err)
+
+	d := &v1beta1.Deployment{}
+	err = runtime.DefaultUnstructuredConverter.FromUnstructured(manifest.Resources()[0].Object, d)
+	assertNoEror(t, err)
+
+	assert.Equal(t, *d.Spec.Replicas, int32(replicasNum))
+	assert.Equal(t, d.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU], resource.MustParse("1"))
+	assert.Equal(t, d.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU], resource.MustParse("2"))
+	assert.Equal(t, d.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory], resource.MustParse("2"))
+	assert.Equal(t, d.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory], resource.MustParse("4"))
+
+	assert.Equal(t, len(d.Spec.Template.Spec.Containers[0].Args), 5)
+	assert.Equal(t, d.Spec.Template.Spec.Containers[0].Args[3], "-kube-api-qps")
+	assert.Equal(t, d.Spec.Template.Spec.Containers[0].Args[4], "50")
+
+	assert.Equal(t, d.Spec.Template.Spec.Containers[0].Env[0], corev1.EnvVar{
+		Name:  "KUBERNETES_MIN_VERSION",
+		Value: "v1.23.0",
+	})
+}
+
+func TestDeploymentReplicasOverrideTransform(t *testing.T) {
+
+	testData := path.Join("testdata", "test-add-configurations.yaml")
+	manifest, err := mf.ManifestFrom(mf.Recursive(testData))
+	assertNoEror(t, err)
+
+	HAreplicas := int32(3)
+	overReplicas := int32(2)
+
+	config := v1alpha1.Config{
+		HighAvailability: v1alpha1.HighAvailability{
+			Replicas: &HAreplicas,
+		},
+		DeploymentOverride: []v1alpha1.DeploymentOverride{
+			{
+				Name:     "controller",
+				Replicas: &overReplicas,
+			},
+		},
+	}
+
+	manifest, err = manifest.Transform(HighAvailabilityTransform(config.HighAvailability))
+	assertNoEror(t, err)
+	manifest, err = manifest.Transform(DeploymentOverrideTransform(config.DeploymentOverride))
+	assertNoEror(t, err)
+
+	d := &v1beta1.Deployment{}
+	err = runtime.DefaultUnstructuredConverter.FromUnstructured(manifest.Resources()[0].Object, d)
+	assertNoEror(t, err)
+
+	assert.Equal(t, *d.Spec.Replicas, overReplicas)
+
+}
+
 func TestAddPSA(t *testing.T) {
 	testData := path.Join("testdata", "test-add-psa.yaml")
 	manifest, err := mf.ManifestFrom(mf.Recursive(testData))

diff --git a/pkg/reconciler/kubernetes/tektonpipeline/transform.go b/pkg/reconciler/kubernetes/tektonpipeline/transform.go
@@ -57,6 +57,8 @@ func filterAndTransform(extension common.Extension) client.FilterAndTransform {
 			common.DeploymentImages(images),
 			common.InjectLabelOnNamespace(proxyLabel),
 			common.AddConfiguration(pipeline.Spec.Config),
+			common.HighAvailabilityTransform(pipeline.Spec.Config.HighAvailability),
+			common.DeploymentOverrideTransform(pipeline.Spec.Config.DeploymentOverride),
 			common.CopyConfigMap(bundleResolverConfig, pipeline.Spec.BundlesResolverConfig),
 			common.CopyConfigMap(hubResolverConfig, pipeline.Spec.HubResolverConfig),
 			common.CopyConfigMap(clusterResolverConfig, pipeline.Spec.ClusterResolverConfig),