Skip to content

Commit

Permalink
Set CPU limit and request for Autoscaler Buffer when deploying it (#592)
Browse files Browse the repository at this point in the history
alexeykazakov authored Aug 9, 2024
1 parent 7835285 commit a0cdbff
Showing 6 changed files with 77 additions and 28 deletions.
Original file line number Diff line number Diff line change
@@ -71,7 +71,7 @@ func (r *Reconciler) handleAutoscalerDeploy(ctx context.Context, cfg membercfg.C
logger := log.FromContext(ctx)
if cfg.Autoscaler().Deploy() {
logger.Info("(Re)Deploying autoscaling buffer")
if err := autoscaler.Deploy(ctx, r.Client, r.Client.Scheme(), namespace, cfg.Autoscaler().BufferMemory(), cfg.Autoscaler().BufferReplicas()); err != nil {
if err := autoscaler.Deploy(ctx, r.Client, r.Client.Scheme(), namespace, cfg.Autoscaler()); err != nil {
return err
}
logger.Info("(Re)Deployed autoscaling buffer")
4 changes: 4 additions & 0 deletions deploy/autoscaler/member-operator-autoscaler.yaml
Original file line number Diff line number Diff line change
@@ -39,12 +39,16 @@ objects:
resources:
requests:
memory: ${MEMORY}
cpu: ${CPU}
limits:
memory: ${MEMORY}
cpu: ${CPU}
parameters:
- name: NAMESPACE
value: 'toolchain-member-operator'
- name: MEMORY
required: true
- name: CPU
required: true
- name: REPLICAS
required: true
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
module github.com/codeready-toolchain/member-operator

require (
github.com/codeready-toolchain/api v0.0.0-20240801160201-ed4387f19137
github.com/codeready-toolchain/toolchain-common v0.0.0-20240801160555-2f18a82d9752
github.com/codeready-toolchain/api v0.0.0-20240802163003-cce070815e69
github.com/codeready-toolchain/toolchain-common v0.0.0-20240802180627-14c41a99df18
github.com/go-logr/logr v1.2.3
github.com/google/go-cmp v0.5.9
// using latest commit from 'github.com/openshift/api branch release-4.12'
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
@@ -134,10 +134,10 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH
github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/codeready-toolchain/api v0.0.0-20240801160201-ed4387f19137 h1:axAZH/Ku+QHu2eQjlx/a7dctWRgj0T3K4gfnq1fzKFo=
github.com/codeready-toolchain/api v0.0.0-20240801160201-ed4387f19137/go.mod h1:ie9p4LenCCS0LsnbWp6/xwpFDdCWYE0KWzUO6Sk1g0E=
github.com/codeready-toolchain/toolchain-common v0.0.0-20240801160555-2f18a82d9752 h1:6cFp7ql7CTdk83GktAbVLYZDHRC63gei27uIFSbq8ho=
github.com/codeready-toolchain/toolchain-common v0.0.0-20240801160555-2f18a82d9752/go.mod h1:VIJaZyLIblt9/HqIsJn7Hlu4RqG9/hNCznE0Uge4H+g=
github.com/codeready-toolchain/api v0.0.0-20240802163003-cce070815e69 h1:e6up2k4O7QdG6hu0iOwNGap60b932N0JXlweDmmvCAQ=
github.com/codeready-toolchain/api v0.0.0-20240802163003-cce070815e69/go.mod h1:ie9p4LenCCS0LsnbWp6/xwpFDdCWYE0KWzUO6Sk1g0E=
github.com/codeready-toolchain/toolchain-common v0.0.0-20240802180627-14c41a99df18 h1:ZSXXcz3BmYvYmw1BexF4xvRjlr/3W5IOKxvc1NwLjKg=
github.com/codeready-toolchain/toolchain-common v0.0.0-20240802180627-14c41a99df18/go.mod h1:ulg9vY3W6Lsrn84/qVneCIrTvMkBC1mzmQqGfYDv7lQ=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
35 changes: 27 additions & 8 deletions pkg/autoscaler/autoscaler.go
Original file line number Diff line number Diff line change
@@ -3,10 +3,8 @@ package autoscaler
import (
"context"
"fmt"

applycl "github.com/codeready-toolchain/toolchain-common/pkg/client"
"github.com/codeready-toolchain/toolchain-common/pkg/template"

tmplv1 "github.com/openshift/api/template/v1"
errs "github.com/pkg/errors"
"k8s.io/apimachinery/pkg/api/errors"
@@ -15,28 +13,42 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)

func Deploy(ctx context.Context, cl client.Client, s *runtime.Scheme, namespace, requestsMemory string, replicas int) error {
objs, err := getTemplateObjects(s, namespace, requestsMemory, replicas)
type BufferConfiguration interface {
BufferMemory() string
BufferCPU() string
BufferReplicas() int
}

func Deploy(ctx context.Context, cl client.Client, s *runtime.Scheme, namespace string, config BufferConfiguration) error {
objs, err := getTemplateObjects(s, namespace, config)
if err != nil {
return err
}
logger := log.FromContext(ctx)

applyClient := applycl.NewApplyClient(cl)
// create all objects that are within the template, and update only when the object has changed.
for _, obj := range objs {
if _, err := applyClient.ApplyObject(ctx, obj); err != nil {
applied, err := applyClient.ApplyObject(ctx, obj)
if err != nil {
return errs.Wrap(err, "cannot deploy autoscaling buffer template")
}
if applied {
logger.Info("Autoscaling Buffer object created or updated", "kind", obj.GetObjectKind(), "namespace", obj.GetNamespace(), "name", obj.GetName())
} else {
logger.Info("Autoscaling Buffer object has not changed", "kind", obj.GetObjectKind(), "namespace", obj.GetNamespace(), "name", obj.GetName())
}
}
return nil
}

// Delete deletes the autoscaling buffer app if it's deployed. Does nothing if it's not.
// Returns true if the app was deleted.
func Delete(ctx context.Context, cl client.Client, s *runtime.Scheme, namespace string) (bool, error) {
objs, err := getTemplateObjects(s, namespace, "0", 0)
objs, err := getTemplateObjects(s, namespace, nil)
if err != nil {
return false, err
}
@@ -60,7 +72,7 @@ func Delete(ctx context.Context, cl client.Client, s *runtime.Scheme, namespace
return deleted, nil
}

func getTemplateObjects(s *runtime.Scheme, namespace, requestsMemory string, replicas int) ([]client.Object, error) {
func getTemplateObjects(s *runtime.Scheme, namespace string, config BufferConfiguration) ([]client.Object, error) {
deployment, err := Asset("member-operator-autoscaler.yaml")
if err != nil {
return nil, err
@@ -71,9 +83,16 @@ func getTemplateObjects(s *runtime.Scheme, namespace, requestsMemory string, rep
return nil, err
}

memory, cpu, replicas := "0", "0", 0
if config != nil {
memory = config.BufferMemory()
cpu = config.BufferCPU()
replicas = config.BufferReplicas()
}
return template.NewProcessor(s).Process(deploymentTemplate, map[string]string{
"NAMESPACE": namespace,
"MEMORY": requestsMemory,
"MEMORY": memory,
"CPU": cpu,
"REPLICAS": fmt.Sprintf("%d", replicas),
})
}
52 changes: 39 additions & 13 deletions pkg/autoscaler/autoscaler_test.go
Original file line number Diff line number Diff line change
@@ -26,13 +26,13 @@ func TestGetTemplateObjects(t *testing.T) {
s := setScheme(t)

// when
objs, err := getTemplateObjects(s, test.MemberOperatorNs, "8Gi", 3)
objs, err := getTemplateObjects(s, test.MemberOperatorNs, bufferConfiguration("8Gi", "2000m", 3))

// then
require.NoError(t, err)
require.Len(t, objs, 2)
priorityClassEquals(t, priorityClass(), objs[0])
deploymentEquals(t, deployment("8Gi", 3), objs[1])
deploymentEquals(t, deployment("8Gi", "2000m", 3), objs[1])
}

func TestDeploy(t *testing.T) {
@@ -44,11 +44,11 @@ func TestDeploy(t *testing.T) {
fakeClient := test.NewFakeClient(t)

// when
err := Deploy(context.TODO(), fakeClient, s, test.MemberOperatorNs, "500Mi", 10)
err := Deploy(context.TODO(), fakeClient, s, test.MemberOperatorNs, bufferConfiguration("500Mi", "3", 10))

// then
require.NoError(t, err)
verifyAutoscalerDeployment(t, fakeClient, "500Mi", 10)
verifyAutoscalerDeployment(t, fakeClient, "500Mi", "3", 10)
})

t.Run("when updated", func(t *testing.T) {
@@ -57,17 +57,17 @@ func TestDeploy(t *testing.T) {
priorityClass.Labels = map[string]string{}
priorityClass.Value = 100

deployment := unmarshalDeployment(t, deployment("1Gi", 5))
deployment := unmarshalDeployment(t, deployment("1Gi", "1", 5))
deployment.Spec.Template.Spec.Containers[0].Image = "some-dummy"

fakeClient := test.NewFakeClient(t, priorityClass, deployment)

// when
err := Deploy(context.TODO(), fakeClient, s, test.MemberOperatorNs, "7Gi", 1)
err := Deploy(context.TODO(), fakeClient, s, test.MemberOperatorNs, bufferConfiguration("7Gi", "100m", 1))

// then
require.NoError(t, err)
verifyAutoscalerDeployment(t, fakeClient, "7Gi", 1)
verifyAutoscalerDeployment(t, fakeClient, "7Gi", "100m", 1)
})

t.Run("when creation fails", func(t *testing.T) {
@@ -78,7 +78,7 @@ func TestDeploy(t *testing.T) {
}

// when
err := Deploy(context.TODO(), fakeClient, s, test.MemberOperatorNs, "7Gi", 3)
err := Deploy(context.TODO(), fakeClient, s, test.MemberOperatorNs, bufferConfiguration("7Gi", "1", 3))

// then
require.EqualError(t, err, "cannot deploy autoscaling buffer template: unable to create resource of kind: PriorityClass, version: v1: some error")
@@ -89,7 +89,7 @@ func TestDelete(t *testing.T) {
// given
s := setScheme(t)
prioClass := unmarshalPriorityClass(t, priorityClass())
dm := unmarshalDeployment(t, deployment("100Mi", 3))
dm := unmarshalDeployment(t, deployment("100Mi", "100m", 3))
namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: test.MemberOperatorNs}}

t.Run("when previously deployed", func(t *testing.T) {
@@ -151,7 +151,7 @@ func TestDelete(t *testing.T) {
})
}

func verifyAutoscalerDeployment(t *testing.T, fakeClient *test.FakeClient, memory string, replicas int) {
func verifyAutoscalerDeployment(t *testing.T, fakeClient *test.FakeClient, memory, cpu string, replicas int) {
expPrioClass := unmarshalPriorityClass(t, priorityClass())
actualPrioClass := &schedulingv1.PriorityClass{}
AssertObject(t, fakeClient, "", "member-operator-autoscaling-buffer", actualPrioClass, func() {
@@ -161,7 +161,7 @@ func verifyAutoscalerDeployment(t *testing.T, fakeClient *test.FakeClient, memor
assert.Equal(t, expPrioClass.Description, actualPrioClass.Description)
})

expDeployment := unmarshalDeployment(t, deployment(memory, replicas))
expDeployment := unmarshalDeployment(t, deployment(memory, cpu, replicas))
actualDeployment := &appsv1.Deployment{}
AssertObject(t, fakeClient, test.MemberOperatorNs, "autoscaling-buffer", actualDeployment, func() {
assert.Equal(t, expDeployment.Labels, actualDeployment.Labels)
@@ -224,6 +224,32 @@ func priorityClass() string {
return `{"apiVersion":"scheduling.k8s.io/v1","kind":"PriorityClass","metadata":{"name":"member-operator-autoscaling-buffer","labels":{"toolchain.dev.openshift.com/provider":"codeready-toolchain"}},"value":-5,"globalDefault":false,"description":"This priority class is to be used by the autoscaling buffer pod only"}`
}

func deployment(memory string, replicas int) string {
return fmt.Sprintf(`{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"name":"autoscaling-buffer","namespace":"%s","labels":{"app":"autoscaling-buffer","toolchain.dev.openshift.com/provider":"codeready-toolchain"}},"spec":{"replicas":%d,"selector":{"matchLabels":{"app":"autoscaling-buffer"}},"template":{"metadata":{"labels":{"app":"autoscaling-buffer"}},"spec":{"priorityClassName":"member-operator-autoscaling-buffer","terminationGracePeriodSeconds":0,"containers":[{"name":"autoscaling-buffer","image":"gcr.io/google_containers/pause-amd64:3.2","imagePullPolicy":"IfNotPresent","resources":{"requests":{"memory":"%s"},"limits":{"memory":"%s"}}}]}}}}`, test.MemberOperatorNs, replicas, memory, memory)
func deployment(memory, cpu string, replicas int) string {
return fmt.Sprintf(`{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"name":"autoscaling-buffer","namespace":"%[1]s","labels":{"app":"autoscaling-buffer","toolchain.dev.openshift.com/provider":"codeready-toolchain"}},"spec":{"replicas":%[2]d,"selector":{"matchLabels":{"app":"autoscaling-buffer"}},"template":{"metadata":{"labels":{"app":"autoscaling-buffer"}},"spec":{"priorityClassName":"member-operator-autoscaling-buffer","terminationGracePeriodSeconds":0,"containers":[{"name":"autoscaling-buffer","image":"gcr.io/google_containers/pause-amd64:3.2","imagePullPolicy":"IfNotPresent","resources":{"requests":{"memory":"%[3]s","cpu":"%[4]s"},"limits":{"memory":"%[3]s","cpu":"%[4]s"}}}]}}}}`, test.MemberOperatorNs, replicas, memory, cpu)
}

func bufferConfiguration(memory, cpu string, replicas int) BufferConfiguration {
return &bufferConfig{
memory: memory,
cpu: cpu,
replicas: replicas,
}
}

type bufferConfig struct {
memory string
cpu string
replicas int
}

func (b *bufferConfig) BufferMemory() string {
return b.memory
}

func (b *bufferConfig) BufferCPU() string {
return b.cpu
}

func (b *bufferConfig) BufferReplicas() int {
return b.replicas
}

0 comments on commit a0cdbff

Please sign in to comment.