From ba083405a3ef08df5db29c60d54e338436821735 Mon Sep 17 00:00:00 2001 From: mengyipeng <413183498@qq.com> Date: Mon, 4 Nov 2024 15:18:59 +0800 Subject: [PATCH 1/4] cluster create support specify schedulingPolicy --- pkg/cluster/cluster_chart.go | 3 ++ pkg/cmd/cluster/cluster_test.go | 22 +++++++++ pkg/cmd/cluster/create_subcmds.go | 79 +++++++++++++++++++++++++++---- pkg/cmd/cluster/register_test.go | 14 +++--- pkg/util/util.go | 12 +++++ 5 files changed, 113 insertions(+), 17 deletions(-) diff --git a/pkg/cluster/cluster_chart.go b/pkg/cluster/cluster_chart.go index 5ac020571..03128a956 100644 --- a/pkg/cluster/cluster_chart.go +++ b/pkg/cluster/cluster_chart.go @@ -129,6 +129,9 @@ func (c *ChartInfo) BuildClusterSchema() error { var err error cht := c.Chart buildSchema := func(bs []byte) (*spec.Schema, error) { + if bs == nil { + return nil, nil + } schema := &spec.Schema{} if err = json.Unmarshal(bs, schema); err != nil { return nil, errors.Wrapf(err, "failed to build schema for engine %s", cht.Name()) diff --git a/pkg/cmd/cluster/cluster_test.go b/pkg/cmd/cluster/cluster_test.go index 45f4cb258..90ece248e 100644 --- a/pkg/cmd/cluster/cluster_test.go +++ b/pkg/cmd/cluster/cluster_test.go @@ -93,6 +93,28 @@ var _ = Describe("Cluster", func() { Expect(o.ChartInfo).ShouldNot(BeNil()) o.Format = printer.YAML + Expect(o.CreateOptions.Complete()).To(Succeed()) + o.Client = testing.FakeClientSet() + fakeDiscovery1, _ := o.Client.Discovery().(*fakediscovery.FakeDiscovery) + fakeDiscovery1.FakedServerVersion = &version.Info{Major: "1", Minor: "27", GitVersion: "v1.27.0"} + Expect(o.Complete(nil)).To(Succeed()) + Expect(o.Validate()).To(Succeed()) + Expect(o.Name).ShouldNot(BeEmpty()) + Expect(o.Run()).Should(Succeed()) + }) + It("with schedulingPolicy", func() { + o, err := NewSubCmdsOptions(createOptions, clusterType) + o.Tenancy = "SharedNode" + o.TopologyKeys = []string{"zone", "hostname"} + o.NodeLabels = map[string]string{"environment": "environment", "region": "region"} + o.TolerationsRaw = []string{"key=value:effect", " key:effect"} + o.PodAntiAffinity = "Preferred" + + Expect(err).Should(Succeed()) + Expect(o).ShouldNot(BeNil()) + Expect(o.ChartInfo).ShouldNot(BeNil()) + o.Format = printer.YAML + Expect(o.CreateOptions.Complete()).To(Succeed()) o.Client = testing.FakeClientSet() fakeDiscovery1, _ := o.Client.Discovery().(*fakediscovery.FakeDiscovery) diff --git a/pkg/cmd/cluster/create_subcmds.go b/pkg/cmd/cluster/create_subcmds.go index 423a7732c..ff290a2ca 100644 --- a/pkg/cmd/cluster/create_subcmds.go +++ b/pkg/cmd/cluster/create_subcmds.go @@ -21,16 +21,21 @@ package cluster import ( "context" + "encoding/json" "fmt" "os" "regexp" "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" cmdutil "k8s.io/kubectl/pkg/cmd/util" + v1 "github.com/apecloud/kubeblocks/apis/apps/v1" + "github.com/apecloud/kbcli/pkg/action" "github.com/apecloud/kbcli/pkg/cluster" "github.com/apecloud/kbcli/pkg/printer" @@ -57,10 +62,11 @@ type CreateSubCmdsOptions struct { // Configuration and options for cluster affinity and tolerations PodAntiAffinity string `json:"podAntiAffinity"` // TopologyKeys if TopologyKeys is nil, add omitempty json tag, because CueLang can not covert null to list. - TopologyKeys []string `json:"topologyKeys,omitempty"` - NodeLabels map[string]string `json:"nodeLabels,omitempty"` - Tenancy string `json:"tenancy"` - TolerationsRaw []string `json:"-"` + TopologyKeys []string `json:"topologyKeys,omitempty"` + NodeLabels map[string]string `json:"nodeLabels,omitempty"` + Tenancy string `json:"tenancy"` + TolerationsRaw []string `json:"-"` + schedulingPolicy *v1.SchedulingPolicy *action.CreateOptions } @@ -119,7 +125,6 @@ func buildCreateSubCmds(createOptions *action.CreateOptions) []*cobra.Command { util.CheckErr(addCreateFlags(cmd, o.Factory, o.ChartInfo, t.String())) // Schedule policy - // TODO: implement them, and check whether the flag has been defined cmd.Flags().StringVar(&o.PodAntiAffinity, "pod-anti-affinity", "Preferred", "Pod anti-affinity type, one of: (Preferred, Required)") cmd.Flags().StringArrayVar(&o.TopologyKeys, "topology-keys", nil, "Topology keys for affinity") cmd.Flags().StringToStringVar(&o.NodeLabels, "node-labels", nil, "Node label selector") @@ -192,6 +197,50 @@ func (o *CreateSubCmdsOptions) Complete(cmd *cobra.Command) error { if o.ChartInfo.ClusterDef == "" && len(o.ChartInfo.ComponentDef) == 0 { return fmt.Errorf("cannot find clusterDef in cluster spec or componentDef in componentSpecs or shardingSpecs") } + + // Define scheduling related variables + var ( + tolerations = make([]corev1.Toleration, 0) + NodeLabels = make(map[string]string) + podAntiAffinity = &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: make([]corev1.PodAffinityTerm, 0), + PreferredDuringSchedulingIgnoredDuringExecution: make([]corev1.WeightedPodAffinityTerm, 0), + } + ) + + hasSchedulingPolicy := false + // Build tolerations if raw toleration rules are configured + if o.TolerationsRaw != nil { + tolerationsResult, err := util.BuildTolerations(o.TolerationsRaw) + if err != nil { + return err + } + jsonData, err := json.Marshal(tolerationsResult) + if err != nil { + return err + } + err = json.Unmarshal(jsonData, &tolerations) + if err != nil { + return err + } + } + + // Set node labels if specified + if o.NodeLabels != nil { + hasSchedulingPolicy = true + NodeLabels = o.NodeLabels + } + + // Build pod anti-affinity if either anti-affinity rule or topology keys are specified + if o.PodAntiAffinity != "" || len(o.TopologyKeys) > 0 { + hasSchedulingPolicy = true + podAntiAffinity = util.BuildPodAntiAffinity(o.PodAntiAffinity, o.TopologyKeys) + } + + // Construct the final scheduling policy combining tolerations, node labels and pod anti-affinity + if hasSchedulingPolicy { + o.schedulingPolicy = util.BuildSchedulingPolicy(tolerations, NodeLabels, podAntiAffinity) + } return nil } @@ -222,12 +271,24 @@ func (o *CreateSubCmdsOptions) Run() error { return nil, fmt.Errorf("failed to find cluster object from manifests rendered from %s chart", o.ClusterType) } + clusterObj, err := getClusterObj() + if err != nil { + return err + } + + if clusterObj != nil { + if o.schedulingPolicy != nil { + converted, err := runtime.DefaultUnstructuredConverter.ToUnstructured(o.schedulingPolicy) + if err != nil { + return err + } + _ = unstructured.SetNestedField(clusterObj.Object, converted, "spec", "schedulingPolicy") + } + _ = unstructured.SetNestedField(clusterObj.Object, o.Tenancy, "spec", "tenancy") + } + // only edits the cluster object, other dependency objects are created directly if o.EditBeforeCreate { - clusterObj, err := getClusterObj() - if err != nil { - return err - } customEdit := action.NewCustomEditOptions(o.Factory, o.IOStreams, "create") if err = customEdit.Run(clusterObj); err != nil { return err diff --git a/pkg/cmd/cluster/register_test.go b/pkg/cmd/cluster/register_test.go index e623e0f1c..d3fdd51eb 100644 --- a/pkg/cmd/cluster/register_test.go +++ b/pkg/cmd/cluster/register_test.go @@ -91,16 +91,14 @@ var _ = Describe("cluster register", func() { Context("test register cluster chart", func() { var ( - source = "https://github.com/apecloud/helm-charts/releases/download/mysql-cluster-1.0.0-alpha.0/mysql-cluster-1.0.0-alpha.0.tgz" - engine = "mysql" - apecloudMysqlEngine = "apecloud-mysql" - version = "1.0.0-alpha.0" - repo = types.ClusterChartsRepoURL + source = "https://github.com/apecloud/helm-charts/releases/download/apecloud-mysql-cluster-1.0.0-alpha.0/apecloud-mysql-cluster-1.0.0-alpha.0.tgz" + engine = "apecloud-mysql" + version = "1.0.0-alpha.0" + repo = types.ClusterChartsRepoURL ) AfterEach(func() { cluster.ClearCharts(cluster.ClusterType(engine)) - cluster.ClearCharts(cluster.ClusterType(apecloudMysqlEngine)) }) It("test register chart by source", func() { @@ -112,9 +110,9 @@ var _ = Describe("cluster register", func() { }) It("test register built-in chart and test validate", func() { - Expect(RegisterClusterChart(tf, streams, "", apecloudMysqlEngine, version, repo)).Should(Succeed()) + Expect(RegisterClusterChart(tf, streams, "", engine, version, repo)).Should(Succeed()) validatedChart := &cluster.TypeInstance{ - Name: cluster.ClusterType(apecloudMysqlEngine), + Name: cluster.ClusterType(engine), URL: "", Alias: "", ChartName: filepath.Base(source), diff --git a/pkg/util/util.go b/pkg/util/util.go index 310fffe41..a4d2c9a66 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1083,6 +1083,18 @@ func DisplayDiffWithColor(out io.Writer, diffText string) { } } +func BuildSchedulingPolicy(tolerations []corev1.Toleration, nodeLabels map[string]string, podAntiAffinity *corev1.PodAntiAffinity) *kbappsv1.SchedulingPolicy { + schedulingPolicy := &kbappsv1.SchedulingPolicy{ + NodeSelector: nodeLabels, + Affinity: &corev1.Affinity{ + PodAntiAffinity: podAntiAffinity, + }, + Tolerations: tolerations, + } + + return schedulingPolicy +} + // BuildTolerations toleration format: key=value:effect or key:effect, func BuildTolerations(raw []string) ([]interface{}, error) { tolerations := make([]interface{}, 0) From e142b1e8d7065a4fda122565014f3a98d98fe683 Mon Sep 17 00:00:00 2001 From: mengyipeng <413183498@qq.com> Date: Thu, 7 Nov 2024 13:50:46 +0800 Subject: [PATCH 2/4] modify refer to buildPodTopologySpreadConstraints4Legacy in KB-release-0.9 --- pkg/cmd/cluster/create_subcmds.go | 30 +++-------------------------- pkg/util/util.go | 32 +++++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/pkg/cmd/cluster/create_subcmds.go b/pkg/cmd/cluster/create_subcmds.go index f2a81ad56..e5dd5c231 100644 --- a/pkg/cmd/cluster/create_subcmds.go +++ b/pkg/cmd/cluster/create_subcmds.go @@ -132,7 +132,7 @@ func buildCreateSubCmds(createOptions *action.CreateOptions) []*cobra.Command { cmd.Flags().StringVar(&o.PodAntiAffinity, "pod-anti-affinity", "Preferred", "Pod anti-affinity type, one of: (Preferred, Required)") cmd.Flags().StringArrayVar(&o.TopologyKeys, "topology-keys", nil, "Topology keys for affinity") cmd.Flags().StringToStringVar(&o.NodeLabels, "node-labels", nil, "Node label selector") - cmd.Flags().StringSliceVar(&o.TolerationsRaw, "tolerations", nil, `Tolerations for cluster, such as "key=value:effect, key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"'`) + cmd.Flags().StringSliceVar(&o.TolerationsRaw, "tolerations", nil, `Tolerations for cluster, such as "key=value:effect,key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"'`) if cmd.Flag("tenancy") == nil { cmd.Flags().StringVar(&o.Tenancy, "tenancy", "SharedNode", "Tenancy options, one of: (SharedNode, DedicatedNode)") } @@ -202,18 +202,8 @@ func (o *CreateSubCmdsOptions) Complete(cmd *cobra.Command) error { return fmt.Errorf("cannot find clusterDef in cluster spec or componentDef in componentSpecs or shardingSpecs") } - // Define scheduling related variables - var ( - tolerations = make([]corev1.Toleration, 0) - NodeLabels = make(map[string]string) - podAntiAffinity = &corev1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: make([]corev1.PodAffinityTerm, 0), - PreferredDuringSchedulingIgnoredDuringExecution: make([]corev1.WeightedPodAffinityTerm, 0), - } - ) - - hasSchedulingPolicy := false // Build tolerations if raw toleration rules are configured + tolerations := make([]corev1.Toleration, 0) if o.TolerationsRaw != nil { tolerationsResult, err := util.BuildTolerations(o.TolerationsRaw) if err != nil { @@ -229,22 +219,8 @@ func (o *CreateSubCmdsOptions) Complete(cmd *cobra.Command) error { } } - // Set node labels if specified - if o.NodeLabels != nil { - hasSchedulingPolicy = true - NodeLabels = o.NodeLabels - } - - // Build pod anti-affinity if either anti-affinity rule or topology keys are specified - if o.PodAntiAffinity != "" || len(o.TopologyKeys) > 0 { - hasSchedulingPolicy = true - podAntiAffinity = util.BuildPodAntiAffinity(o.PodAntiAffinity, o.TopologyKeys) - } + o.schedulingPolicy = util.BuildSchedulingPolicy(o.Name, tolerations, o.NodeLabels, o.PodAntiAffinity, o.TopologyKeys) - // Construct the final scheduling policy combining tolerations, node labels and pod anti-affinity - if hasSchedulingPolicy { - o.schedulingPolicy = util.BuildSchedulingPolicy(tolerations, NodeLabels, podAntiAffinity) - } return nil } diff --git a/pkg/util/util.go b/pkg/util/util.go index a4d2c9a66..1665d763c 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1083,13 +1083,33 @@ func DisplayDiffWithColor(out io.Writer, diffText string) { } } -func BuildSchedulingPolicy(tolerations []corev1.Toleration, nodeLabels map[string]string, podAntiAffinity *corev1.PodAntiAffinity) *kbappsv1.SchedulingPolicy { +func BuildSchedulingPolicy(clusterName string, tolerations []corev1.Toleration, nodeLabels map[string]string, podAntiAffinity string, topologyKeys []string) *kbappsv1.SchedulingPolicy { + + var topologySpreadConstraints []corev1.TopologySpreadConstraint + + var whenUnsatisfiable corev1.UnsatisfiableConstraintAction + if kbappsv1alpha1.PodAntiAffinity(podAntiAffinity) == kbappsv1alpha1.Required { + whenUnsatisfiable = corev1.DoNotSchedule + } else { + whenUnsatisfiable = corev1.ScheduleAnyway + } + for _, topologyKey := range topologyKeys { + topologySpreadConstraints = append(topologySpreadConstraints, corev1.TopologySpreadConstraint{ + MaxSkew: 1, + WhenUnsatisfiable: whenUnsatisfiable, + TopologyKey: topologyKey, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + constant.AppInstanceLabelKey: clusterName, + }, + }, + }) + } + schedulingPolicy := &kbappsv1.SchedulingPolicy{ - NodeSelector: nodeLabels, - Affinity: &corev1.Affinity{ - PodAntiAffinity: podAntiAffinity, - }, - Tolerations: tolerations, + NodeSelector: nodeLabels, + Tolerations: tolerations, + TopologySpreadConstraints: topologySpreadConstraints, } return schedulingPolicy From 891d17191ef79de8ad06314f9652915da357a0d2 Mon Sep 17 00:00:00 2001 From: yipeng1030 Date: Thu, 7 Nov 2024 05:53:48 +0000 Subject: [PATCH 3/4] chore: auto update cli doc changes --- docs/user_docs/cli/kbcli_cluster_create_apecloud-mysql.md | 2 +- docs/user_docs/cli/kbcli_cluster_create_elasticsearch.md | 2 +- docs/user_docs/cli/kbcli_cluster_create_kafka.md | 2 +- docs/user_docs/cli/kbcli_cluster_create_llm.md | 2 +- docs/user_docs/cli/kbcli_cluster_create_mongodb.md | 2 +- docs/user_docs/cli/kbcli_cluster_create_mysql.md | 2 +- docs/user_docs/cli/kbcli_cluster_create_postgresql.md | 2 +- docs/user_docs/cli/kbcli_cluster_create_qdrant.md | 2 +- docs/user_docs/cli/kbcli_cluster_create_redis.md | 2 +- docs/user_docs/cli/kbcli_cluster_create_xinference.md | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/user_docs/cli/kbcli_cluster_create_apecloud-mysql.md b/docs/user_docs/cli/kbcli_cluster_create_apecloud-mysql.md index 1b8b5aa8a..b8ca2c109 100644 --- a/docs/user_docs/cli/kbcli_cluster_create_apecloud-mysql.md +++ b/docs/user_docs/cli/kbcli_cluster_create_apecloud-mysql.md @@ -41,7 +41,7 @@ kbcli cluster create apecloud-mysql NAME [flags] --storage-class-name string Storage class name of the data volume --tenancy string The tenancy of cluster. Legal values [SharedNode, DedicatedNode]. (default "SharedNode") --termination-policy string The termination policy of cluster. Legal values [DoNotTerminate, Halt, Delete, WipeOut]. (default "Delete") - --tolerations strings Tolerations for cluster, such as "key=value:effect, key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' + --tolerations strings Tolerations for cluster, such as "key=value:effect,key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' --topology-keys stringArray Topology keys for affinity --version string Cluster version. (default "ac-mysql-8.0.30") ``` diff --git a/docs/user_docs/cli/kbcli_cluster_create_elasticsearch.md b/docs/user_docs/cli/kbcli_cluster_create_elasticsearch.md index aea986bac..32188fa57 100644 --- a/docs/user_docs/cli/kbcli_cluster_create_elasticsearch.md +++ b/docs/user_docs/cli/kbcli_cluster_create_elasticsearch.md @@ -39,7 +39,7 @@ kbcli cluster create elasticsearch NAME [flags] --storage float Storage size, the unit is Gi. Value range [1, 10000]. (default 20) --tenancy string The tenancy of cluster. Legal values [SharedNode, DedicatedNode]. (default "SharedNode") --termination-policy string The termination policy of cluster. Legal values [DoNotTerminate, Halt, Delete, WipeOut]. (default "Delete") - --tolerations strings Tolerations for cluster, such as "key=value:effect, key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' + --tolerations strings Tolerations for cluster, such as "key=value:effect,key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' --topology-keys stringArray Topology keys for affinity --version string The version of ElasticSearch. (default "8.8.2") ``` diff --git a/docs/user_docs/cli/kbcli_cluster_create_kafka.md b/docs/user_docs/cli/kbcli_cluster_create_kafka.md index 68a993ab7..8d7998ea0 100644 --- a/docs/user_docs/cli/kbcli_cluster_create_kafka.md +++ b/docs/user_docs/cli/kbcli_cluster_create_kafka.md @@ -51,7 +51,7 @@ kbcli cluster create kafka NAME [flags] --storage-enable Enable storage for Kafka. --tenancy string The tenancy of cluster. Legal values [SharedNode, DedicatedNode]. (default "SharedNode") --termination-policy string The termination policy of cluster. Legal values [DoNotTerminate, Halt, Delete, WipeOut]. (default "Delete") - --tolerations strings Tolerations for cluster, such as "key=value:effect, key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' + --tolerations strings Tolerations for cluster, such as "key=value:effect,key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' --topology-keys stringArray Topology keys for affinity --version string Cluster version. (default "kafka-3.3.2") ``` diff --git a/docs/user_docs/cli/kbcli_cluster_create_llm.md b/docs/user_docs/cli/kbcli_cluster_create_llm.md index 8ebd43237..b923b7e10 100644 --- a/docs/user_docs/cli/kbcli_cluster_create_llm.md +++ b/docs/user_docs/cli/kbcli_cluster_create_llm.md @@ -42,7 +42,7 @@ kbcli cluster create llm NAME [flags] --replicas int The number of replicas, for standalone mode, the replicas is 1, for replication mode, the default replicas is 2. Value range [1, 5]. (default 1) --tenancy string The tenancy of cluster. Legal values [SharedNode, DedicatedNode]. (default "SharedNode") --termination-policy string The termination policy of cluster. Legal values [DoNotTerminate, Halt, Delete, WipeOut]. (default "Delete") - --tolerations strings Tolerations for cluster, such as "key=value:effect, key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' + --tolerations strings Tolerations for cluster, such as "key=value:effect,key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' --topology-keys stringArray Topology keys for affinity --url string Model URL, only work for CPU mode --version string Cluster version. diff --git a/docs/user_docs/cli/kbcli_cluster_create_mongodb.md b/docs/user_docs/cli/kbcli_cluster_create_mongodb.md index 1413bf047..38e2d2bcc 100644 --- a/docs/user_docs/cli/kbcli_cluster_create_mongodb.md +++ b/docs/user_docs/cli/kbcli_cluster_create_mongodb.md @@ -40,7 +40,7 @@ kbcli cluster create mongodb NAME [flags] --storage-class-name string Storage class name of the data volume --tenancy string The tenancy of cluster. Legal values [SharedNode, DedicatedNode]. (default "SharedNode") --termination-policy string The termination policy of cluster. Legal values [DoNotTerminate, Halt, Delete, WipeOut]. (default "Delete") - --tolerations strings Tolerations for cluster, such as "key=value:effect, key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' + --tolerations strings Tolerations for cluster, such as "key=value:effect,key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' --topology-keys stringArray Topology keys for affinity --version string Cluster version. Legal values [7.0.12, 6.0.16, 5.0.28, 4.4.29, 4.2.24, 4.0.28]. (default "6.0.16") ``` diff --git a/docs/user_docs/cli/kbcli_cluster_create_mysql.md b/docs/user_docs/cli/kbcli_cluster_create_mysql.md index dc6c82ae2..34b5ad97f 100644 --- a/docs/user_docs/cli/kbcli_cluster_create_mysql.md +++ b/docs/user_docs/cli/kbcli_cluster_create_mysql.md @@ -39,7 +39,7 @@ kbcli cluster create mysql NAME [flags] --storage float Storage size, the unit is Gi. Value range [1, 10000]. (default 20) --tenancy string The tenancy of cluster. Legal values [SharedNode, DedicatedNode]. (default "SharedNode") --termination-policy string The termination policy of cluster. Legal values [DoNotTerminate, Halt, Delete, WipeOut]. (default "Delete") - --tolerations strings Tolerations for cluster, such as "key=value:effect, key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' + --tolerations strings Tolerations for cluster, such as "key=value:effect,key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' --topology-keys stringArray Topology keys for affinity --version string Cluster version, run "kbcli cv list --devel" to see all versions. Legal values [mysql-8.4, mysql-8.0, mysql-5.7]. (default "mysql-8.0") ``` diff --git a/docs/user_docs/cli/kbcli_cluster_create_postgresql.md b/docs/user_docs/cli/kbcli_cluster_create_postgresql.md index 316d8bbf3..8d8ed6042 100644 --- a/docs/user_docs/cli/kbcli_cluster_create_postgresql.md +++ b/docs/user_docs/cli/kbcli_cluster_create_postgresql.md @@ -40,7 +40,7 @@ kbcli cluster create postgresql NAME [flags] --storage-class-name string Storage class name of the data volume --tenancy string The tenancy of cluster. Legal values [SharedNode, DedicatedNode]. (default "SharedNode") --termination-policy string The termination policy of cluster. Legal values [DoNotTerminate, Halt, Delete, WipeOut]. (default "Delete") - --tolerations strings Tolerations for cluster, such as "key=value:effect, key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' + --tolerations strings Tolerations for cluster, such as "key=value:effect,key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' --topology-keys stringArray Topology keys for affinity --version string service version. (default "15.7.0") ``` diff --git a/docs/user_docs/cli/kbcli_cluster_create_qdrant.md b/docs/user_docs/cli/kbcli_cluster_create_qdrant.md index f40f90d47..49e0bfdd6 100644 --- a/docs/user_docs/cli/kbcli_cluster_create_qdrant.md +++ b/docs/user_docs/cli/kbcli_cluster_create_qdrant.md @@ -39,7 +39,7 @@ kbcli cluster create qdrant NAME [flags] --storage-class-name string Storage class name of the data volume --tenancy string The tenancy of cluster. Legal values [SharedNode, DedicatedNode]. (default "SharedNode") --termination-policy string The termination policy of cluster. Legal values [DoNotTerminate, Halt, Delete, WipeOut]. (default "Delete") - --tolerations strings Tolerations for cluster, such as "key=value:effect, key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' + --tolerations strings Tolerations for cluster, such as "key=value:effect,key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' --topology-keys stringArray Topology keys for affinity --version string The version of Qdrant. (default "1.10.0") ``` diff --git a/docs/user_docs/cli/kbcli_cluster_create_redis.md b/docs/user_docs/cli/kbcli_cluster_create_redis.md index 66d7fa15e..3d3e24ee8 100644 --- a/docs/user_docs/cli/kbcli_cluster_create_redis.md +++ b/docs/user_docs/cli/kbcli_cluster_create_redis.md @@ -47,7 +47,7 @@ kbcli cluster create redis NAME [flags] --storage-class-name string Storage class name of the data volume --tenancy string The tenancy of cluster. Legal values [SharedNode, DedicatedNode]. (default "SharedNode") --termination-policy string The termination policy of cluster. Legal values [DoNotTerminate, Halt, Delete, WipeOut]. (default "Delete") - --tolerations strings Tolerations for cluster, such as "key=value:effect, key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' + --tolerations strings Tolerations for cluster, such as "key=value:effect,key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' --topology-keys stringArray Topology keys for affinity --twemproxy.cpu float twemproxy component cpu cores. Value range [0.1, 8]. (default 0.2) --twemproxy.enabled Whether have twemproxy component, default is false diff --git a/docs/user_docs/cli/kbcli_cluster_create_xinference.md b/docs/user_docs/cli/kbcli_cluster_create_xinference.md index 3a5078d6c..e021d8455 100644 --- a/docs/user_docs/cli/kbcli_cluster_create_xinference.md +++ b/docs/user_docs/cli/kbcli_cluster_create_xinference.md @@ -40,7 +40,7 @@ kbcli cluster create xinference NAME [flags] --shm-size string shm size (default "64Mi") --tenancy string The tenancy of cluster. Legal values [SharedNode, DedicatedNode]. (default "SharedNode") --termination-policy string The termination policy of cluster. Legal values [DoNotTerminate, Halt, Delete, WipeOut]. (default "Delete") - --tolerations strings Tolerations for cluster, such as "key=value:effect, key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' + --tolerations strings Tolerations for cluster, such as "key=value:effect,key:effect", for example '"engineType=mongo:NoSchedule", "diskType:NoSchedule"' --topology-keys stringArray Topology keys for affinity ``` From 0a65defdda3f19d3568eb652db2ef95757c5ed45 Mon Sep 17 00:00:00 2001 From: mengyipeng <413183498@qq.com> Date: Fri, 8 Nov 2024 14:07:51 +0800 Subject: [PATCH 4/4] set schedulingPolicy to each component/shard --- pkg/cmd/cluster/cluster_test.go | 33 +++++++++++++--- pkg/cmd/cluster/create_subcmds.go | 56 ++++++++++++++++++--------- pkg/util/util.go | 64 +++++++++++++++++++++++++++++-- 3 files changed, 127 insertions(+), 26 deletions(-) diff --git a/pkg/cmd/cluster/cluster_test.go b/pkg/cmd/cluster/cluster_test.go index 90ece248e..5188d51d5 100644 --- a/pkg/cmd/cluster/cluster_test.go +++ b/pkg/cmd/cluster/cluster_test.go @@ -102,15 +102,15 @@ var _ = Describe("Cluster", func() { Expect(o.Name).ShouldNot(BeEmpty()) Expect(o.Run()).Should(Succeed()) }) - It("with schedulingPolicy", func() { + It("should apply SharedNode tenancy and Preferred PodAntiAffinity", func() { o, err := NewSubCmdsOptions(createOptions, clusterType) + Expect(err).Should(Succeed()) o.Tenancy = "SharedNode" - o.TopologyKeys = []string{"zone", "hostname"} - o.NodeLabels = map[string]string{"environment": "environment", "region": "region"} - o.TolerationsRaw = []string{"key=value:effect", " key:effect"} + o.TopologyKeys = []string{"test-topology1", "test-topology2"} + o.NodeLabels = map[string]string{"environment": "test-env", "region": "test-region"} + o.TolerationsRaw = []string{"testKey1=testValue1:NoSchedule", "testKey2=testValue2:NoExecute"} o.PodAntiAffinity = "Preferred" - Expect(err).Should(Succeed()) Expect(o).ShouldNot(BeNil()) Expect(o.ChartInfo).ShouldNot(BeNil()) o.Format = printer.YAML @@ -124,6 +124,29 @@ var _ = Describe("Cluster", func() { Expect(o.Name).ShouldNot(BeEmpty()) Expect(o.Run()).Should(Succeed()) }) + + It("should apply DedicatedNode tenancy and Required PodAntiAffinity", func() { + o, err := NewSubCmdsOptions(createOptions, clusterType) + Expect(err).Should(Succeed()) + o.Tenancy = "DedicatedNode" + o.TopologyKeys = []string{"test-region", "test-zone"} + o.NodeLabels = map[string]string{"cluster": "test-cluster", "env": "test-production"} + o.TolerationsRaw = []string{"testKey3=testValue3:NoSchedule", "testKey4=testValue4:NoExecute"} + o.PodAntiAffinity = "Required" + + Expect(o).ShouldNot(BeNil()) + Expect(o.ChartInfo).ShouldNot(BeNil()) + o.Format = printer.YAML + + Expect(o.CreateOptions.Complete()).To(Succeed()) + o.Client = testing.FakeClientSet() + fakeDiscovery2, _ := o.Client.Discovery().(*fakediscovery.FakeDiscovery) + fakeDiscovery2.FakedServerVersion = &version.Info{Major: "1", Minor: "27", GitVersion: "v1.27.0"} + Expect(o.Complete(nil)).To(Succeed()) + Expect(o.Validate()).To(Succeed()) + Expect(o.Name).ShouldNot(BeEmpty()) + Expect(o.Run()).Should(Succeed()) + }) }) Context("create validate", func() { diff --git a/pkg/cmd/cluster/create_subcmds.go b/pkg/cmd/cluster/create_subcmds.go index e5dd5c231..50939300f 100644 --- a/pkg/cmd/cluster/create_subcmds.go +++ b/pkg/cmd/cluster/create_subcmds.go @@ -34,8 +34,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" cmdutil "k8s.io/kubectl/pkg/cmd/util" - v1 "github.com/apecloud/kubeblocks/apis/apps/v1" - "github.com/apecloud/kbcli/pkg/action" "github.com/apecloud/kbcli/pkg/cluster" "github.com/apecloud/kbcli/pkg/printer" @@ -62,11 +60,11 @@ type CreateSubCmdsOptions struct { // Configuration and options for cluster affinity and tolerations PodAntiAffinity string `json:"podAntiAffinity"` // TopologyKeys if TopologyKeys is nil, add omitempty json tag, because CueLang can not covert null to list. - TopologyKeys []string `json:"topologyKeys,omitempty"` - NodeLabels map[string]string `json:"nodeLabels,omitempty"` - Tenancy string `json:"tenancy"` - TolerationsRaw []string `json:"-"` - schedulingPolicy *v1.SchedulingPolicy + TopologyKeys []string `json:"topologyKeys,omitempty"` + NodeLabels map[string]string `json:"nodeLabels,omitempty"` + Tenancy string `json:"tenancy"` + TolerationsRaw []string `json:"-"` + Tolerations []corev1.Toleration // SkipSchemaValidation is used to skip the schema validation of the helm chart. SkipSchemaValidation bool `json:"-"` @@ -203,7 +201,6 @@ func (o *CreateSubCmdsOptions) Complete(cmd *cobra.Command) error { } // Build tolerations if raw toleration rules are configured - tolerations := make([]corev1.Toleration, 0) if o.TolerationsRaw != nil { tolerationsResult, err := util.BuildTolerations(o.TolerationsRaw) if err != nil { @@ -213,14 +210,11 @@ func (o *CreateSubCmdsOptions) Complete(cmd *cobra.Command) error { if err != nil { return err } - err = json.Unmarshal(jsonData, &tolerations) + err = json.Unmarshal(jsonData, &o.Tolerations) if err != nil { return err } } - - o.schedulingPolicy = util.BuildSchedulingPolicy(o.Name, tolerations, o.NodeLabels, o.PodAntiAffinity, o.TopologyKeys) - return nil } @@ -256,15 +250,27 @@ func (o *CreateSubCmdsOptions) Run() error { return err } - if clusterObj != nil { - if o.schedulingPolicy != nil { - converted, err := runtime.DefaultUnstructuredConverter.ToUnstructured(o.schedulingPolicy) - if err != nil { + spec, _ := clusterObj.Object["spec"].(map[string]interface{}) + if compSpec, ok := spec["componentSpecs"].([]interface{}); ok { + for i := range compSpec { + comp := compSpec[i].(map[string]interface{}) + if err := o.applySchedulingPolicyToComponent(comp); err != nil { return err } - _ = unstructured.SetNestedField(clusterObj.Object, converted, "spec", "schedulingPolicy") + compSpec[i] = comp + } + } + + if shardingSpec, ok := spec["shardings"].([]interface{}); ok { + for i := range shardingSpec { + shard := shardingSpec[i].(map[string]interface{}) + if compSpec, ok := shard["template"].(map[string]interface{}); ok { + if err := o.applySchedulingPolicyToComponent(compSpec); err != nil { + return err + } + shard["template"] = compSpec + } } - _ = unstructured.SetNestedField(clusterObj.Object, o.Tenancy, "spec", "tenancy") } // only edits the cluster object, other dependency objects are created directly @@ -356,3 +362,17 @@ func (o *CreateSubCmdsOptions) getClusterObj(objs []*objectInfo) (*unstructured. } return nil, fmt.Errorf("failed to find cluster object from manifests rendered from %s chart", o.ClusterType) } + +func (o *CreateSubCmdsOptions) applySchedulingPolicyToComponent(item map[string]interface{}) error { + if compName, ok := item["name"]; ok { + schedulingPolicy, needSet := util.BuildSchedulingPolicy(o.Tenancy, o.Name, compName.(string), o.Tolerations, o.NodeLabels, o.PodAntiAffinity, o.TopologyKeys) + if needSet { + converted, err := runtime.DefaultUnstructuredConverter.ToUnstructured(schedulingPolicy) + if err != nil { + return err + } + _ = unstructured.SetNestedField(item, converted, "schedulingPolicy") + } + } + return nil +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 1665d763c..78c38c8c6 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1083,7 +1083,14 @@ func DisplayDiffWithColor(out io.Writer, diffText string) { } } -func BuildSchedulingPolicy(clusterName string, tolerations []corev1.Toleration, nodeLabels map[string]string, podAntiAffinity string, topologyKeys []string) *kbappsv1.SchedulingPolicy { +func BuildSchedulingPolicy(tenancy string, clusterName string, compName string, tolerations []corev1.Toleration, nodeLabels map[string]string, podAntiAffinity string, topologyKeys []string) (*kbappsv1.SchedulingPolicy, bool) { + if len(tolerations) == 0 && len(nodeLabels) == 0 && len(topologyKeys) == 0 { + return nil, false + } + affinity := &corev1.Affinity{} + if podAntiAffinity != "" || len(topologyKeys) > 0 { + affinity.PodAntiAffinity = BuildPodAntiAffinityForComponent(tenancy, clusterName, compName, podAntiAffinity, topologyKeys) + } var topologySpreadConstraints []corev1.TopologySpreadConstraint @@ -1100,19 +1107,21 @@ func BuildSchedulingPolicy(clusterName string, tolerations []corev1.Toleration, TopologyKey: topologyKey, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - constant.AppInstanceLabelKey: clusterName, + constant.AppInstanceLabelKey: clusterName, + constant.KBAppComponentLabelKey: compName, }, }, }) } schedulingPolicy := &kbappsv1.SchedulingPolicy{ + Affinity: affinity, NodeSelector: nodeLabels, Tolerations: tolerations, TopologySpreadConstraints: topologySpreadConstraints, } - return schedulingPolicy + return schedulingPolicy, true } // BuildTolerations toleration format: key=value:effect or key:effect, @@ -1203,6 +1212,55 @@ func BuildPodAntiAffinity(podAntiAffinityStrategy string, topologyKeys []string) return podAntiAffinity } +// BuildPodAntiAffinityForComponent build pod anti affinity from topology keys and tenancy for cluster +func BuildPodAntiAffinityForComponent(tenancy string, clusterName string, compName string, podAntiAffinityStrategy string, topologyKeys []string) *corev1.PodAntiAffinity { + var podAntiAffinity *corev1.PodAntiAffinity + var podAffinityTerms []corev1.PodAffinityTerm + for _, topologyKey := range topologyKeys { + podAffinityTerms = append(podAffinityTerms, corev1.PodAffinityTerm{ + TopologyKey: topologyKey, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + constant.AppInstanceLabelKey: clusterName, + constant.KBAppComponentLabelKey: compName, + }, + }, + }) + } + if podAntiAffinityStrategy == string(kbappsv1alpha1.Required) { + podAntiAffinity = &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: podAffinityTerms, + } + } else { + var weightedPodAffinityTerms []corev1.WeightedPodAffinityTerm + for _, podAffinityTerm := range podAffinityTerms { + weightedPodAffinityTerms = append(weightedPodAffinityTerms, corev1.WeightedPodAffinityTerm{ + Weight: 100, + PodAffinityTerm: podAffinityTerm, + }) + } + podAntiAffinity = &corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: weightedPodAffinityTerms, + } + } + + // Add pod PodAffinityTerm for dedicated node + if kbappsv1alpha1.TenancyType(tenancy) == kbappsv1alpha1.DedicatedNode { + podAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append( + podAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, corev1.PodAffinityTerm{ + TopologyKey: corev1.LabelHostname, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + constant.AppInstanceLabelKey: clusterName, + constant.KBAppComponentLabelKey: compName, + }, + }, + }) + } + + return podAntiAffinity +} + // AddDirToPath add a dir to the PATH environment variable func AddDirToPath(dir string) error { if dir == "" {