support: refactor embed charts to source

.gitignore

@@ -16,8 +16,6 @@ bin
 testbin/*
 __debug_bin
 
-pkg/cluster/charts/*.tgz
-
 output
 helm-output
 
@@ -54,8 +52,8 @@ bin/
 tmp/
 
 # cli embed chart directory
-# exclude embed charts for go package
-!pkg/cluster/charts/*.tgz
+# exclude embed charts for go package, such as kblib
+!pkg/cluster/charts/**/*.tgz
 
 # ignore the addons that clone from apecloud/kubeblocks-addons
 addons/

Makefile

@@ -207,20 +207,25 @@ else
 endif
 	mv $(chart)-*.tgz pkg/cluster/charts/$(chart).tgz
 
+cp-embed-charts.%: chart=$(word 2,$(subst ., ,$@))
+	# cp -r addons/addons/*-cluster pkg/cluster/charts/
+cp-embed-charts.%:
+	cp -r addons/addons/$(chart) pkg/cluster/charts/
+
 .PHONY: build-kbcli-embed-chart
 build-kbcli-embed-chart: helmtool fetch-addons create-kbcli-embed-charts-dir \
-	build-single-kbcli-embed-chart.apecloud-mysql-cluster \
-	build-single-kbcli-embed-chart.redis-cluster \
-	build-single-kbcli-embed-chart.postgresql-cluster \
-	build-single-kbcli-embed-chart.kafka-cluster \
-	build-single-kbcli-embed-chart.mongodb-cluster \
-	build-single-kbcli-embed-chart.llm-cluster \
-	build-single-kbcli-embed-chart.xinference-cluster \
-	build-single-kbcli-embed-chart.neon-cluster \
-	build-single-kbcli-embed-chart.clickhouse-cluster \
-	build-single-kbcli-embed-chart.milvus-cluster \
-	build-single-kbcli-embed-chart.qdrant-cluster \
-	build-single-kbcli-embed-chart.weaviate-cluster
+	cp-embed-charts.apecloud-mysql-cluster \
+	cp-embed-charts.redis-cluster \
+	cp-embed-charts.postgresql-cluster \
+	cp-embed-charts.kafka-cluster \
+	cp-embed-charts.mongodb-cluster \
+	cp-embed-charts.llm-cluster \
+	cp-embed-charts.xinference-cluster \
+	cp-embed-charts.neon-cluster \
+	cp-embed-charts.clickhouse-cluster \
+	cp-embed-charts.milvus-cluster \
+	cp-embed-charts.qdrant-cluster \
+	cp-embed-charts.weaviate-cluster
 
 .PHONY: kbcli
 kbcli: build-checks kbcli-fast ## Build bin/kbcli.

pkg/cluster/builtin_charts.go

@@ -58,92 +58,44 @@ func (e *embedConfig) getChartFileName() string {
 
 var (
 	// run `make generate` to generate this embed file
-	//go:embed charts/apecloud-mysql-cluster.tgz
-	mysqlChart embed.FS
-	//go:embed charts/postgresql-cluster.tgz
-	postgresqlChart embed.FS
-	//go:embed charts/kafka-cluster.tgz
-	kafkaChart embed.FS
-	//go:embed charts/redis-cluster.tgz
-	redisChart embed.FS
-	//go:embed charts/mongodb-cluster.tgz
-	mongodbChart embed.FS
-	//go:embed charts/llm-cluster.tgz
-	llmChart embed.FS
-	//go:embed charts/xinference-cluster.tgz
-	xinferenceChart embed.FS
-	//go:embed charts
+	// must add all: to include files start with _, such as _helpers.tpl
+	//go:embed all:charts
 	defaultChart embed.FS
 )
 
-func IsbuiltinCharts(chart string) bool {
-	return chart == "mysql" || chart == "postgresql" || chart == "kafka" || chart == "redis" || chart == "mongodb" || chart == "llm"
+var builtinClusterTypes = []ClusterType{
+	"mysql",
+	"postgresql",
+	"kafka",
+	"redis",
+	"mongodb",
+	"llm",
+	"xinference",
 }
 
-// internal_chart registers embed chart
-
-func init() {
-
-	mysql := &embedConfig{
-		chartFS: mysqlChart,
-		name:    "apecloud-mysql-cluster.tgz",
-		alias:   "",
-	}
-	if err := mysql.register("mysql"); err != nil {
-		fmt.Println(err.Error())
-	}
-
-	postgresql := &embedConfig{
-		chartFS: postgresqlChart,
-		name:    "postgresql-cluster.tgz",
-		alias:   "",
-	}
-	if err := postgresql.register("postgresql"); err != nil {
-		fmt.Println(err.Error())
-	}
-
-	kafka := &embedConfig{
-		chartFS: kafkaChart,
-		name:    "kafka-cluster.tgz",
-		alias:   "",
-	}
-	if err := kafka.register("kafka"); err != nil {
-		fmt.Println(err.Error())
-	}
+var builtinChartMap = map[ClusterType]string{
+	"mysql": "apecloud-mysql",
+}
 
-	redis := &embedConfig{
-		chartFS: redisChart,
-		name:    "redis-cluster.tgz",
-		alias:   "",
-	}
-	if err := redis.register("redis"); err != nil {
-		fmt.Println(err.Error())
+func IsbuiltinCharts(chart string) bool {
+	for _, c := range builtinClusterTypes {
+		if chart == string(c) {
+			return true
+		}
 	}
+	return false
+}
 
-	mongodb := &embedConfig{
-		chartFS: mongodbChart,
-		name:    "mongodb-cluster.tgz",
-		alias:   "",
-	}
-	if err := mongodb.register("mongodb"); err != nil {
-		fmt.Println(err.Error())
+func getChartName(t ClusterType) string {
+	name, ok := builtinChartMap[t]
+	if !ok {
+		// if not found in map, return original name
+		return string(t)
 	}
+	return name
+}
 
-	llm := &embedConfig{
-		chartFS: llmChart,
-		name:    "llm-cluster.tgz",
-		alias:   "",
-	}
-	if err := llm.register("llm"); err != nil {
-		fmt.Println(err.Error())
-	}
+// internal_chart registers embed chart
 
-	xinference := &embedConfig{
-		chartFS: xinferenceChart,
-		name:    "xinference-cluster.tgz",
-		alias:   "",
-	}
-	if err := xinference.register("xinference"); err != nil {
-		fmt.Println(err.Error())
-	}
+func init() {
 }

pkg/cluster/charts/apecloud-mysql-cluster/.helmignore

@@ -0,0 +1,24 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+*.lock

pkg/cluster/charts/apecloud-mysql-cluster/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: kblib
+  repository: file://../kblib
+  version: 0.1.0
+digest: sha256:6894d645e270bae072e3334549187801abc343630044c9da70e7a22e2ebfc55b
+generated: "2023-12-15T18:19:52.160151+08:00"

pkg/cluster/charts/apecloud-mysql-cluster/Chart.yaml

@@ -0,0 +1,37 @@
+annotations:
+  category: Database
+apiVersion: v2
+name: apecloud-mysql-cluster
+type: application
+version: 0.7.0
+description: ApeCloud MySQL is a database that is compatible with MySQL syntax and achieves high availability
+  through the utilization of the RAFT consensus protocol.
+dependencies:
+  - name: kblib
+    version: 0.1.0
+    repository: file://../kblib
+    alias: extra
+
+# This is the version number of the ApeCloud MySQL being deployed,
+# rather than the version number of ApeCloud MySQL-Scale itself.
+# From a user's perspective, the version  number of the frontend
+# proxy of the MySQL being used is not relevant.
+appVersion: "8.0.30"
+
+keywords:
+  - mysql
+  - database
+  - sql
+  - cluster
+  - raft
+
+home: https://github.com/apecloud/kubeblocks/tree/main/deploy/apecloud-mysql-cluster
+icon: https://kubeblocks.io/img/logo.png
+
+maintainers:
+  - name: ApeCloud
+    url: https://kubeblocks.io/
+
+sources:
+  - https://github.com/apecloud/kubeblocks/
+  - https://github.com/apecloud/kubeblocks/tree/main/deploy/apecloud-mysql

pkg/cluster/charts/apecloud-mysql-cluster/templates/NOTES.txt

@@ -0,0 +1,2 @@
+1. Get the application URL by running these commands:
+

pkg/cluster/charts/apecloud-mysql-cluster/templates/_helpers.tpl

@@ -0,0 +1,60 @@
+{{/*
+Define the cluster componnets with proxy.
+The proxy cpu cores is 1/6 of the cluster total cpu cores and is multiple of 0.5.
+The minimum proxy cpu cores is 0.5 and the maximum cpu cores is 64.
+*/}}
+{{- define "apecloud-mysql-cluster.proxyComponents" }}
+{{- $replicas := (include "apecloud-mysql-cluster.replicas" .) }}
+{{- $proxyCPU := divf (mulf $replicas .Values.cpu) 6.0 }}
+{{- $proxyCPU = divf $proxyCPU 0.5 | ceil | mulf 0.5 }}
+{{- if lt $proxyCPU 0.5 }}
+{{- $proxyCPU = 0.5 }}
+{{- else if gt $proxyCPU 64.0 }}
+{{- $proxyCPU = 64 }}
+{{- end }}
+- name: vtcontroller
+  componentDefRef: vtcontroller # ref clusterdefinition componentDefs.name
+  enabledLogs:
+    - error
+    - warning
+    - info
+  volumeClaimTemplates:
+    - name: data
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 20Gi
+  replicas: 1
+  resources:
+    limits:
+      cpu: 500m
+      memory: 128Mi
+- name: vtgate
+  componentDefRef: vtgate # ref clusterdefinition componentDefs.name
+  replicas: 1
+  enabledLogs:
+    - error
+    - warning
+    - info
+    - queryLog
+  resources:
+    requests:
+      cpu: {{ $proxyCPU | quote }}
+    limits:
+      cpu: {{ $proxyCPU | quote }}
+{{- end }}
+
+{{/*
+Define replicas.
+standalone mode: 1
+raftGroup mode: max(replicas, 3)
+*/}}
+{{- define "apecloud-mysql-cluster.replicas" }}
+{{- if eq .Values.mode "standalone" }}
+{{- 1 }}
+{{- else if eq .Values.mode "raftGroup" }}
+{{- max .Values.replicas 3 }}
+{{- end }}
+{{- end -}}

pkg/cluster/charts/apecloud-mysql-cluster/templates/cluster.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps.kubeblocks.io/v1alpha1
+kind: Cluster
+metadata:
+  name: {{ include "kblib.clusterName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels: {{ include "kblib.clusterLabels" . | nindent 4 }}
+  {{- if and (eq .Values.mode "raftGroup") .Values.proxyEnabled }}
+  annotations:
+    kubeblocks.io/extra-env: '{"KB_PROXY_ENABLED":"on"}'
+  {{- end }}
+spec:
+  clusterVersionRef: {{ .Values.version }}
+  terminationPolicy: {{ .Values.extra.terminationPolicy }}
+  {{- include "kblib.affinity" . | indent 2 }}
+  clusterDefinitionRef: apecloud-mysql # ref clusterdefinition.name
+  componentSpecs:
+    - name: mysql
+      componentDefRef: mysql # ref clusterdefinition componentDefs.name
+      {{- include "kblib.componentMonitor" . | indent 6 }}
+      replicas: {{ include "apecloud-mysql-cluster.replicas" . }}
+      enabledLogs:
+        - slow
+        - error
+      serviceAccountName: {{ include "kblib.serviceAccountName" . }}
+      {{- include "kblib.componentResources" . | indent 6 }}
+      {{- include "kblib.componentStorages" . | indent 6 }}
+      {{- include "kblib.componentServices" . | indent 6 }}
+    {{- if and (eq .Values.mode "raftGroup") .Values.proxyEnabled }}
+    {{- include "apecloud-mysql-cluster.proxyComponents" . | indent 4 }}
+    {{- end }}

pkg/cluster/charts/apecloud-mysql-cluster/templates/rbac.yaml

@@ -0,0 +1 @@
+{{- include "kblib.rbac" . }}

pkg/cluster/charts/apecloud-mysql-cluster/templates/validate.yaml

@@ -0,0 +1,15 @@
+{{/*
+Validate cpu, memory and storage
+*/}}
+{{- $cpu := (float64 .Values.cpu) }}
+{{- $memory := (float64 .Values.memory) }}
+{{- $storage := (float64 .Values.storage) }}
+{{- if or (lt $cpu 0.5) (gt $cpu 64.0) }}
+{{- fail (print "cpu must be between 0.5 and 64, got " $cpu) }}
+{{- end }}
+{{- if or (lt $memory 0.5) (gt $memory 1000.0) }}
+{{- fail (print "memory must be between 0.5 and 1000, got " $memory) }}
+{{- end }}
+{{- if or (lt $storage 1.0 ) (gt $storage 10000.0) }}
+{{- fail (print "storage must be between 1 and 10000, got " $storage) }}
+{{- end }}