From f9123481b538fe37be93d270aeb9c377e9e83b4e Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 12:01:19 +0700 Subject: [PATCH 001/188] [mayastor] Initial Commit --- charts/mayastor/.helmignore | 25 ++ charts/mayastor/Chart.yaml | 44 ++++ charts/mayastor/crds/MayastorPool.yaml | 142 ++++++++++ charts/mayastor/templates/ClusterRole.yaml | 58 +++++ .../templates/ClusterRoleBinding.yaml | 23 ++ charts/mayastor/templates/DaemonSet.yaml | 121 +++++++++ charts/mayastor/templates/MayastorPool.yaml | 15 ++ charts/mayastor/templates/ServiceAccount.yaml | 21 ++ charts/mayastor/templates/_helper.tpl | 38 +++ .../templates/core-agents/Deployment.yaml | 76 ++++++ charts/mayastor/templates/csi/DaemonSet.yaml | 145 +++++++++++ .../mayastor/templates/rest/Deployment.yaml | 72 ++++++ charts/mayastor/templates/rest/Service.yaml | 73 ++++++ charts/mayastor/values-test.yaml | 13 + charts/mayastor/values.yaml | 244 ++++++++++++++++++ 15 files changed, 1110 insertions(+) create mode 100644 charts/mayastor/.helmignore create mode 100644 charts/mayastor/Chart.yaml create mode 100644 charts/mayastor/crds/MayastorPool.yaml create mode 100644 charts/mayastor/templates/ClusterRole.yaml create mode 100644 charts/mayastor/templates/ClusterRoleBinding.yaml create mode 100644 charts/mayastor/templates/DaemonSet.yaml create mode 100644 charts/mayastor/templates/MayastorPool.yaml create mode 100644 charts/mayastor/templates/ServiceAccount.yaml create mode 100644 charts/mayastor/templates/_helper.tpl create mode 100644 charts/mayastor/templates/core-agents/Deployment.yaml create mode 100644 charts/mayastor/templates/csi/DaemonSet.yaml create mode 100644 charts/mayastor/templates/rest/Deployment.yaml create mode 100644 charts/mayastor/templates/rest/Service.yaml create mode 100644 charts/mayastor/values-test.yaml create mode 100644 charts/mayastor/values.yaml diff --git a/charts/mayastor/.helmignore b/charts/mayastor/.helmignore new file mode 100644 index 00000000..b47f5974 --- /dev/null +++ b/charts/mayastor/.helmignore @@ -0,0 +1,25 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store + +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ + +# Common backup files +*.swp +*.bak +*.tmp +*~ + +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ \ No newline at end of file diff --git a/charts/mayastor/Chart.yaml b/charts/mayastor/Chart.yaml new file mode 100644 index 00000000..fa43a123 --- /dev/null +++ b/charts/mayastor/Chart.yaml @@ -0,0 +1,44 @@ +annotations: + category: Storage +apiVersion: v2 +appVersion: 1.0.2 +dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x + - name: etcd + condition: etcd.enabled + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-etcd + version: 8.x.x + - name: nats + condition: nats.enabled + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-nats + version: 7.x.x +description: Mayastor is a cloud-native declarative data plane written in Rust. + Our goal is to abstract storage resources and their differences through the data + plane such that users only need to supply the what and do not have to worry about + the how so that individual teams stay in control. +home: https://github.com/startechnica/apps/tree/main/charts/mayastor +icon: https://openebs.io/images/seo/openebs.png +keywords: + - mayastor + - openebs +maintainers: + - name: firmansyahn + email: firmansyah@nainggolan.id + url: https://firmansyah.nainggolan.id +name: mayastor +sources: + - https://github.com/openebs/mayastor + - https://github.com/openebs/mayastor-control-plane + - https://github.com/openebs/mayastor-api + - https://openebs.io/docs/concepts/mayastor + - https://mayastor.gitbook.io +type: application +version: 0.1.0 diff --git a/charts/mayastor/crds/MayastorPool.yaml b/charts/mayastor/crds/MayastorPool.yaml new file mode 100644 index 00000000..93f59fb2 --- /dev/null +++ b/charts/mayastor/crds/MayastorPool.yaml @@ -0,0 +1,142 @@ +--- +# Source: mayastor-control-plane/templates/mayastorpoolcrd.yaml +{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": { + "name": "mayastorpools.openebs.io" + }, + "spec": { + "group": "openebs.io", + "names": { + "categories": [], + "kind": "MayastorPool", + "plural": "mayastorpools", + "shortNames": [ + "msp" + ], + "singular": "mayastorpool" + }, + "scope": "Namespaced", + "versions": [ + { + "additionalPrinterColumns": [ + { + "description": "node the pool is on", + "jsonPath": ".spec.node", + "name": "node", + "type": "string" + }, + { + "description": "pool status", + "jsonPath": ".status.state", + "name": "status", + "type": "string" + }, + { + "description": "total bytes", + "format": "int64", + "jsonPath": ".status.capacity", + "name": "capacity", + "type": "integer" + }, + { + "description": "used bytes", + "format": "int64", + "jsonPath": ".status.used", + "name": "used", + "type": "integer" + }, + { + "description": "available bytes", + "format": "int64", + "jsonPath": ".status.available", + "name": "available", + "type": "integer" + } + ], + "name": "v1alpha1", + "schema": { + "openAPIV3Schema": { + "description": "Auto-generated derived type for MayastorPoolSpec via `CustomResource`", + "properties": { + "spec": { + "description": "The pool spec which contains the parameters we use when creating the pool", + "properties": { + "disks": { + "description": "The disk device the pool is located on", + "items": { + "type": "string" + }, + "type": "array" + }, + "node": { + "description": "The node the pool is placed on", + "type": "string" + } + }, + "required": [ + "disks", + "node" + ], + "type": "object" + }, + "status": { + "description": "Status of the pool which is driven and changed by the controller loop", + "nullable": true, + "properties": { + "available": { + "description": "Available number of bytes", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "capacity": { + "description": "Capacity as number of bytes", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "state": { + "description": "The state of the pool", + "enum": [ + "Creating", + "Created", + "Online", + "Unknown", + "Error" + ], + "type": "string" + }, + "used": { + "description": "Used number of bytes", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "available", + "capacity", + "state", + "used" + ], + "type": "object" + } + }, + "required": [ + "spec" + ], + "title": "MayastorPool", + "type": "object" + } + }, + "served": true, + "storage": true, + "subresources": { + "status": {} + } + } + ] + } +} \ No newline at end of file diff --git a/charts/mayastor/templates/ClusterRole.yaml b/charts/mayastor/templates/ClusterRole.yaml new file mode 100644 index 00000000..8ef0b989 --- /dev/null +++ b/charts/mayastor/templates/ClusterRole.yaml @@ -0,0 +1,58 @@ +# Source: mayastor-control-plane/templates/operator-rbac.yaml + +{{- if and .Values.serviceAccount.create .Values.rbac.create }} +apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} +kind: ClusterRole +metadata: + name: {{ include "common.names.fullname" . }} +rules: + # must create mayastor crd if it doesn't exist +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list"] + # must read mayastor pools info +- apiGroups: ["openebs.io"] + resources: ["mayastorpools"] + verbs: ["get", "list", "watch", "update", "replace", "patch"] + # must update mayastor pools status +- apiGroups: ["openebs.io"] + resources: ["mayastorpools/status"] + verbs: ["update", "patch"] + # external provisioner & attacher +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "create", "delete", "patch"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + # external provisioner +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + # external attacher +- apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] +- apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + # CSI nodes must be listed +- apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/ClusterRoleBinding.yaml b/charts/mayastor/templates/ClusterRoleBinding.yaml new file mode 100644 index 00000000..a972cab9 --- /dev/null +++ b/charts/mayastor/templates/ClusterRoleBinding.yaml @@ -0,0 +1,23 @@ +# Source: mayastor-control-plane/includes/operator-rbac.yaml + +{{- if and .Values.serviceAccount.create .Values.rbac.create }} +apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} +kind: ClusterRoleBinding +metadata: + name: {{ include "common.names.fullname" . }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "mayastor.serviceAccountName" . }} + namespace: {{ include "common.names.namespace" . | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "common.names.fullname" . }} +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml new file mode 100644 index 00000000..591bcb7e --- /dev/null +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -0,0 +1,121 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + openebs/engine: mayastor + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: mayastor + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + minReadySeconds: 10 + template: + metadata: + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "common.labels.standard" . | nindent 8 }} + app.kubernetes.io/component: mayastor + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + spec: + hostNetwork: true + # To resolve services from mayastor's namespace + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + openebs.io/engine: mayastor + kubernetes.io/arch: amd64 + initContainers: + - name: message-bus-probe + image: busybox:latest + command: ['sh', '-c', 'until nc -vz nats 4222; do echo "Waiting for message bus..."; sleep 1; done;'] + containers: + - name: mayastor + image: {{ include "mayastor.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + env: + - name: RUST_LOG + value: info,mayastor={{ .Values.mayastorLogLevel }} + - name: NVMF_TCP_MAX_QUEUE_DEPTH + value: "32" + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + args: + # The -l argument accepts cpu-list. Indexing starts at zero. + # For example -l 1,2,10-20 means use core 1, 2, 10 to 20. + # Note: Ensure that the CPU resources are updated accordingly. + # If you use 2 CPUs, the CPU: field should also read 2. + - "-N$(MY_NODE_NAME)" + - "-g$(MY_POD_IP)" + - "-nnats" + - "-y/var/local/mayastor/config.yaml" + - "-l{{ include "mayastorCpuSpec" . }}" + - "-pmayastor-etcd" + command: + - mayastor + securityContext: + privileged: true + volumeMounts: + - name: device + mountPath: /dev + - name: udev + mountPath: /run/udev + - name: dshm + mountPath: /dev/shm + - name: configlocation + mountPath: /var/local/mayastor/ + resources: + # NOTE: Each container must have mem/cpu limits defined in order to + # belong to Guaranteed QoS class, hence can never get evicted in case of + # pressure unless they exceed those limits. limits and requests must be the same. + limits: + cpu: "{{ .Values.mayastorCpuCount }}" + memory: "1Gi" + hugepages-2Mi: "{{ max .Values.mayastorHugePagesGiB 2 }}Gi" + requests: + cpu: "{{ .Values.mayastorCpuCount }}" + memory: "1Gi" + hugepages-2Mi: "{{ max .Values.mayastorHugePagesGiB 2 }}Gi" + ports: + - containerPort: 10124 + protocol: TCP + name: mayastor + volumes: + - name: device + hostPath: + path: /dev + type: Directory + - name: udev + hostPath: + path: /run/udev + type: Directory + - name: dshm + emptyDir: + medium: Memory + sizeLimit: "1Gi" + - name: hugepage + emptyDir: + medium: HugePages + - name: configlocation + hostPath: + path: /var/local/mayastor/ + type: DirectoryOrCreate \ No newline at end of file diff --git a/charts/mayastor/templates/MayastorPool.yaml b/charts/mayastor/templates/MayastorPool.yaml new file mode 100644 index 00000000..8377f82e --- /dev/null +++ b/charts/mayastor/templates/MayastorPool.yaml @@ -0,0 +1,15 @@ +{{- range .Values.mayastorPools }} +--- +apiVersion: "openebs.io/v1alpha1" +kind: MayastorPool +metadata: + # Name can be anything as long as it is unique + name: pool-on-{{ .node }} + # or let k8s to generate a unique pool name + #generateName: pool- + namespace: {{ include "common.names.namespace" . | quote }} +spec: + node: {{ .node }} + # ATM only one disk device is supported (i.e. /dev/nvme1n1) + disks: ["{{ .device }}"] +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/ServiceAccount.yaml b/charts/mayastor/templates/ServiceAccount.yaml new file mode 100644 index 00000000..7f14f862 --- /dev/null +++ b/charts/mayastor/templates/ServiceAccount.yaml @@ -0,0 +1,21 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "mayastor.serviceAccountName" . }} + namespace: {{ include "multus.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if or .Values.serviceAccount.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.serviceAccount.annotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.serviceAccount.annotations "context" $) | nindent 4 }} + {{- end }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl new file mode 100644 index 00000000..ab436907 --- /dev/null +++ b/charts/mayastor/templates/_helper.tpl @@ -0,0 +1,38 @@ +{{/* Return the proper Mayastor image name */}} +{{- define "mayastor.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }} +{{- end -}} + +{{/* Return the proper Mayastor CSI image name */}} +{{- define "mayastor.csi.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.csi.image "global" .Values.global) }} +{{- end -}} + +{{/* Return the proper image name (for the init container volume-permissions image) */}} +{{- define "mayastor.volumePermissions.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.volumePermissions.image "global" .Values.global) }} +{{- end -}} + +{{/* Return the proper Docker Image Registry Secret Names */}} +{{- define "mayastor.imagePullSecrets" -}} + {{- include "common.images.pullSecrets" (dict "images" (list .Values.image .Values.volumePermissions.image) "global" .Values.global) -}} +{{- end -}} + +{{/* Enforce trailing slash to mayastorImagesPrefix or leave empty */}} +{{- define "mayastorImagesPrefix" -}} +{{- if .Values.mayastorImagesRegistry }} +{{- printf "%s/" (.Values.mayastorImagesRegistry | trimSuffix "/") }} +{{- else }} +{{- "" }} +{{- end }} +{{- end }} + +{{/* Generate CPU list specification based on CPU count (-l param of mayastor) */}} +{{- define "mayastorCpuSpec" -}} +{{- range $i, $e := until (int .Values.mayastorCpuCount) }} +{{- if gt $i 0 }} +{{- printf "," }} +{{- end }} +{{- printf "%d" (add $i 1) }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/core-agents/Deployment.yaml b/charts/mayastor/templates/core-agents/Deployment.yaml new file mode 100644 index 00000000..f4c58d7a --- /dev/null +++ b/charts/mayastor/templates/core-agents/Deployment.yaml @@ -0,0 +1,76 @@ +## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/core-agents-deployment.yaml + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ printf "%s-core-agents" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: core-agents + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: core-agents + template: + metadata: + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "common.labels.standard" . | nindent 8 }} + app.kubernetes.io/component: core-agents + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + spec: + imagePullSecrets: + - name: regcred + initContainers: + - command: + - sh + - -c + - trap "exit 1" TERM; until nc -vz nats 4222; do echo "Waiting for nats..."; sleep + 1; done; + image: busybox:latest + name: nats-probe + - command: + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; + sleep 1; done; + image: busybox:latest + name: etcd-probe + containers: + - name: core + resources: + limits: + cpu: 1000m + memory: 32Mi + requests: + cpu: 500m + memory: 16Mi + image: mayadata/mcp-core:v1.0.1 + imagePullPolicy: IfNotPresent + args: + - "-smayastor-etcd" + - "-nnats" + - "--request-timeout=5s" + - "--cache-period=30s" + env: + - name: RUST_LOG + value: info + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace \ No newline at end of file diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml new file mode 100644 index 00000000..c0d84bde --- /dev/null +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -0,0 +1,145 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ printf "%s-csi" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + openebs/engine: mayastor + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: mayastor-csi + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + minReadySeconds: 10 + template: + metadata: + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "common.labels.standard" . | nindent 8 }} + app.kubernetes.io/component: mayastor-csi + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + spec: + hostNetwork: true + nodeSelector: + kubernetes.io/arch: amd64 + # NOTE: Each container must have mem/cpu limits defined in order to + # belong to Guaranteed QoS class, hence can never get evicted in case of + # pressure unless they exceed those limits. limits and requests must be + # the same. + containers: + - name: mayastor-csi + image: {{ include "mayastor.csi.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + # we need privileged because we mount filesystems and use mknod + securityContext: + privileged: true + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: RUST_BACKTRACE + value: "1" + {{- if .Values.moac }} + - name: MOAC + value: "true" + {{- end }} + args: + - "--csi-socket=/csi/csi.sock" + - "--node-name=$(MY_NODE_NAME)" + - "--grpc-endpoint=$(MY_POD_IP):10199" + {{- if .Values.csi.nvme.io_timeout_enabled }} + - "--nvme-core-io-timeout={{ .Values.csi.nvme.io_timeout }}" + {{- end }} + - "-v" + command: + - mayastor-csi + volumeMounts: + - name: device + mountPath: /dev + - name: sys + mountPath: /sys + - name: run-udev + mountPath: /run/udev + - name: host-root + mountPath: /host + - name: plugin-dir + mountPath: /csi + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + resources: + limits: + cpu: "100m" + memory: "50Mi" + requests: + cpu: "100m" + memory: "50Mi" + - name: csi-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v2.1.0 + args: + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/mayastor.openebs.io/csi.sock" + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + limits: + cpu: "100m" + memory: "50Mi" + requests: + cpu: "100m" + memory: "50Mi" + # Mayastor node plugin gRPC server + ports: + - containerPort: 10199 + protocol: TCP + name: mayastor-node + volumes: + - name: device + hostPath: + path: /dev + type: Directory + - name: sys + hostPath: + path: /sys + type: Directory + - name: run-udev + hostPath: + path: /run/udev + type: Directory + - name: host-root + hostPath: + path: / + type: Directory + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/mayastor.openebs.io/ + type: DirectoryOrCreate + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory \ No newline at end of file diff --git a/charts/mayastor/templates/rest/Deployment.yaml b/charts/mayastor/templates/rest/Deployment.yaml new file mode 100644 index 00000000..0e274619 --- /dev/null +++ b/charts/mayastor/templates/rest/Deployment.yaml @@ -0,0 +1,72 @@ +## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/rest-deployment.yaml + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ printf "%s-rest" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: rest + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: rest + template: + metadata: + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "common.labels.standard" . | nindent 8 }} + app.kubernetes.io/component: rest + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + spec: + imagePullSecrets: + - name: regcred + initContainers: + - command: + - sh + - -c + - trap "exit 1" TERM; until nc -vz nats 4222; do echo "Waiting for nats..."; sleep + 1; done; + image: busybox:latest + name: nats-probe + - command: + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; + sleep 1; done; + image: busybox:latest + name: etcd-probe + containers: + - name: rest + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 50m + memory: 32Mi + image: mayadata/mcp-rest:v1.0.1 + imagePullPolicy: IfNotPresent + args: + - "--dummy-certificates" + - "--no-auth" + - "-nnats" + - "--http=0.0.0.0:8081" + - "--request-timeout=5s" + ports: + - containerPort: 8080 + - containerPort: 8081 + env: + - name: RUST_LOG + value: info \ No newline at end of file diff --git a/charts/mayastor/templates/rest/Service.yaml b/charts/mayastor/templates/rest/Service.yaml new file mode 100644 index 00000000..78aff08c --- /dev/null +++ b/charts/mayastor/templates/rest/Service.yaml @@ -0,0 +1,73 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-rest" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: rest + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if or .Values.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.service.annotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.annotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.annotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.allocateLoadBalancerNodePorts)) }} + allocateLoadBalancerNodePorts: {{ .Values.service.allocateLoadBalancerNodePorts }} + {{- end }} + {{- if and .Values.service.clusterIP (eq .Values.service.type "ClusterIP") }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if and .Values.service.externalTrafficPolicy (or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort")) }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | quote }} + {{- end }} + ipFamilyPolicy: {{ .Values.service.ipFamilyPolicy }} + {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerClass)) }} + loadBalancerClass: {{ .Values.service.loadBalancerClass }} + {{- end }} + {{- if (and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP))) }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerSourceRanges)) }} + loadBalancerSourceRanges: {{ .Values.service.loadBalancerSourceRanges }} + {{- end }} + {{- if .Values.service.sessionAffinity }} + sessionAffinity: {{ .Values.service.sessionAffinity }} + {{- end }} + {{- if .Values.service.sessionAffinityConfig }} + sessionAffinityConfig: {{- include "common.tplvalues.render" (dict "value" .Values.service.sessionAffinityConfig "context" $) | nindent 4 }} + {{- end }} + ports: + - name: http + port: {{ .Values.service.ports.http }} + protocol: UDP + targetPort: {{ .Values.containerPorts.http }} + {{- if (and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) .Values.service.nodePorts.http) }} + nodePort: {{ coalesce .Values.service.nodePorts.http .Values.service.nodePort }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- if .Values.tls.enabled }} + - name: https + port: {{ .Values.service.ports.https }} + protocol: TCP + targetPort: {{ .Values.containerPorts.https }} + {{- if (and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) .Values.service.nodePorts.https) }} + nodePort: {{ .Values.service.nodePorts.https }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- end }} + selector: {{ include "common.labels.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: rest +--- \ No newline at end of file diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml new file mode 100644 index 00000000..7eb3c943 --- /dev/null +++ b/charts/mayastor/values-test.yaml @@ -0,0 +1,13 @@ + +clusterDomain: nebula.mci.net.id + +etcd: + clusterDomain: nebula.mci.net.id + replicaCount: 3 + persistence: + enabled: true + storageClass: openebs-hostpath + +nats: + clusterDomain: nebula.mci.net.id + replicaCount: 3 diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml new file mode 100644 index 00000000..c9d10791 --- /dev/null +++ b/charts/mayastor/values.yaml @@ -0,0 +1,244 @@ +## @section Global parameters +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass + +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker registry secret names as an array +## @param global.storageClass Global StorageClass for Persistent Volume(s) +## +global: + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## + imagePullSecrets: [] + storageClass: "" + +## @section Common parameters + +## @param kubeVersion Force target Kubernetes version (using Helm capabilities if not set) +## +kubeVersion: "" +## @param nameOverride String to partially override adminer.fullname +## +nameOverride: "" +## @param fullnameOverride String to fully override adminer.fullname +## +fullnameOverride: "" +## @param hostAliases Add deployment host aliases +## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ +## +hostAliases: [] +## @param commonLabels Labels to add to all deployed objects +## +commonLabels: {} +## @param commonAnnotations Annotations to add to all deployed objects +## +commonAnnotations: {} +## @param clusterDomain Default Kubernetes cluster domain +## +clusterDomain: cluster.local +## @param extraDeploy Array of extra objects to deploy with the release +## +extraDeploy: [] + +## Enable diagnostic mode in the deployment +## +diagnosticMode: + ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden) + ## + enabled: false + ## @param diagnosticMode.command Command to override all containers in the deployment + ## + command: + - sleep + ## @param diagnosticMode.args Args to override all containers in the deployment + ## + args: + - infinity + +## @section Mayastor common parameters + +## Mayastor image +## ref: https://hub.docker.com/r/mayadata/mayastor/tags +## @param image.registry Adminer image registry +## @param image.repository Adminer image repository +## @param image.tag Adminer image tag (immutable tags are recommended) +## @param image.pullPolicy Adminer image pull policy +## @param image.pullSecrets Specify docker-registry secret names as an array +## @param image.debug Specify if debug logs should be enabled +## +image: + registry: docker.io + repository: mayadata/mayastor + tag: release-1.0.2 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Set to true if you would like to see extra information on logs + ## It turns BASH and/or NAMI debugging in the image + ## + debug: false + +csi: + enabled: true + + ## Mayastor CSI image + ## ref: https://hub.docker.com/r/mayadata/mayastor/tags + ## @param image.registry Adminer image registry + ## @param image.repository Adminer image repository + ## @param image.tag Adminer image tag (immutable tags are recommended) + ## @param image.pullPolicy Adminer image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + ## @param image.debug Specify if debug logs should be enabled + ## + image: + registry: docker.io + repository: mayadata/mayastor-csi-node + tag: develop + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Set to true if you would like to see extra information on logs + ## It turns BASH and/or NAMI debugging in the image + ## + debug: false + + nvme: + # nvme_core module io timeout in seconds + io_timeout: "30" + io_timeout_enabled: true + +etcd: + enabled: true + ## Number of replicas + ## + replicaCount: 1 + ## Kubernetes Cluster Domain + ## + clusterDomain: cluster.local + ## TLS authentication for client-to-server communications + ## ref: https://etcd.io/docs/current/op-guide/security/ + ## + client: + secureTransport: false + ## TLS authentication for server-to-server communications + ## ref: https://etcd.io/docs/current/op-guide/security/ + ## + peer: + secureTransport: false + ## Enable persistence using Persistent Volume Claims + ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + ## If true, use a Persistent Volume Claim. If false, use emptyDir. + ## + enabled: true + ## Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + storageClass: "manual" + ## Persistent Volume size + ## + size: 2Gi + + ## Init containers parameters: + ## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. + ## + volumePermissions: + enabled: true + + ## Set to true if you would like to see extra information on logs + ## + debug: false + + ## Pod anti-affinity preset + ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## Allowed values: soft, hard + ## + podAntiAffinityPreset: "hard" + +nats: + cluster: + enabled: false + exporter: + enabled: false + logging: + debug: false + trace: false + +mayastorCpuCount: "1" +mayastorHugePagesGiB: "1" +mayastorImagesRegistry: "" +mayastorPools: + - node: "NODE_NAME" + device: "DEVICE" +# This option is intended for development yamls and motivated by the problem of +# moac that does not update status of msp resource in some cases. Feel free to +# remove when no longer needed. +moacDebug: false +moac: false + +## @section RBAC parameter +# + +## Specifies whether a ServiceAccount should be created +## +serviceAccount: + ## @param serviceAccount.create Enable the creation of a ServiceAccount for Adminer pods + ## + create: true + ## @param serviceAccount.name Name of the created ServiceAccount + ## If not set and create is true, a name is generated using the fullname template + ## + name: "" + ## @param serviceAccount.automountServiceAccountToken Auto-mount the service account token in the pod + ## + automountServiceAccountToken: false + ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount + ## + annotations: {} +## Role Based Access +## Ref: https://kubernetes.io/docs/admin/authorization/rbac/ +## +rbac: + ## @param rbac.create Specify whether RBAC resources should be created and used + ## + create: true + ## @param rbac.rules Custom RBAC rules + ## Example: + ## rules: + ## - apiGroups: + ## - "" + ## resources: + ## - pods + ## verbs: + ## - get + ## - list + ## + rules: [] + From 4786453106b81c5c039d1f4c98ec29bdcf1ea253 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 13:52:41 +0700 Subject: [PATCH 002/188] Add additional components --- charts/mayastor/templates/DaemonSet.yaml | 2 +- charts/mayastor/templates/MayastorPool.yaml | 1 - charts/mayastor/templates/_helper.tpl | 29 +++ .../templates/csi-controller/Deployment.yaml | 95 +++++++ charts/mayastor/templates/csi/DaemonSet.yaml | 1 + .../{core-agents => mcp-core}/Deployment.yaml | 16 +- .../{rest => mcp-rest}/Deployment.yaml | 26 +- .../templates/{rest => mcp-rest}/Service.yaml | 40 +-- .../templates/msp-operator/Deployment.yaml | 72 ++++++ charts/mayastor/values-test.yaml | 4 +- charts/mayastor/values.yaml | 244 ++++++++++++++++-- 11 files changed, 472 insertions(+), 58 deletions(-) create mode 100644 charts/mayastor/templates/csi-controller/Deployment.yaml rename charts/mayastor/templates/{core-agents => mcp-core}/Deployment.yaml (80%) rename charts/mayastor/templates/{rest => mcp-rest}/Deployment.yaml (72%) rename charts/mayastor/templates/{rest => mcp-rest}/Service.yaml (55%) create mode 100644 charts/mayastor/templates/msp-operator/Deployment.yaml diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index 591bcb7e..36ce4dac 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -66,7 +66,7 @@ spec: # If you use 2 CPUs, the CPU: field should also read 2. - "-N$(MY_NODE_NAME)" - "-g$(MY_POD_IP)" - - "-nnats" + - "-nmayastor-nats" - "-y/var/local/mayastor/config.yaml" - "-l{{ include "mayastorCpuSpec" . }}" - "-pmayastor-etcd" diff --git a/charts/mayastor/templates/MayastorPool.yaml b/charts/mayastor/templates/MayastorPool.yaml index 8377f82e..7203d45f 100644 --- a/charts/mayastor/templates/MayastorPool.yaml +++ b/charts/mayastor/templates/MayastorPool.yaml @@ -1,5 +1,4 @@ {{- range .Values.mayastorPools }} ---- apiVersion: "openebs.io/v1alpha1" kind: MayastorPool metadata: diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index ab436907..79e3a050 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -1,13 +1,42 @@ +{{/* Create the name of the service account to use for the deployment */}} +{{- define "mayastor.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (printf "%s" (include "common.names.fullname" .)) .Values.serviceAccount.name | trunc 63 | trimSuffix "-" }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + {{/* Return the proper Mayastor image name */}} {{- define "mayastor.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }} {{- end -}} +{{/* Return the proper Mayastor Rest image name */}} +{{- define "mayastor.coreAgents.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.coreAgents.image "global" .Values.global) }} +{{- end -}} + {{/* Return the proper Mayastor CSI image name */}} {{- define "mayastor.csi.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.csi.image "global" .Values.global) }} {{- end -}} +{{/* Return the proper Mayastor CSI image name */}} +{{- define "mayastor.csiController.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} +{{- end -}} + +{{/* Return the proper Mayastor Rest image name */}} +{{- define "mayastor.mcpRest.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.mcpRest.image "global" .Values.global) }} +{{- end -}} + +{{/* Return the proper Mayastor Operator image name */}} +{{- define "mayastor.mspOperator.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.mspOperator.image "global" .Values.global) }} +{{- end -}} + {{/* Return the proper image name (for the init container volume-permissions image) */}} {{- define "mayastor.volumePermissions.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.volumePermissions.image "global" .Values.global) }} diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml new file mode 100644 index 00000000..7b4d4bd0 --- /dev/null +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -0,0 +1,95 @@ +## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/csi-deployment.yaml + +{{- if .Values.rest.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ printf "%s-csi-controller" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: csi-controller + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: csi-controller + template: + metadata: + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "common.labels.standard" . | nindent 8 }} + app.kubernetes.io/component: csi-controller + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + spec: + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + serviceAccountName: {{ include "mayastor.serviceAccountName" . }} + {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + initContainers: + - command: + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-rest 8081; do echo "Waiting for REST API endpoint + to become available"; sleep 1; done; + image: busybox:latest + name: rest-probe + containers: + - name: csi-provisioner + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1 + args: + - "--v=2" + - "--csi-address=$(ADDRESS)" + - "--feature-gates=Topology=true" + - "--strict-topology=false" + - "--default-fstype=ext4" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1 + args: + - "--v=2" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-controller + resources: + limits: + cpu: 32m + memory: 128Mi + requests: + cpu: 16m + memory: 64Mi + image: {{ include "mayastor.csiController.image" . }} + imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} + args: + - "--csi-socket=/var/lib/csi/sockets/pluginproxy/csi.sock" + - "--rest-endpoint=http://mayastor-rest:8081" + env: + - name: RUST_LOG + value: info + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: \ No newline at end of file diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index c0d84bde..3e77700b 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -32,6 +32,7 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: + {{- include "mayastor.imagePullSecrets" . | nindent 6 }} hostNetwork: true nodeSelector: kubernetes.io/arch: amd64 diff --git a/charts/mayastor/templates/core-agents/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml similarity index 80% rename from charts/mayastor/templates/core-agents/Deployment.yaml rename to charts/mayastor/templates/mcp-core/Deployment.yaml index f4c58d7a..e67cad54 100644 --- a/charts/mayastor/templates/core-agents/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -1,5 +1,6 @@ ## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/core-agents-deployment.yaml +{{- if .Values.core-agents.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -30,13 +31,13 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: - imagePullSecrets: - - name: regcred + {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "mayastor.serviceAccountName" . }} initContainers: - command: - sh - -c - - trap "exit 1" TERM; until nc -vz nats 4222; do echo "Waiting for nats..."; sleep + - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep 1; done; image: busybox:latest name: nats-probe @@ -56,11 +57,11 @@ spec: requests: cpu: 500m memory: 16Mi - image: mayadata/mcp-core:v1.0.1 - imagePullPolicy: IfNotPresent + image: {{ include "mayastor.coreAgents.image" . }} + imagePullPolicy: {{ .Values.coreAgents.image.pullPolicy | quote }} args: - "-smayastor-etcd" - - "-nnats" + - "-nmayastor-nats" - "--request-timeout=5s" - "--cache-period=30s" env: @@ -73,4 +74,5 @@ spec: - name: MY_POD_NAMESPACE valueFrom: fieldRef: - fieldPath: metadata.namespace \ No newline at end of file + fieldPath: metadata.namespace +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/rest/Deployment.yaml b/charts/mayastor/templates/mcp-rest/Deployment.yaml similarity index 72% rename from charts/mayastor/templates/rest/Deployment.yaml rename to charts/mayastor/templates/mcp-rest/Deployment.yaml index 0e274619..b24b8042 100644 --- a/charts/mayastor/templates/rest/Deployment.yaml +++ b/charts/mayastor/templates/mcp-rest/Deployment.yaml @@ -1,12 +1,13 @@ ## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/rest-deployment.yaml +{{- if .Values.mcpRest.enabled }} apiVersion: apps/v1 kind: Deployment metadata: name: {{ printf "%s-rest" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: rest + app.kubernetes.io/component: mcp-rest {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} @@ -17,7 +18,7 @@ spec: replicas: 1 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} - app.kubernetes.io/component: rest + app.kubernetes.io/component: mcp-rest template: metadata: {{- if .Values.podAnnotations }} @@ -25,13 +26,13 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} {{- end }} labels: {{- include "common.labels.standard" . | nindent 8 }} - app.kubernetes.io/component: rest + app.kubernetes.io/component: mcp-rest {{- if .Values.podLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: - imagePullSecrets: - - name: regcred + {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "mayastor.serviceAccountName" . }} initContainers: - command: - sh @@ -56,8 +57,8 @@ spec: requests: cpu: 50m memory: 32Mi - image: mayadata/mcp-rest:v1.0.1 - imagePullPolicy: IfNotPresent + image: {{ include "mayastor.mcpRest.image" . }} + imagePullPolicy: {{ .Values.mcpRest.image.pullPolicy | quote }} args: - "--dummy-certificates" - "--no-auth" @@ -65,8 +66,13 @@ spec: - "--http=0.0.0.0:8081" - "--request-timeout=5s" ports: - - containerPort: 8080 - - containerPort: 8081 + - name: http + containerPort: {{ .Values.mcpRest.containerPorts.http }} + protocol: TCP + - name: https + containerPort: {{ .Values.mcpRest.containerPorts.https }} + protocol: TCP env: - name: RUST_LOG - value: info \ No newline at end of file + value: info +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/rest/Service.yaml b/charts/mayastor/templates/mcp-rest/Service.yaml similarity index 55% rename from charts/mayastor/templates/rest/Service.yaml rename to charts/mayastor/templates/mcp-rest/Service.yaml index 78aff08c..27ba44d6 100644 --- a/charts/mayastor/templates/rest/Service.yaml +++ b/charts/mayastor/templates/mcp-rest/Service.yaml @@ -4,7 +4,7 @@ metadata: name: {{ printf "%s-rest" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: rest + app.kubernetes.io/component: mcp-rest {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} @@ -21,24 +21,24 @@ metadata: {{- end }} {{- end }} spec: - type: {{ .Values.service.type }} - {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.allocateLoadBalancerNodePorts)) }} + type: {{ .Values.mcpRest.service.type }} + {{- if and (eq .Values.mcpRest.service.type "LoadBalancer") (not (empty .Values.service.allocateLoadBalancerNodePorts)) }} allocateLoadBalancerNodePorts: {{ .Values.service.allocateLoadBalancerNodePorts }} {{- end }} - {{- if and .Values.service.clusterIP (eq .Values.service.type "ClusterIP") }} + {{- if and .Values.service.clusterIP (eq .Values.mcpRest.service.type "ClusterIP") }} clusterIP: {{ .Values.service.clusterIP }} {{- end }} - {{- if and .Values.service.externalTrafficPolicy (or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort")) }} + {{- if and .Values.service.externalTrafficPolicy (or (eq .Values.mcpRest.service.type "LoadBalancer") (eq .Values.mcpRest.service.type "NodePort")) }} externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | quote }} {{- end }} ipFamilyPolicy: {{ .Values.service.ipFamilyPolicy }} - {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerClass)) }} + {{- if and (eq .Values.mcpRest.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerClass)) }} loadBalancerClass: {{ .Values.service.loadBalancerClass }} {{- end }} - {{- if (and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP))) }} + {{- if (and (eq .Values.mcpRest.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP))) }} loadBalancerIP: {{ .Values.service.loadBalancerIP }} {{- end }} - {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerSourceRanges)) }} + {{- if and (eq .Values.mcpRest.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerSourceRanges)) }} loadBalancerSourceRanges: {{ .Values.service.loadBalancerSourceRanges }} {{- end }} {{- if .Values.service.sessionAffinity }} @@ -49,25 +49,25 @@ spec: {{- end }} ports: - name: http - port: {{ .Values.service.ports.http }} + port: {{ .Values.mcpRest.service.ports.http }} protocol: UDP - targetPort: {{ .Values.containerPorts.http }} - {{- if (and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) .Values.service.nodePorts.http) }} - nodePort: {{ coalesce .Values.service.nodePorts.http .Values.service.nodePort }} - {{- else if eq .Values.service.type "ClusterIP" }} + targetPort: {{ .Values.mcpRest.containerPorts.http }} + {{- if (and (or (eq .Values.mcpRest.service.type "NodePort") (eq .Values.mcpRest.service.type "LoadBalancer")) .Values.mcpRest.service.nodePorts.http) }} + nodePort: {{ coalesce .Values.mcpRest.service.nodePorts.http .Values.mcpRest.service.nodePort }} + {{- else if eq .Values.mcpRest.service.type "ClusterIP" }} nodePort: null {{- end }} - {{- if .Values.tls.enabled }} + {{- if .Values.mcpRest.tls.enabled }} - name: https - port: {{ .Values.service.ports.https }} + port: {{ .Values.mcpRest.service.ports.https }} protocol: TCP - targetPort: {{ .Values.containerPorts.https }} - {{- if (and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) .Values.service.nodePorts.https) }} - nodePort: {{ .Values.service.nodePorts.https }} - {{- else if eq .Values.service.type "ClusterIP" }} + targetPort: {{ .Values.mcpRest.containerPorts.https }} + {{- if (and (or (eq .Values.mcpRest.service.type "NodePort") (eq .Values.mcpRest.service.type "LoadBalancer")) .Values.mcpRest.service.nodePorts.https) }} + nodePort: {{ .Values.mcpRest.service.nodePorts.https }} + {{- else if eq .Values.mcpRest.service.type "ClusterIP" }} nodePort: null {{- end }} {{- end }} selector: {{ include "common.labels.matchLabels" . | nindent 4 }} - app.kubernetes.io/component: rest + app.kubernetes.io/component: mcp-rest --- \ No newline at end of file diff --git a/charts/mayastor/templates/msp-operator/Deployment.yaml b/charts/mayastor/templates/msp-operator/Deployment.yaml new file mode 100644 index 00000000..2504aece --- /dev/null +++ b/charts/mayastor/templates/msp-operator/Deployment.yaml @@ -0,0 +1,72 @@ +##ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/msp-deployment.yaml + +{{- if .Values.mspOperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ printf "%s-msp-operator" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: msp-operator + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: msp-operator + template: + metadata: + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "common.labels.standard" . | nindent 8 }} + app.kubernetes.io/component: msp-operator + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + spec: + {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "mayastor.serviceAccountName" . }} + initContainers: + - command: + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep + 1; done; + image: busybox:latest + name: nats-probe + - command: + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; + sleep 1; done; + image: busybox:latest + name: etcd-probe + containers: + - name: msp-operator + resources: + limits: + cpu: 100m + memory: 32Mi + requests: + cpu: 50m + memory: 16Mi + image: {{ include "mayastor.mspOperator.image" . }} + imagePullPolicy: {{ .Values.mspOperator.image.pullPolicy | quote }} + args: + - "-e http://mayastor-rest:8081" + - "--interval=30s" + env: + - name: RUST_LOG + value: info,msp_operator=info + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 7eb3c943..656b8724 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -3,11 +3,11 @@ clusterDomain: nebula.mci.net.id etcd: clusterDomain: nebula.mci.net.id - replicaCount: 3 + replicaCount: 1 persistence: enabled: true storageClass: openebs-hostpath nats: clusterDomain: nebula.mci.net.id - replicaCount: 3 + replicaCount: 1 diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index c9d10791..13755543 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -21,10 +21,10 @@ global: ## @param kubeVersion Force target Kubernetes version (using Helm capabilities if not set) ## kubeVersion: "" -## @param nameOverride String to partially override adminer.fullname +## @param nameOverride String to partially override Mayastor Control Plane Rest.fullname ## nameOverride: "" -## @param fullnameOverride String to fully override adminer.fullname +## @param fullnameOverride String to fully override Mayastor Control Plane Rest.fullname ## fullnameOverride: "" ## @param hostAliases Add deployment host aliases @@ -63,10 +63,10 @@ diagnosticMode: ## Mayastor image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags -## @param image.registry Adminer image registry -## @param image.repository Adminer image repository -## @param image.tag Adminer image tag (immutable tags are recommended) -## @param image.pullPolicy Adminer image pull policy +## @param image.registry Mayastor Control Plane Rest image registry +## @param image.repository Mayastor Control Plane Rest image repository +## @param image.tag Mayastor Control Plane Rest image tag (immutable tags are recommended) +## @param image.pullPolicy Mayastor Control Plane Rest image pull policy ## @param image.pullSecrets Specify docker-registry secret names as an array ## @param image.debug Specify if debug logs should be enabled ## @@ -91,15 +91,42 @@ image: ## debug: false -csi: +coreAgents: enabled: true + ## Mayastor Control Plane Core Agent image + ## ref: https://hub.docker.com/r/mayadata/mayastor/tags + ## @param image.registry Mayastor Control Plane Rest image registry + ## @param image.repository Mayastor Control Plane Rest image repository + ## @param image.tag Mayastor Control Plane Rest image tag (immutable tags are recommended) + ## @param image.pullPolicy Mayastor Control Plane Rest image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + ## @param image.debug Specify if debug logs should be enabled + ## + image: + registry: docker.io + repository: mayadata/mcp-core + tag: release-1.0.2 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] +csi: + enabled: true ## Mayastor CSI image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags - ## @param image.registry Adminer image registry - ## @param image.repository Adminer image repository - ## @param image.tag Adminer image tag (immutable tags are recommended) - ## @param image.pullPolicy Adminer image pull policy + ## @param image.registry Mayastor Control Plane Rest image registry + ## @param image.repository Mayastor Control Plane Rest image repository + ## @param image.tag Mayastor Control Plane Rest image tag (immutable tags are recommended) + ## @param image.pullPolicy Mayastor Control Plane Rest image pull policy ## @param image.pullSecrets Specify docker-registry secret names as an array ## @param image.debug Specify if debug logs should be enabled ## @@ -119,16 +146,40 @@ csi: ## - myRegistryKeySecretName ## pullSecrets: [] - ## Set to true if you would like to see extra information on logs - ## It turns BASH and/or NAMI debugging in the image - ## - debug: false nvme: # nvme_core module io timeout in seconds io_timeout: "30" io_timeout_enabled: true +csiController: + enabled: true + ## Mayastor Control Plane Core Agent image + ## ref: https://hub.docker.com/r/mayadata/mayastor/tags + ## @param image.registry Mayastor Control Plane Rest image registry + ## @param image.repository Mayastor Control Plane Rest image repository + ## @param image.tag Mayastor Control Plane Rest image tag (immutable tags are recommended) + ## @param image.pullPolicy Mayastor Control Plane Rest image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + ## @param image.debug Specify if debug logs should be enabled + ## + image: + registry: docker.io + repository: mayadata/mcp-csi-controller + tag: release-1.0.2 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + etcd: enabled: true ## Number of replicas @@ -183,6 +234,41 @@ etcd: podAntiAffinityPreset: "hard" nats: + enabled: true + ## Number of replicas + ## + replicaCount: 1 + ## Kubernetes Cluster Domain + ## + clusterDomain: cluster.local + ## TLS authentication for client-to-server communications + ## ref: https://etcd.io/docs/current/op-guide/security/ + ## + client: + secureTransport: false + ## TLS authentication for server-to-server communications + ## ref: https://etcd.io/docs/current/op-guide/security/ + ## + peer: + secureTransport: false + ## Enable persistence using Persistent Volume Claims + ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + ## If true, use a Persistent Volume Claim. If false, use emptyDir. + ## + enabled: true + ## Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + storageClass: "manual" + ## Persistent Volume size + ## + size: 2Gi cluster: enabled: false exporter: @@ -191,9 +277,133 @@ nats: debug: false trace: false +mcpRest: + enabled: true + ## Mayastor Control Plane Rest image + ## ref: https://hub.docker.com/r/mayadata/mcp-rest/tags + ## @param image.registry Mayastor Control Plane Rest image registry + ## @param image.repository Mayastor Control Plane Rest image repository + ## @param image.tag Mayastor Control Plane Rest image tag (immutable tags are recommended) + ## @param image.pullPolicy Mayastor Control Plane Rest image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + ## @param image.debug Specify if debug logs should be enabled + ## + image: + registry: docker.io + repository: mayadata/mcp-rest + tag: release-1.0.2 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param mcpRest.containerPorts.auth Mayastor Control Plane Rest HTTP container port + ## @param mcpRest.containerPorts.acct Mayastor Control Plane Rest HTTPS container port + ## + containerPorts: + http: 8080 + https: 8081 + tls: + enabled: true + ## Mayastor Control Plane Rest service parameters + ## + service: + ## @param service.type Mayastor Control Plane Rest Kubernetes service type + ## + type: ClusterIP + ## @param service.ports.auth Mayastor Control Plane Rest Kubernetes service port + ## + ports: + http: 8080 + https: 8081 + ## @param service.nodePorts.mysql Mayastor Control Plane Rest Kubernetes service node port + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + http: "" + https: "" + ## @param service.clusterIP Mayastor Control Plane Rest Kubernetes service clusterIP IP + ## + clusterIP: "" + ## @param service.loadBalancerIP Mayastor Control Plane Rest loadBalancerIP if service type is `LoadBalancer` + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + loadBalancerIP: "" + ## @param service.ipFamilyPolicy Mayastor Control Plane Rest Kubernetes service ipFamilyPolicy policy + ## + ipFamilyPolicy: SingleStack + ## @param service.externalTrafficPolicy Enable client source IP preservation + ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + ## @param service.allocateLoadBalancerNodePorts Allow users to disable node ports for Service Type=LoadBalancer. This is useful for + ## bare metal / on-prem environments that rely on VIP based LB implementations. + allocateLoadBalancerNodePorts: false + ## @param service.loadBalancerClass Enables to use a load balancer implementation other than the cloud provider default. + ## https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-class + ## + loadBalancerClass: "" + ## @param service.loadBalancerSourceRanges Address that are allowed when Mayastor Control Plane Rest service is LoadBalancer + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## E.g. + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## @param service.extraPorts Extra ports to expose (normally used with the `sidecar` value) + ## + extraPorts: [] + ## @param service.annotations Provide any additional annotations which may be required + ## + annotations: {} + ## @param service.sessionAffinity Session Affinity for Kubernetes service, can be "None" or "ClientIP" + ## If "ClientIP", consecutive client requests will be directed to the same Pod + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + ## + sessionAffinity: None + ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity + ## sessionAffinityConfig: + ## clientIP: + ## timeoutSeconds: 300 + sessionAffinityConfig: {} + +mspOperator: + enabled: true + ## Mayastor Control Plane MSP Operator image + ## ref: https://hub.docker.com/r/mayadata/mcp-msp-operator/tags + ## @param image.registry Mayastor Control Plane MSP Operator image registry + ## @param image.repository Mayastor Control Plane MSP Operator image repository + ## @param image.tag Mayastor Control Plane MSP Operator image tag (immutable tags are recommended) + ## @param image.pullPolicy Mayastor Control Plane MSP Operator image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + ## @param image.debug Specify if debug logs should be enabled + ## + image: + registry: docker.io + repository: mayadata/mcp-msp-operator + tag: release-1.0.2 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + mayastorCpuCount: "1" mayastorHugePagesGiB: "1" -mayastorImagesRegistry: "" mayastorPools: - node: "NODE_NAME" device: "DEVICE" @@ -209,7 +419,7 @@ moac: false ## Specifies whether a ServiceAccount should be created ## serviceAccount: - ## @param serviceAccount.create Enable the creation of a ServiceAccount for Adminer pods + ## @param serviceAccount.create Enable the creation of a ServiceAccount for Mayastor Control Plane Rest pods ## create: true ## @param serviceAccount.name Name of the created ServiceAccount From 5ba26800a9abb2d7d69534bcb0cfa422d1f08d1f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 13:57:38 +0700 Subject: [PATCH 003/188] Fix mcpCore --- charts/mayastor/templates/_helper.tpl | 10 +++++----- charts/mayastor/templates/mcp-core/Deployment.yaml | 6 +++--- charts/mayastor/values.yaml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index 79e3a050..0c62c57a 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -12,11 +12,6 @@ {{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }} {{- end -}} -{{/* Return the proper Mayastor Rest image name */}} -{{- define "mayastor.coreAgents.image" -}} - {{ include "common.images.image" (dict "imageRoot" .Values.coreAgents.image "global" .Values.global) }} -{{- end -}} - {{/* Return the proper Mayastor CSI image name */}} {{- define "mayastor.csi.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.csi.image "global" .Values.global) }} @@ -27,6 +22,11 @@ {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} {{- end -}} +{{/* Return the proper Mayastor Rest image name */}} +{{- define "mayastor.mcpCore.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.mcpCore.image "global" .Values.global) }} +{{- end -}} + {{/* Return the proper Mayastor Rest image name */}} {{- define "mayastor.mcpRest.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.mcpRest.image "global" .Values.global) }} diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index e67cad54..90c2ece2 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -1,6 +1,6 @@ ## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/core-agents-deployment.yaml -{{- if .Values.core-agents.enabled }} +{{- if .Values.mcpCore.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -57,8 +57,8 @@ spec: requests: cpu: 500m memory: 16Mi - image: {{ include "mayastor.coreAgents.image" . }} - imagePullPolicy: {{ .Values.coreAgents.image.pullPolicy | quote }} + image: {{ include "mayastor.mcpCore.image" . }} + imagePullPolicy: {{ .Values.mcpCore.image.pullPolicy | quote }} args: - "-smayastor-etcd" - "-nmayastor-nats" diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 13755543..cd4655dd 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -91,7 +91,7 @@ image: ## debug: false -coreAgents: +mcpCore: enabled: true ## Mayastor Control Plane Core Agent image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags From d235443330a5229d5c48c49797283f54edd069ce Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:09:49 +0700 Subject: [PATCH 004/188] Add replica values --- charts/mayastor/templates/DaemonSet.yaml | 1 + charts/mayastor/templates/csi/DaemonSet.yaml | 103 +++++++------- charts/mayastor/values-test.yaml | 2 +- charts/mayastor/values.yaml | 137 +++++++++++++++---- 4 files changed, 163 insertions(+), 80 deletions(-) diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index 36ce4dac..9a849d50 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -12,6 +12,7 @@ metadata: annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} spec: + replicas: {{ .Values.replicaCount }} selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: mayastor diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index 3e77700b..28068f98 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -12,6 +12,7 @@ metadata: annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} spec: + replicas: {{ .Values.replicaCount }} selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: mayastor-csi @@ -71,21 +72,21 @@ spec: {{- end }} - "-v" command: - - mayastor-csi + - mayastor-csi volumeMounts: - - name: device - mountPath: /dev - - name: sys - mountPath: /sys - - name: run-udev - mountPath: /run/udev - - name: host-root - mountPath: /host - - name: plugin-dir - mountPath: /csi - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" + - name: device + mountPath: /dev + - name: sys + mountPath: /sys + - name: run-udev + mountPath: /run/udev + - name: host-root + mountPath: /host + - name: plugin-dir + mountPath: /csi + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" resources: limits: cpu: "100m" @@ -96,13 +97,13 @@ spec: - name: csi-driver-registrar image: quay.io/k8scsi/csi-node-driver-registrar:v2.1.0 args: - - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/mayastor.openebs.io/csi.sock" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/mayastor.openebs.io/csi.sock" volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration resources: limits: cpu: "100m" @@ -112,35 +113,35 @@ spec: memory: "50Mi" # Mayastor node plugin gRPC server ports: - - containerPort: 10199 - protocol: TCP - name: mayastor-node + - containerPort: 10199 + name: mayastor-node + protocol: TCP volumes: - - name: device - hostPath: - path: /dev - type: Directory - - name: sys - hostPath: - path: /sys - type: Directory - - name: run-udev - hostPath: - path: /run/udev - type: Directory - - name: host-root - hostPath: - path: / - type: Directory - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/mayastor.openebs.io/ - type: DirectoryOrCreate - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory \ No newline at end of file + - name: device + hostPath: + path: /dev + type: Directory + - name: sys + hostPath: + path: /sys + type: Directory + - name: run-udev + hostPath: + path: /run/udev + type: Directory + - name: host-root + hostPath: + path: / + type: Directory + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/mayastor.openebs.io/ + type: DirectoryOrCreate + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory \ No newline at end of file diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 656b8724..05b2a208 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -5,7 +5,7 @@ etcd: clusterDomain: nebula.mci.net.id replicaCount: 1 persistence: - enabled: true + enabled: false storageClass: openebs-hostpath nats: diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index cd4655dd..2628de98 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -91,33 +91,86 @@ image: ## debug: false -mcpCore: - enabled: true - ## Mayastor Control Plane Core Agent image - ## ref: https://hub.docker.com/r/mayadata/mayastor/tags - ## @param image.registry Mayastor Control Plane Rest image registry - ## @param image.repository Mayastor Control Plane Rest image repository - ## @param image.tag Mayastor Control Plane Rest image tag (immutable tags are recommended) - ## @param image.pullPolicy Mayastor Control Plane Rest image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - ## @param image.debug Specify if debug logs should be enabled +## @section Mayastor Deployment parameters + +## @param replicaCount Desired number of cluster nodes +## +replicaCount: 1 +## @param updateStrategy.type updateStrategy for Mayastor Daemonset +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies +## +updateStrategy: + type: RollingUpdate +## @param podLabels Extra labels for Mayastor pods +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +## +podLabels: {} +## @param podAnnotations Annotations for Mayastor pods +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +## +podAnnotations: {} +## @param podAffinityPreset Pod affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard` +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity +## +podAffinityPreset: "" +## @param podAntiAffinityPreset Pod anti-affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard` +## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity +## +podAntiAffinityPreset: soft +## Node affinity preset +## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity +## +nodeAffinityPreset: + ## @param nodeAffinityPreset.type Node affinity preset type. Ignored if `affinity` is set. Allowed values: `soft` or `hard` ## - image: - registry: docker.io - repository: mayadata/mcp-core - tag: release-1.0.2 - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] + type: "" + ## @param nodeAffinityPreset.key Node label key to match. Ignored if `affinity` is set. + ## E.g. + ## key: "kubernetes.io/e2e-az-name" + ## + key: "" + ## @param nodeAffinityPreset.values Node label values to match. Ignored if `affinity` is set. + ## E.g. + ## values: + ## - e2e-az1 + ## - e2e-az2 + ## + values: [] +## @param affinity Affinity for Mayastor pods assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set +## +affinity: {} +## @param nodeSelector Node labels for Mayastor pods assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} +## @param tolerations Tolerations for Mayastor pods assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] +## @param topologySpreadConstraints Topology Spread Constraints for Mayastor pods assignment +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## E.g. +## topologySpreadConstraints: +## - maxSkew: 1 +## topologyKey: topology.kubernetes.io/zone +## whenUnsatisfiable: DoNotSchedule +## +topologySpreadConstraints: {} + +## @param priorityClassName Priority class for Mayastor pods assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +## +priorityClassName: "" +## @param schedulerName Name of the k8s scheduler (other than default) +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +schedulerName: "" +## @param podManagementPolicy podManagementPolicy to manage scaling operation of Mayastor pods +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies +## +podManagementPolicy: "" csi: enabled: true @@ -132,8 +185,8 @@ csi: ## image: registry: docker.io - repository: mayadata/mayastor-csi-node - tag: develop + repository: mayadata/mayastor + tag: release-1.0.2 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -277,6 +330,34 @@ nats: debug: false trace: false +mcpCore: + enabled: true + ## Mayastor Control Plane Core Agent image + ## ref: https://hub.docker.com/r/mayadata/mayastor/tags + ## @param image.registry Mayastor Control Plane Rest image registry + ## @param image.repository Mayastor Control Plane Rest image repository + ## @param image.tag Mayastor Control Plane Rest image tag (immutable tags are recommended) + ## @param image.pullPolicy Mayastor Control Plane Rest image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + ## @param image.debug Specify if debug logs should be enabled + ## + image: + registry: docker.io + repository: mayadata/mcp-core + tag: release-1.0.2 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + mcpRest: enabled: true ## Mayastor Control Plane Rest image From e599e29a78d5274b4c07d78b4a1b5f364ddb0d6f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:16:33 +0700 Subject: [PATCH 005/188] Fix csiController deployment --- charts/mayastor/templates/csi-controller/Deployment.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 7b4d4bd0..79654dfd 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -1,6 +1,6 @@ ## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/csi-deployment.yaml -{{- if .Values.rest.enabled }} +{{- if .Values.csiController.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -92,4 +92,5 @@ spec: mountPath: /var/lib/csi/sockets/pluginproxy/ volumes: - name: socket-dir - emptyDir: \ No newline at end of file + emptyDir: {} +{{- end }} \ No newline at end of file From 69029d3d240261f74f5403a78efc17bd45766af4 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:20:12 +0700 Subject: [PATCH 006/188] Add volumePermissions --- charts/mayastor/values.yaml | 99 +++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 2628de98..4c16a62f 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -172,6 +172,105 @@ schedulerName: "" ## podManagementPolicy: "" +## @section Persistence Parameters + +## Persistence Parameters +## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + ## @param persistence.enabled Enable persistence on FreeRADIUS replicas using a `PersistentVolumeClaim` + ## + enabled: false + ## @param persistence.existingClaim Name of an existing `PersistentVolumeClaim` for FreeRADIUS primary replicas + ## NOTE: When it's set the rest of persistence parameters are ignored + ## + existingClaim: "" + ## @param persistence.subPath Subdirectory of the volume to mount at + ## + subPath: "" + ## @param persistence.storageClass FreeRADIUS persistent volume storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + storageClass: "" + ## @param persistence.annotations FreeRADIUS persistent volume claim annotations + ## + annotations: {} + ## @param persistence.accessModes FreeRADIUS persistent volume access Modes + ## + accessModes: + - ReadWriteOnce + ## @param persistence.size FreeRADIUS persistent volume size + ## + size: 8Gi + ## @param persistence.selector Selector to match an existing Persistent Volume + ## selector: + ## matchLabels: + ## app: my-app + ## + selector: {} +## 'volumePermissions' init container parameters +## Changes the owner and group of the persistent volume mount point to runAsUser:fsGroup values +## based on the podSecurityContext/containerSecurityContext parameters +## +volumePermissions: + ## @param volumePermissions.enabled Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` + ## + enabled: false + ## Bitnami Shell image + ## ref: https://hub.docker.com/r/bitnami/bitnami-shell/tags/ + ## @param volumePermissions.image.registry Bitnami Shell image registry + ## @param volumePermissions.image.repository Bitnami Shell image repository + ## @param volumePermissions.image.tag Bitnami Shell image tag (immutable tags are recommended) + ## @param volumePermissions.image.pullPolicy Bitnami Shell image pull policy + ## @param volumePermissions.image.pullSecrets Bitnami Shell image pull secrets + ## + image: + registry: docker.io + repository: bitnami/bitnami-shell + tag: 11-debian-11-r0 + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## e.g: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Init container's resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## @param volumePermissions.resources.limits The resources limits for the init container + ## @param volumePermissions.resources.requests The requested resources for the init container + ## + resources: + limits: {} + requests: {} + ## Init container Container Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## @param volumePermissions.securityContext.runAsUser Set init container's Security Context runAsUser + ## NOTE: when runAsUser is set to special value "auto", init container will try to chown the + ## data folder to auto-determined user&group, using commands: `id -u`:`id -G | cut -d" " -f2` + ## "auto" is especially useful for OpenShift which has scc with dynamic user ids (and 0 is not allowed) + ## + securityContext: + runAsUser: 0 +## @param extraVolumes Optionally specify extra list of additional volumes to the FreeRADIUS pod(s) +## +extraVolumes: [] +## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for the FreeRADIUS container(s) +## +extraVolumeMounts: [] +## @param initContainers Add additional init containers for the FreeRADIUS pod(s) +## +initContainers: [] +## @param sidecars Add additional sidecar containers for the FreeRADIUS pod(s) +## +sidecars: [] + csi: enabled: true ## Mayastor CSI image From e4e6c065cd4d2d8ee4404f811ef0b60c713f246d Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:23:17 +0700 Subject: [PATCH 007/188] Add global service --- charts/mayastor/values.yaml | 84 ++++++++++++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 10 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 4c16a62f..3bf14c9f 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -178,17 +178,17 @@ podManagementPolicy: "" ## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/ ## persistence: - ## @param persistence.enabled Enable persistence on FreeRADIUS replicas using a `PersistentVolumeClaim` + ## @param persistence.enabled Enable persistence on Mayastor replicas using a `PersistentVolumeClaim` ## enabled: false - ## @param persistence.existingClaim Name of an existing `PersistentVolumeClaim` for FreeRADIUS primary replicas + ## @param persistence.existingClaim Name of an existing `PersistentVolumeClaim` for Mayastor primary replicas ## NOTE: When it's set the rest of persistence parameters are ignored ## existingClaim: "" ## @param persistence.subPath Subdirectory of the volume to mount at ## subPath: "" - ## @param persistence.storageClass FreeRADIUS persistent volume storage Class + ## @param persistence.storageClass Mayastor persistent volume storage Class ## If defined, storageClassName: ## If set to "-", storageClassName: "", which disables dynamic provisioning ## If undefined (the default) or set to null, no storageClassName spec is @@ -196,14 +196,14 @@ persistence: ## GKE, AWS & OpenStack) ## storageClass: "" - ## @param persistence.annotations FreeRADIUS persistent volume claim annotations + ## @param persistence.annotations Mayastor persistent volume claim annotations ## annotations: {} - ## @param persistence.accessModes FreeRADIUS persistent volume access Modes + ## @param persistence.accessModes Mayastor persistent volume access Modes ## accessModes: - ReadWriteOnce - ## @param persistence.size FreeRADIUS persistent volume size + ## @param persistence.size Mayastor persistent volume size ## size: 8Gi ## @param persistence.selector Selector to match an existing Persistent Volume @@ -258,19 +258,83 @@ volumePermissions: ## securityContext: runAsUser: 0 -## @param extraVolumes Optionally specify extra list of additional volumes to the FreeRADIUS pod(s) +## @param extraVolumes Optionally specify extra list of additional volumes to the Mayastor pod(s) ## extraVolumes: [] -## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for the FreeRADIUS container(s) +## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Mayastor container(s) ## extraVolumeMounts: [] -## @param initContainers Add additional init containers for the FreeRADIUS pod(s) +## @param initContainers Add additional init containers for the Mayastor pod(s) ## initContainers: [] -## @param sidecars Add additional sidecar containers for the FreeRADIUS pod(s) +## @param sidecars Add additional sidecar containers for the Mayastor pod(s) ## sidecars: [] +## @section Traffic Exposure Parameters + +## Mayastor service parameters +## +service: + ## @param service.type Mayastor Kubernetes service type + ## + type: ClusterIP + ## @param service.ports.auth Mayastor Kubernetes service port + ## + ports: + http: 1812 + https: 1813 + ## @param service.nodePorts.mysql Mayastor Kubernetes service node port + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + http: "" + https: "" + ## @param service.clusterIP Mayastor Kubernetes service clusterIP IP + ## + clusterIP: "" + ## @param service.loadBalancerIP Mayastor loadBalancerIP if service type is `LoadBalancer` + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + loadBalancerIP: "" + ## @param service.ipFamilyPolicy Mayastor Kubernetes service ipFamilyPolicy policy + ## + ipFamilyPolicy: SingleStack + ## @param service.externalTrafficPolicy Enable client source IP preservation + ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + ## @param service.allocateLoadBalancerNodePorts Allow users to disable node ports for Service Type=LoadBalancer. This is useful for + ## bare metal / on-prem environments that rely on VIP based LB implementations. + allocateLoadBalancerNodePorts: false + ## @param service.loadBalancerClass Enables to use a load balancer implementation other than the cloud provider default. + ## https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-class + ## + loadBalancerClass: "" + ## @param service.loadBalancerSourceRanges Address that are allowed when Mayastor service is LoadBalancer + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## E.g. + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## @param service.extraPorts Extra ports to expose (normally used with the `sidecar` value) + ## + extraPorts: [] + ## @param service.annotations Provide any additional annotations which may be required + ## + annotations: {} + ## @param service.sessionAffinity Session Affinity for Kubernetes service, can be "None" or "ClientIP" + ## If "ClientIP", consecutive client requests will be directed to the same Pod + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + ## + sessionAffinity: None + ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity + ## sessionAffinityConfig: + ## clientIP: + ## timeoutSeconds: 300 + sessionAffinityConfig: {} + csi: enabled: true ## Mayastor CSI image From 4ce79fa60c9262ad4fcc087903e1ca2553883fda Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:25:26 +0700 Subject: [PATCH 008/188] Fix ServiceAccount namespace --- charts/mayastor/templates/ServiceAccount.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/ServiceAccount.yaml b/charts/mayastor/templates/ServiceAccount.yaml index 7f14f862..f4927760 100644 --- a/charts/mayastor/templates/ServiceAccount.yaml +++ b/charts/mayastor/templates/ServiceAccount.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "mayastor.serviceAccountName" . }} - namespace: {{ include "multus.namespace" . | quote }} + namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} From ca43dbcf298b5ff04714b49d70890a62b5964d86 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:31:14 +0700 Subject: [PATCH 009/188] Add namespaceOverride --- charts/mayastor/values.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 3bf14c9f..7e25630a 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -24,6 +24,7 @@ kubeVersion: "" ## @param nameOverride String to partially override Mayastor Control Plane Rest.fullname ## nameOverride: "" +namespaceOverride: ## @param fullnameOverride String to fully override Mayastor Control Plane Rest.fullname ## fullnameOverride: "" @@ -420,7 +421,7 @@ etcd: persistence: ## If true, use a Persistent Volume Claim. If false, use emptyDir. ## - enabled: true + enabled: false ## Persistent Volume Storage Class ## If defined, storageClassName: ## If set to "-", storageClassName: "", which disables dynamic provisioning @@ -428,7 +429,7 @@ etcd: ## set, choosing the default provisioner. (gp2 on AWS, standard on ## GKE, AWS & OpenStack) ## - storageClass: "manual" + storageClass: "" ## Persistent Volume size ## size: 2Gi @@ -473,7 +474,7 @@ nats: persistence: ## If true, use a Persistent Volume Claim. If false, use emptyDir. ## - enabled: true + enabled: false ## Persistent Volume Storage Class ## If defined, storageClassName: ## If set to "-", storageClassName: "", which disables dynamic provisioning @@ -481,7 +482,7 @@ nats: ## set, choosing the default provisioner. (gp2 on AWS, standard on ## GKE, AWS & OpenStack) ## - storageClass: "manual" + storageClass: "" ## Persistent Volume size ## size: 2Gi From e6be7d2b1ac2d142604defd021f798412777a49c Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:35:16 +0700 Subject: [PATCH 010/188] Fix MayastorPool --- charts/mayastor/templates/MayastorPool.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/MayastorPool.yaml b/charts/mayastor/templates/MayastorPool.yaml index 7203d45f..d44cdd41 100644 --- a/charts/mayastor/templates/MayastorPool.yaml +++ b/charts/mayastor/templates/MayastorPool.yaml @@ -6,7 +6,7 @@ metadata: name: pool-on-{{ .node }} # or let k8s to generate a unique pool name #generateName: pool- - namespace: {{ include "common.names.namespace" . | quote }} + namespace: {{ include "common.names.namespace" $ | quote }} spec: node: {{ .node }} # ATM only one disk device is supported (i.e. /dev/nvme1n1) From 9448a77b7a0e309281d6a10e382b5f02a861766f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:46:03 +0700 Subject: [PATCH 011/188] Fix comments --- charts/mayastor/templates/MayastorPool.yaml | 4 ---- charts/mayastor/templates/csi-controller/Deployment.yaml | 4 +++- charts/mayastor/templates/csi/DaemonSet.yaml | 1 - 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/charts/mayastor/templates/MayastorPool.yaml b/charts/mayastor/templates/MayastorPool.yaml index d44cdd41..85d04cc7 100644 --- a/charts/mayastor/templates/MayastorPool.yaml +++ b/charts/mayastor/templates/MayastorPool.yaml @@ -2,13 +2,9 @@ apiVersion: "openebs.io/v1alpha1" kind: MayastorPool metadata: - # Name can be anything as long as it is unique name: pool-on-{{ .node }} - # or let k8s to generate a unique pool name - #generateName: pool- namespace: {{ include "common.names.namespace" $ | quote }} spec: node: {{ .node }} - # ATM only one disk device is supported (i.e. /dev/nvme1n1) disks: ["{{ .device }}"] {{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 79654dfd..7e4e497b 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -1,4 +1,6 @@ -## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/csi-deployment.yaml +{{/* +ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/csi-deployment.yaml +*/}} {{- if .Values.csiController.enabled }} apiVersion: apps/v1 diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index 28068f98..6941737b 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -45,7 +45,6 @@ spec: - name: mayastor-csi image: {{ include "mayastor.csi.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} - # we need privileged because we mount filesystems and use mknod securityContext: privileged: true env: From 299124149558a117e02548fee0865a89ce5d13e0 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:49:28 +0700 Subject: [PATCH 012/188] Fix extra space on labels --- charts/mayastor/templates/csi-controller/Deployment.yaml | 2 +- charts/mayastor/templates/mcp-core/Deployment.yaml | 2 +- charts/mayastor/templates/mcp-rest/Deployment.yaml | 2 +- charts/mayastor/templates/msp-operator/Deployment.yaml | 2 +- charts/mayastor/values.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 7e4e497b..16fcdca7 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -9,7 +9,7 @@ metadata: name: {{ printf "%s-csi-controller" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: csi-controller + app.kubernetes.io/component: csi-controller {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index 90c2ece2..1f9bd87e 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -7,7 +7,7 @@ metadata: name: {{ printf "%s-core-agents" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: core-agents + app.kubernetes.io/component: core-agents {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} diff --git a/charts/mayastor/templates/mcp-rest/Deployment.yaml b/charts/mayastor/templates/mcp-rest/Deployment.yaml index b24b8042..328c7b7c 100644 --- a/charts/mayastor/templates/mcp-rest/Deployment.yaml +++ b/charts/mayastor/templates/mcp-rest/Deployment.yaml @@ -7,7 +7,7 @@ metadata: name: {{ printf "%s-rest" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: mcp-rest + app.kubernetes.io/component: mcp-rest {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} diff --git a/charts/mayastor/templates/msp-operator/Deployment.yaml b/charts/mayastor/templates/msp-operator/Deployment.yaml index 2504aece..94e1c656 100644 --- a/charts/mayastor/templates/msp-operator/Deployment.yaml +++ b/charts/mayastor/templates/msp-operator/Deployment.yaml @@ -7,7 +7,7 @@ metadata: name: {{ printf "%s-msp-operator" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: msp-operator + app.kubernetes.io/component: msp-operator {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 7e25630a..a2e3674d 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -24,7 +24,7 @@ kubeVersion: "" ## @param nameOverride String to partially override Mayastor Control Plane Rest.fullname ## nameOverride: "" -namespaceOverride: +namespaceOverride: "" ## @param fullnameOverride String to fully override Mayastor Control Plane Rest.fullname ## fullnameOverride: "" From 54727322f849cd93fee57722a8d1beedce1b5f3f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:57:05 +0700 Subject: [PATCH 013/188] Add mayastorPools --- charts/mayastor/templates/csi-controller/Deployment.yaml | 4 +--- charts/mayastor/values-test.yaml | 8 ++++++++ charts/mayastor/values.yaml | 6 +++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 16fcdca7..57e9601a 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -1,6 +1,4 @@ -{{/* -ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/csi-deployment.yaml -*/}} +##ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/csi-deployment.yaml {{- if .Values.csiController.enabled }} apiVersion: apps/v1 diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 05b2a208..9c2aed01 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -11,3 +11,11 @@ etcd: nats: clusterDomain: nebula.mci.net.id replicaCount: 1 + +mayastorPools: + - node: alberta-lin-k8m.jkt2.mci.net.id + device: /dev/sdb + - node: rachel-lin-k8w.jkt2.mci.net.id + device: /dev/sdb + - node: verus-lin-k8w.jkt2.mci.net.id + device: /dev/sdb \ No newline at end of file diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index a2e3674d..b134957a 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -649,9 +649,9 @@ mspOperator: mayastorCpuCount: "1" mayastorHugePagesGiB: "1" -mayastorPools: - - node: "NODE_NAME" - device: "DEVICE" +mayastorPools: [] +# - node: "NODE_NAME" +# device: "DEVICE" # This option is intended for development yamls and motivated by the problem of # moac that does not update status of msp resource in some cases. Feel free to # remove when no longer needed. From 340e7932206ce80a170060b5de400daea0c549ed Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 14:59:59 +0700 Subject: [PATCH 014/188] Fix MayastorPool --- charts/mayastor/templates/MayastorPool.yaml | 1 + charts/mayastor/values-test.yaml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/charts/mayastor/templates/MayastorPool.yaml b/charts/mayastor/templates/MayastorPool.yaml index 85d04cc7..c7b5ee46 100644 --- a/charts/mayastor/templates/MayastorPool.yaml +++ b/charts/mayastor/templates/MayastorPool.yaml @@ -7,4 +7,5 @@ metadata: spec: node: {{ .node }} disks: ["{{ .device }}"] +--- {{- end }} \ No newline at end of file diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 9c2aed01..07248709 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -11,6 +11,9 @@ etcd: nats: clusterDomain: nebula.mci.net.id replicaCount: 1 + persistence: + enabled: false + storageClass: openebs-hostpath mayastorPools: - node: alberta-lin-k8m.jkt2.mci.net.id From c3330d71936448f3aafd915e9e0af9c1a85589a7 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 15:17:59 +0700 Subject: [PATCH 015/188] Fix replica --- charts/mayastor/templates/DaemonSet.yaml | 1 - charts/mayastor/templates/MayastorPool.yaml | 2 ++ charts/mayastor/templates/mcp-core/Deployment.yaml | 2 +- charts/mayastor/templates/mcp-rest/Deployment.yaml | 2 +- charts/mayastor/templates/msp-operator/Deployment.yaml | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index 9a849d50..36ce4dac 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -12,7 +12,6 @@ metadata: annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} spec: - replicas: {{ .Values.replicaCount }} selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: mayastor diff --git a/charts/mayastor/templates/MayastorPool.yaml b/charts/mayastor/templates/MayastorPool.yaml index c7b5ee46..6d6f8592 100644 --- a/charts/mayastor/templates/MayastorPool.yaml +++ b/charts/mayastor/templates/MayastorPool.yaml @@ -1,3 +1,5 @@ +##ref: https://github.com/openebs/mayastor/blob/master/deploy/pool.yaml + {{- range .Values.mayastorPools }} apiVersion: "openebs.io/v1alpha1" kind: MayastorPool diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index 1f9bd87e..cd01b08c 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -15,7 +15,7 @@ metadata: annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} spec: - replicas: 1 + replicas: {{ .Values.replicaCount }} selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: core-agents diff --git a/charts/mayastor/templates/mcp-rest/Deployment.yaml b/charts/mayastor/templates/mcp-rest/Deployment.yaml index 328c7b7c..5aa628e6 100644 --- a/charts/mayastor/templates/mcp-rest/Deployment.yaml +++ b/charts/mayastor/templates/mcp-rest/Deployment.yaml @@ -15,7 +15,7 @@ metadata: annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} spec: - replicas: 1 + replicas: {{ .Values.replicaCount }} selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: mcp-rest diff --git a/charts/mayastor/templates/msp-operator/Deployment.yaml b/charts/mayastor/templates/msp-operator/Deployment.yaml index 94e1c656..8a2eb0fa 100644 --- a/charts/mayastor/templates/msp-operator/Deployment.yaml +++ b/charts/mayastor/templates/msp-operator/Deployment.yaml @@ -15,7 +15,7 @@ metadata: annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} spec: - replicas: 1 + replicas: {{ .Values.replicaCount }} selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: msp-operator From 63be07aa5aca631c1ae6721f3f7e156025f1d840 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 15:22:04 +0700 Subject: [PATCH 016/188] Add StorageClass --- charts/mayastor/templates/StorageClass.yaml | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 charts/mayastor/templates/StorageClass.yaml diff --git a/charts/mayastor/templates/StorageClass.yaml b/charts/mayastor/templates/StorageClass.yaml new file mode 100644 index 00000000..0e8d33cf --- /dev/null +++ b/charts/mayastor/templates/StorageClass.yaml @@ -0,0 +1,28 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: mayastor-1 +parameters: + repl: '1' + protocol: 'nvmf' + ioTimeout: '60' + local: 'true' + # It is recommended to use xfs for Mayastor + # fsType: 'xfs' +provisioner: io.openebs.csi-mayastor +volumeBindingMode: WaitForFirstConsumer +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: mayastor-3 +parameters: + repl: '3' + protocol: 'nvmf' + ioTimeout: '60' + local: 'true' + # It is recommended to use xfs for Mayastor + # fsType: 'xfs' +provisioner: io.openebs.csi-mayastor +volumeBindingMode: WaitForFirstConsumer +--- \ No newline at end of file From 697dcb5508051c309f3286d15bf6be189937559a Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 15:23:28 +0700 Subject: [PATCH 017/188] Fix replica DaemonSet --- charts/mayastor/templates/SC.yaml | 28 ++++++++++++++++++++ charts/mayastor/templates/csi/DaemonSet.yaml | 1 - 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 charts/mayastor/templates/SC.yaml diff --git a/charts/mayastor/templates/SC.yaml b/charts/mayastor/templates/SC.yaml new file mode 100644 index 00000000..0e8d33cf --- /dev/null +++ b/charts/mayastor/templates/SC.yaml @@ -0,0 +1,28 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: mayastor-1 +parameters: + repl: '1' + protocol: 'nvmf' + ioTimeout: '60' + local: 'true' + # It is recommended to use xfs for Mayastor + # fsType: 'xfs' +provisioner: io.openebs.csi-mayastor +volumeBindingMode: WaitForFirstConsumer +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: mayastor-3 +parameters: + repl: '3' + protocol: 'nvmf' + ioTimeout: '60' + local: 'true' + # It is recommended to use xfs for Mayastor + # fsType: 'xfs' +provisioner: io.openebs.csi-mayastor +volumeBindingMode: WaitForFirstConsumer +--- \ No newline at end of file diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index 6941737b..49ac14b6 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -12,7 +12,6 @@ metadata: annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} spec: - replicas: {{ .Values.replicaCount }} selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: mayastor-csi From 66ba580bb4ca14108eb01dbf2abe979828f0fcb7 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 15:29:44 +0700 Subject: [PATCH 018/188] Fix DaemonSet initContainer --- charts/mayastor/templates/DaemonSet.yaml | 106 +++++++++++------------ charts/mayastor/templates/SC.yaml | 28 ------ 2 files changed, 53 insertions(+), 81 deletions(-) delete mode 100644 charts/mayastor/templates/SC.yaml diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index 36ce4dac..b02fe34d 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -39,50 +39,50 @@ spec: openebs.io/engine: mayastor kubernetes.io/arch: amd64 initContainers: - - name: message-bus-probe - image: busybox:latest - command: ['sh', '-c', 'until nc -vz nats 4222; do echo "Waiting for message bus..."; sleep 1; done;'] + - name: message-bus-probe + image: busybox:latest + command: ['sh', '-c', 'until nc -vz mayastor-nats 4222; do echo "Waiting for message bus..."; sleep 1; done;'] containers: - name: mayastor image: {{ include "mayastor.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} env: - - name: RUST_LOG - value: info,mayastor={{ .Values.mayastorLogLevel }} - - name: NVMF_TCP_MAX_QUEUE_DEPTH - value: "32" - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP + - name: RUST_LOG + value: info,mayastor={{ .Values.mayastorLogLevel }} + - name: NVMF_TCP_MAX_QUEUE_DEPTH + value: "32" + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP args: - # The -l argument accepts cpu-list. Indexing starts at zero. - # For example -l 1,2,10-20 means use core 1, 2, 10 to 20. - # Note: Ensure that the CPU resources are updated accordingly. - # If you use 2 CPUs, the CPU: field should also read 2. - - "-N$(MY_NODE_NAME)" - - "-g$(MY_POD_IP)" - - "-nmayastor-nats" - - "-y/var/local/mayastor/config.yaml" - - "-l{{ include "mayastorCpuSpec" . }}" - - "-pmayastor-etcd" + # The -l argument accepts cpu-list. Indexing starts at zero. + # For example -l 1,2,10-20 means use core 1, 2, 10 to 20. + # Note: Ensure that the CPU resources are updated accordingly. + # If you use 2 CPUs, the CPU: field should also read 2. + - "-N$(MY_NODE_NAME)" + - "-g$(MY_POD_IP)" + - "-nmayastor-nats" + - "-y/var/local/mayastor/config.yaml" + - "-l{{ include "mayastorCpuSpec" . }}" + - "-pmayastor-etcd" command: - - mayastor + - mayastor securityContext: privileged: true volumeMounts: - - name: device - mountPath: /dev - - name: udev - mountPath: /run/udev - - name: dshm - mountPath: /dev/shm - - name: configlocation - mountPath: /var/local/mayastor/ + - name: device + mountPath: /dev + - name: udev + mountPath: /run/udev + - name: dshm + mountPath: /dev/shm + - name: configlocation + mountPath: /var/local/mayastor/ resources: # NOTE: Each container must have mem/cpu limits defined in order to # belong to Guaranteed QoS class, hence can never get evicted in case of @@ -100,22 +100,22 @@ spec: protocol: TCP name: mayastor volumes: - - name: device - hostPath: - path: /dev - type: Directory - - name: udev - hostPath: - path: /run/udev - type: Directory - - name: dshm - emptyDir: - medium: Memory - sizeLimit: "1Gi" - - name: hugepage - emptyDir: - medium: HugePages - - name: configlocation - hostPath: - path: /var/local/mayastor/ - type: DirectoryOrCreate \ No newline at end of file + - name: device + hostPath: + path: /dev + type: Directory + - name: udev + hostPath: + path: /run/udev + type: Directory + - name: dshm + emptyDir: + medium: Memory + sizeLimit: "1Gi" + - name: hugepage + emptyDir: + medium: HugePages + - name: configlocation + hostPath: + path: /var/local/mayastor/ + type: DirectoryOrCreate \ No newline at end of file diff --git a/charts/mayastor/templates/SC.yaml b/charts/mayastor/templates/SC.yaml deleted file mode 100644 index 0e8d33cf..00000000 --- a/charts/mayastor/templates/SC.yaml +++ /dev/null @@ -1,28 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: mayastor-1 -parameters: - repl: '1' - protocol: 'nvmf' - ioTimeout: '60' - local: 'true' - # It is recommended to use xfs for Mayastor - # fsType: 'xfs' -provisioner: io.openebs.csi-mayastor -volumeBindingMode: WaitForFirstConsumer ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: mayastor-3 -parameters: - repl: '3' - protocol: 'nvmf' - ioTimeout: '60' - local: 'true' - # It is recommended to use xfs for Mayastor - # fsType: 'xfs' -provisioner: io.openebs.csi-mayastor -volumeBindingMode: WaitForFirstConsumer ---- \ No newline at end of file From d81d3720d28ffcb2059dae214d2304502e8ed03f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 15:47:18 +0700 Subject: [PATCH 019/188] Fix hostdomain --- .../templates/csi-controller/Deployment.yaml | 14 +++++++------- .../mayastor/templates/mcp-rest/Deployment.yaml | 2 +- .../templates/msp-operator/Deployment.yaml | 16 ++++++++-------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 57e9601a..fbc12ffa 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -37,10 +37,10 @@ spec: {{- include "mayastor.imagePullSecrets" . | nindent 6 }} initContainers: - command: - - sh - - -c - - trap "exit 1" TERM; until nc -vz mayastor-rest 8081; do echo "Waiting for REST API endpoint - to become available"; sleep 1; done; + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-rest 8081; do echo "Waiting for REST API endpoint + to become available"; sleep 1; done; image: busybox:latest name: rest-probe containers: @@ -55,7 +55,7 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "IfNotPresent" + imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -67,7 +67,7 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "IfNotPresent" + imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -83,7 +83,7 @@ spec: imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} args: - "--csi-socket=/var/lib/csi/sockets/pluginproxy/csi.sock" - - "--rest-endpoint=http://mayastor-rest:8081" + - "--rest-endpoint=http://mayastor-rest:8080" env: - name: RUST_LOG value: info diff --git a/charts/mayastor/templates/mcp-rest/Deployment.yaml b/charts/mayastor/templates/mcp-rest/Deployment.yaml index 5aa628e6..bfe55af2 100644 --- a/charts/mayastor/templates/mcp-rest/Deployment.yaml +++ b/charts/mayastor/templates/mcp-rest/Deployment.yaml @@ -37,7 +37,7 @@ spec: - command: - sh - -c - - trap "exit 1" TERM; until nc -vz nats 4222; do echo "Waiting for nats..."; sleep + - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep 1; done; image: busybox:latest name: nats-probe diff --git a/charts/mayastor/templates/msp-operator/Deployment.yaml b/charts/mayastor/templates/msp-operator/Deployment.yaml index 8a2eb0fa..cc9ca3f5 100644 --- a/charts/mayastor/templates/msp-operator/Deployment.yaml +++ b/charts/mayastor/templates/msp-operator/Deployment.yaml @@ -35,17 +35,17 @@ spec: serviceAccountName: {{ include "mayastor.serviceAccountName" . }} initContainers: - command: - - sh - - -c - - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep - 1; done; + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep + 1; done; image: busybox:latest name: nats-probe - command: - - sh - - -c - - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; - sleep 1; done; + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; + sleep 1; done; image: busybox:latest name: etcd-probe containers: From 7659d74458e02c1dad49cde8db8ba7336df630b1 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 15:49:47 +0700 Subject: [PATCH 020/188] Fix command argumen --- .../mayastor/templates/mcp-core/Deployment.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index cd01b08c..43f09316 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -35,17 +35,15 @@ spec: serviceAccountName: {{ include "mayastor.serviceAccountName" . }} initContainers: - command: - - sh - - -c - - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep - 1; done; + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep 1; done; image: busybox:latest name: nats-probe - command: - - sh - - -c - - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; - sleep 1; done; + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; sleep 1; done; image: busybox:latest name: etcd-probe containers: @@ -67,6 +65,8 @@ spec: env: - name: RUST_LOG value: info + - name: RUST_BACKTRACE + value: "1" - name: MY_POD_NAME valueFrom: fieldRef: From 2bbc2ebc0e21d8eadc53424da25d71fe77ab120c Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 15:54:45 +0700 Subject: [PATCH 021/188] Commit --- charts/mayastor/values-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 07248709..e56069e0 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -5,7 +5,7 @@ etcd: clusterDomain: nebula.mci.net.id replicaCount: 1 persistence: - enabled: false + enabled: true storageClass: openebs-hostpath nats: From e743e47d82699dc691aa487fb74e51c455907c82 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 16:00:32 +0700 Subject: [PATCH 022/188] Fix log environment --- charts/mayastor/templates/DaemonSet.yaml | 12 ++++++----- charts/mayastor/templates/csi/DaemonSet.yaml | 20 +++++++++---------- .../templates/mcp-rest/Deployment.yaml | 20 +++++++++---------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index b02fe34d..a7f657f9 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -47,8 +47,6 @@ spec: image: {{ include "mayastor.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} env: - - name: RUST_LOG - value: info,mayastor={{ .Values.mayastorLogLevel }} - name: NVMF_TCP_MAX_QUEUE_DEPTH value: "32" - name: MY_NODE_NAME @@ -59,6 +57,10 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: RUST_LOG + value: info,mayastor={{ .Values.mayastorLogLevel }} + - name: RUST_BACKTRACE + value: full args: # The -l argument accepts cpu-list. Indexing starts at zero. # For example -l 1,2,10-20 means use core 1, 2, 10 to 20. @@ -96,9 +98,9 @@ spec: memory: "1Gi" hugepages-2Mi: "{{ max .Values.mayastorHugePagesGiB 2 }}Gi" ports: - - containerPort: 10124 - protocol: TCP - name: mayastor + - containerPort: 10124 + name: mayastor + protocol: TCP volumes: - name: device hostPath: diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index 49ac14b6..f3f5bc59 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -47,16 +47,16 @@ spec: securityContext: privileged: true env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: RUST_BACKTRACE - value: "1" + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: RUST_BACKTRACE + value: full {{- if .Values.moac }} - name: MOAC value: "true" diff --git a/charts/mayastor/templates/mcp-rest/Deployment.yaml b/charts/mayastor/templates/mcp-rest/Deployment.yaml index bfe55af2..a40bcc99 100644 --- a/charts/mayastor/templates/mcp-rest/Deployment.yaml +++ b/charts/mayastor/templates/mcp-rest/Deployment.yaml @@ -35,17 +35,15 @@ spec: serviceAccountName: {{ include "mayastor.serviceAccountName" . }} initContainers: - command: - - sh - - -c - - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep - 1; done; + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep 1; done; image: busybox:latest name: nats-probe - command: - - sh - - -c - - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; - sleep 1; done; + - sh + - -c + - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; sleep 1; done; image: busybox:latest name: etcd-probe containers: @@ -62,8 +60,8 @@ spec: args: - "--dummy-certificates" - "--no-auth" - - "-nnats" - - "--http=0.0.0.0:8081" + - "-nmayastor-nats" + - "--http=0.0.0.0:8080" - "--request-timeout=5s" ports: - name: http @@ -75,4 +73,6 @@ spec: env: - name: RUST_LOG value: info + - name: RUST_BACKTRACE + value: full {{- end }} \ No newline at end of file From 13550b85112bd685ce157d3b2779278efc091952 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 16:10:36 +0700 Subject: [PATCH 023/188] Adjust etcd --- charts/mayastor/values-test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index e56069e0..3488c4ee 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -4,6 +4,11 @@ clusterDomain: nebula.mci.net.id etcd: clusterDomain: nebula.mci.net.id replicaCount: 1 + auth: + rbac: + create: true + allowNoneAuthentication: true + rootPassword: password persistence: enabled: true storageClass: openebs-hostpath From bee7f3039729070b2f72ae6bf90b5f2bc12ace87 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 16:18:38 +0700 Subject: [PATCH 024/188] Fix nats values --- charts/mayastor/values-test.yaml | 2 ++ charts/mayastor/values.yaml | 14 ++++---------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 3488c4ee..7da6d513 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -15,6 +15,8 @@ etcd: nats: clusterDomain: nebula.mci.net.id + auth: + enabled: false replicaCount: 1 persistence: enabled: false diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index b134957a..4ced197a 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -458,16 +458,6 @@ nats: ## Kubernetes Cluster Domain ## clusterDomain: cluster.local - ## TLS authentication for client-to-server communications - ## ref: https://etcd.io/docs/current/op-guide/security/ - ## - client: - secureTransport: false - ## TLS authentication for server-to-server communications - ## ref: https://etcd.io/docs/current/op-guide/security/ - ## - peer: - secureTransport: false ## Enable persistence using Persistent Volume Claims ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ ## @@ -488,11 +478,15 @@ nats: size: 2Gi cluster: enabled: false + auth: + enabled: false exporter: enabled: false logging: debug: false trace: false + metrics: + enabled: false mcpCore: enabled: true From 586968d67c8541574e607f28a4db02e6c7c04404 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 16:27:47 +0700 Subject: [PATCH 025/188] Fix core deloyment --- charts/mayastor/templates/mcp-core/Deployment.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index 43f09316..40c8b131 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -63,10 +63,7 @@ spec: - "--request-timeout=5s" - "--cache-period=30s" env: - - name: RUST_LOG - value: info - - name: RUST_BACKTRACE - value: "1" + - name: MY_POD_NAME valueFrom: fieldRef: @@ -75,4 +72,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: RUST_LOG + value: info + - name: RUST_BACKTRACE + value: full {{- end }} \ No newline at end of file From d641e1c71f5348a777e650b7b8ce7316d4e1dcaf Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 16:32:45 +0700 Subject: [PATCH 026/188] Fix multiple Deployment --- .../templates/csi-controller/Deployment.yaml | 5 ++--- charts/mayastor/templates/mcp-core/Deployment.yaml | 1 - .../mayastor/templates/msp-operator/Deployment.yaml | 12 ++++++------ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index fbc12ffa..71d48cb6 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -15,7 +15,7 @@ metadata: annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} spec: - replicas: 1 + replicas: {{ .Values.replicaCount }} selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: csi-controller @@ -39,8 +39,7 @@ spec: - command: - sh - -c - - trap "exit 1" TERM; until nc -vz mayastor-rest 8081; do echo "Waiting for REST API endpoint - to become available"; sleep 1; done; + - trap "exit 1" TERM; until nc -vz mayastor-rest 8081; do echo "Waiting for REST API endpoint to become available"; sleep 1; done; image: busybox:latest name: rest-probe containers: diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index 40c8b131..1870d6a2 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -63,7 +63,6 @@ spec: - "--request-timeout=5s" - "--cache-period=30s" env: - - name: MY_POD_NAME valueFrom: fieldRef: diff --git a/charts/mayastor/templates/msp-operator/Deployment.yaml b/charts/mayastor/templates/msp-operator/Deployment.yaml index cc9ca3f5..cb35cd33 100644 --- a/charts/mayastor/templates/msp-operator/Deployment.yaml +++ b/charts/mayastor/templates/msp-operator/Deployment.yaml @@ -37,15 +37,13 @@ spec: - command: - sh - -c - - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep - 1; done; + - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep 1; done; image: busybox:latest name: nats-probe - command: - sh - -c - - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; - sleep 1; done; + - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; sleep 1; done; image: busybox:latest name: etcd-probe containers: @@ -63,10 +61,12 @@ spec: - "-e http://mayastor-rest:8081" - "--interval=30s" env: - - name: RUST_LOG - value: info,msp_operator=info - name: MY_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name + - name: RUST_LOG + value: info,msp_operator=info + - name: RUST_BACKTRACE + value: full {{- end }} \ No newline at end of file From b52bfec2fd5e326f96a158fb1ccc46f91ec32fac Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 16:46:33 +0700 Subject: [PATCH 027/188] Fix daemonset --- charts/mayastor/templates/DaemonSet.yaml | 2 ++ charts/mayastor/templates/csi-controller/Deployment.yaml | 6 ++++-- charts/mayastor/templates/csi/DaemonSet.yaml | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index a7f657f9..2e0ec773 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -33,6 +33,8 @@ spec: {{- end }} spec: hostNetwork: true + {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "mayastor.serviceAccountName" . }} # To resolve services from mayastor's namespace dnsPolicy: ClusterFirstWithHostNet nodeSelector: diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 71d48cb6..9a2815c1 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -44,7 +44,7 @@ spec: name: rest-probe containers: - name: csi-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1 + image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0 args: - "--v=2" - "--csi-address=$(ADDRESS)" @@ -59,7 +59,7 @@ spec: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1 + image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 args: - "--v=2" - "--csi-address=$(ADDRESS)" @@ -86,6 +86,8 @@ spec: env: - name: RUST_LOG value: info + - name: RUST_BACKTRACE + value: full volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index f3f5bc59..a50f2acc 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -33,6 +33,7 @@ spec: {{- end }} spec: {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "mayastor.serviceAccountName" . }} hostNetwork: true nodeSelector: kubernetes.io/arch: amd64 From 691e218316d7b51c4c1ae45365551cf6a8e274fa Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 16:52:59 +0700 Subject: [PATCH 028/188] Fix mcpRest port --- charts/mayastor/templates/mcp-rest/Deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/mcp-rest/Deployment.yaml b/charts/mayastor/templates/mcp-rest/Deployment.yaml index a40bcc99..9d40319c 100644 --- a/charts/mayastor/templates/mcp-rest/Deployment.yaml +++ b/charts/mayastor/templates/mcp-rest/Deployment.yaml @@ -61,7 +61,7 @@ spec: - "--dummy-certificates" - "--no-auth" - "-nmayastor-nats" - - "--http=0.0.0.0:8080" + - "--http=0.0.0.0:8081" - "--request-timeout=5s" ports: - name: http From 70f37936507713c3e0d791b10cbca97da3cd3e49 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 16:57:23 +0700 Subject: [PATCH 029/188] Fix command argument --- charts/mayastor/templates/csi-controller/Deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 9a2815c1..2b1c709a 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -82,7 +82,7 @@ spec: imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} args: - "--csi-socket=/var/lib/csi/sockets/pluginproxy/csi.sock" - - "--rest-endpoint=http://mayastor-rest:8080" + - "--rest-endpoint=http://mayastor-rest:8081" env: - name: RUST_LOG value: info From 6ebe886018c353ce3beee49af48b41105edcecaf Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:02:54 +0700 Subject: [PATCH 030/188] Enable automountServiceAccountToken --- charts/mayastor/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 4ced197a..c59088d6 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -667,7 +667,7 @@ serviceAccount: name: "" ## @param serviceAccount.automountServiceAccountToken Auto-mount the service account token in the pod ## - automountServiceAccountToken: false + automountServiceAccountToken: true ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount ## annotations: {} From a4ccc7d297b01dd4d744f2ce10b196b9d58853f8 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:08:36 +0700 Subject: [PATCH 031/188] Fix csi-controller --- .../mayastor/templates/csi-controller/Deployment.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 2b1c709a..74afc858 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -33,13 +33,19 @@ spec: spec: dnsPolicy: ClusterFirstWithHostNet hostNetwork: true - serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "mayastor.serviceAccountName" . }} initContainers: - command: - sh - -c - - trap "exit 1" TERM; until nc -vz mayastor-rest 8081; do echo "Waiting for REST API endpoint to become available"; sleep 1; done; + - >- + trap "exit 1" TERM + until nc -vz mayastor-rest 8081 + do + echo "Waiting for REST API endpoint to become available" + sleep 1 + done image: busybox:latest name: rest-probe containers: From 9961b029ccedf61960e28ba133113a63b02bf53a Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:11:51 +0700 Subject: [PATCH 032/188] Fix typo --- charts/mayastor/templates/csi-controller/Deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 74afc858..078ead1e 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -42,7 +42,7 @@ spec: - >- trap "exit 1" TERM until nc -vz mayastor-rest 8081 - do + do echo "Waiting for REST API endpoint to become available" sleep 1 done From 9547739c21c34a97415cdae45ac044a6b0da35c0 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:16:35 +0700 Subject: [PATCH 033/188] test: etcd --- charts/mayastor/values-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 7da6d513..137aba38 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -6,7 +6,7 @@ etcd: replicaCount: 1 auth: rbac: - create: true + create: false allowNoneAuthentication: true rootPassword: password persistence: From d8c58804cc039efde2d151b15b4473e7cc557f3f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:18:40 +0700 Subject: [PATCH 034/188] Fix initContainers --- .../mayastor/templates/csi-controller/Deployment.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 078ead1e..7c65e7fa 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -40,12 +40,12 @@ spec: - sh - -c - >- - trap "exit 1" TERM - until nc -vz mayastor-rest 8081 + trap "exit 1" TERM; + until nc -vz mayastor-rest 8081; do - echo "Waiting for REST API endpoint to become available" - sleep 1 - done + echo "Waiting for REST API endpoint to become available"; + sleep 1; + done; image: busybox:latest name: rest-probe containers: From f570cfca1a7f446bd3622130c125e2d0dae9bebe Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:29:13 +0700 Subject: [PATCH 035/188] Add serviceAccount --- charts/mayastor/values.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index c59088d6..7e34ac25 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -433,6 +433,13 @@ etcd: ## Persistent Volume size ## size: 2Gi + serviceAccount: + ## @param serviceAccount.create Enable/disable service account creation + ## + create: false + ## @param serviceAccount.name Name of the service account to create or use + ## + name: {{ include "mayastor.serviceAccountName" . }} ## Init containers parameters: ## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. @@ -476,6 +483,13 @@ nats: ## Persistent Volume size ## size: 2Gi + serviceAccount: + ## @param serviceAccount.create Enable/disable service account creation + ## + create: false + ## @param serviceAccount.name Name of the service account to create or use + ## + name: {{ include "mayastor.serviceAccountName" . }} cluster: enabled: false auth: From fd2d6cb4fc86aee6c0d89e36255d21ffd0be09ab Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:32:18 +0700 Subject: [PATCH 036/188] Fix serviceAccounr --- charts/mayastor/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 7e34ac25..7e3b1e8a 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -489,7 +489,7 @@ nats: create: false ## @param serviceAccount.name Name of the service account to create or use ## - name: {{ include "mayastor.serviceAccountName" . }} + name: {{ include "mayastor.serviceAccountName" $ }} cluster: enabled: false auth: From 7d195ecad8e5cbbfb30a339517b3268abcc568de Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:33:37 +0700 Subject: [PATCH 037/188] Fix serviceACcount --- charts/mayastor/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 7e3b1e8a..9ba26402 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -439,7 +439,7 @@ etcd: create: false ## @param serviceAccount.name Name of the service account to create or use ## - name: {{ include "mayastor.serviceAccountName" . }} + name: {{ include "mayastor.serviceAccountName" $ }} ## Init containers parameters: ## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. From 1af5b3f9d334e72078c5a3abddc311594c32e289 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:36:25 +0700 Subject: [PATCH 038/188] FIx serviceAccount --- charts/mayastor/values.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 9ba26402..89baa0c6 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -439,7 +439,8 @@ etcd: create: false ## @param serviceAccount.name Name of the service account to create or use ## - name: {{ include "mayastor.serviceAccountName" $ }} + # name: {{ include "mayastor.serviceAccountName" $ }} + name: mayastor ## Init containers parameters: ## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. @@ -489,7 +490,8 @@ nats: create: false ## @param serviceAccount.name Name of the service account to create or use ## - name: {{ include "mayastor.serviceAccountName" $ }} + # name: {{ include "mayastor.serviceAccountName" $ }} + name: mayastor cluster: enabled: false auth: From 5050ca223b5621aa25f19be9ae229915b022e2de Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:43:28 +0700 Subject: [PATCH 039/188] Commit --- charts/mayastor/values-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 137aba38..7da6d513 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -6,7 +6,7 @@ etcd: replicaCount: 1 auth: rbac: - create: false + create: true allowNoneAuthentication: true rootPassword: password persistence: From 7eede7d4928be26b32e1211b8a0b595e07952175 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:57:23 +0700 Subject: [PATCH 040/188] Add test --- .../templates/mcp-core/Deployment.yaml | 39 +++++++++++++------ charts/mayastor/values-test.yaml | 2 + 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index 1870d6a2..229276d5 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -37,26 +37,34 @@ spec: - command: - sh - -c - - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep 1; done; + - >- + trap "exit 1" TERM; + until nc -vz mayastor-nats 4222; + do + echo "Waiting for nats..."; + sleep 1; + done; image: busybox:latest name: nats-probe - command: - sh - -c - - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; sleep 1; done; + - >- + trap "exit 1" TERM; + until nc -vz mayastor-etcd 2379; + do + echo "Waiting for etcd..."; + sleep 1; + done; image: busybox:latest name: etcd-probe containers: - name: core - resources: - limits: - cpu: 1000m - memory: 32Mi - requests: - cpu: 500m - memory: 16Mi - image: {{ include "mayastor.mcpCore.image" . }} - imagePullPolicy: {{ .Values.mcpCore.image.pullPolicy | quote }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} + {{- else if .Values.command }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} + {{- end }} args: - "-smayastor-etcd" - "-nmayastor-nats" @@ -75,4 +83,13 @@ spec: value: info - name: RUST_BACKTRACE value: full + image: {{ include "mayastor.mcpCore.image" . }} + imagePullPolicy: {{ .Values.mcpCore.image.pullPolicy | quote }} + resources: + limits: + cpu: 1000m + memory: 32Mi + requests: + cpu: 500m + memory: 16Mi {{- end }} \ No newline at end of file diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 7da6d513..7f7002db 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -1,5 +1,7 @@ clusterDomain: nebula.mci.net.id +diagnosticMode: + enabled: true etcd: clusterDomain: nebula.mci.net.id From 83566c8cfb815030bf50e2e126580b039b48fdbf Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 18:09:04 +0700 Subject: [PATCH 041/188] Add command and args diagnostic bypass --- charts/mayastor/templates/csi/DaemonSet.yaml | 32 +++++++++++++------ .../templates/mcp-core/Deployment.yaml | 6 ++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index a50f2acc..f0086ba5 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -43,6 +43,28 @@ spec: # the same. containers: - name: mayastor-csi + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} + {{- else if .Values.command }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} + {{- else }} + command: + - mayastor-csi + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if .Values.args }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} + {{- else }} + args: + - "--csi-socket=/csi/csi.sock" + - "--node-name=$(MY_NODE_NAME)" + - "--grpc-endpoint=$(MY_POD_IP):10199" + {{- end }} + {{- if .Values.csi.nvme.io_timeout_enabled }} + - "--nvme-core-io-timeout={{ .Values.csi.nvme.io_timeout }}" + {{- end }} + - "-v" image: {{ include "mayastor.csi.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} securityContext: @@ -62,16 +84,6 @@ spec: - name: MOAC value: "true" {{- end }} - args: - - "--csi-socket=/csi/csi.sock" - - "--node-name=$(MY_NODE_NAME)" - - "--grpc-endpoint=$(MY_POD_IP):10199" - {{- if .Values.csi.nvme.io_timeout_enabled }} - - "--nvme-core-io-timeout={{ .Values.csi.nvme.io_timeout }}" - {{- end }} - - "-v" - command: - - mayastor-csi volumeMounts: - name: device mountPath: /dev diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index 229276d5..d79ee280 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -65,11 +65,17 @@ spec: {{- else if .Values.command }} command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} {{- end }} + {{- if .Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if .Values.args }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} + {{- else }} args: - "-smayastor-etcd" - "-nmayastor-nats" - "--request-timeout=5s" - "--cache-period=30s" + {{- end }} env: - name: MY_POD_NAME valueFrom: From d3248b711390a5b809240d0f927dd4ed30b4e773 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 18:11:47 +0700 Subject: [PATCH 042/188] Commit --- charts/mayastor/templates/csi/DaemonSet.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index f0086ba5..ff3cc9fb 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -47,7 +47,7 @@ spec: command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} {{- else if .Values.command }} command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} - {{- else }} + {{ else }} command: - mayastor-csi {{- end }} @@ -55,7 +55,7 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} {{- else if .Values.args }} args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} - {{- else }} + {{ else }} args: - "--csi-socket=/csi/csi.sock" - "--node-name=$(MY_NODE_NAME)" From 3ed63ad225e1efc94539e97017706cf48747fb4e Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 18:12:59 +0700 Subject: [PATCH 043/188] Fix command and args --- charts/mayastor/templates/csi/DaemonSet.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index ff3cc9fb..dc75b0ef 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -47,7 +47,7 @@ spec: command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} {{- else if .Values.command }} command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} - {{ else }} + {{- else }} command: - mayastor-csi {{- end }} @@ -55,16 +55,16 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} {{- else if .Values.args }} args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} - {{ else }} + {{- else }} args: - "--csi-socket=/csi/csi.sock" - "--node-name=$(MY_NODE_NAME)" - "--grpc-endpoint=$(MY_POD_IP):10199" + {{- if .Values.csi.nvme.io_timeout_enabled }} + - "--nvme-core-io-timeout={{ .Values.csi.nvme.io_timeout }}" + {{- end }} + - "-v" {{- end }} - {{- if .Values.csi.nvme.io_timeout_enabled }} - - "--nvme-core-io-timeout={{ .Values.csi.nvme.io_timeout }}" - {{- end }} - - "-v" image: {{ include "mayastor.csi.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} securityContext: From 92cb736901f62286d3c0457118c5a5016935548a Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 18:13:24 +0700 Subject: [PATCH 044/188] Fix typo --- charts/mayastor/templates/csi/DaemonSet.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index dc75b0ef..d582fa93 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -80,10 +80,10 @@ spec: fieldPath: status.podIP - name: RUST_BACKTRACE value: full - {{- if .Values.moac }} - - name: MOAC - value: "true" - {{- end }} + {{- if .Values.moac }} + - name: MOAC + value: "true" + {{- end }} volumeMounts: - name: device mountPath: /dev From 05d768d77008b332ed04b7e9f17b5b266a819869 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 18:37:02 +0700 Subject: [PATCH 045/188] test: Add tolerations --- .../templates/mcp-core/Deployment.yaml | 6 ++- charts/mayastor/values-test.yaml | 10 +++- charts/mayastor/values.yaml | 52 +++++++++++++++---- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index d79ee280..cd800f82 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -71,8 +71,10 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} {{- else }} args: - - "-smayastor-etcd" - - "-nmayastor-nats" + # - "-smayastor-etcd" + # - "-nmayastor-nats" + - "--nats nats://mayastor-nats:4222" + - "--store http://mayastor-etcd:2379" - "--request-timeout=5s" - "--cache-period=30s" {{- end }} diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 7f7002db..58cf4f9c 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -1,7 +1,15 @@ clusterDomain: nebula.mci.net.id diagnosticMode: - enabled: true + enabled: false + +tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists etcd: clusterDomain: nebula.mci.net.id diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 89baa0c6..3d8ceb51 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -399,22 +399,52 @@ csiController: etcd: enabled: true - ## Number of replicas + ## @param replicaCount Number of etcd replicas to deploy ## replicaCount: 1 ## Kubernetes Cluster Domain ## clusterDomain: cluster.local - ## TLS authentication for client-to-server communications - ## ref: https://etcd.io/docs/current/op-guide/security/ - ## - client: - secureTransport: false - ## TLS authentication for server-to-server communications - ## ref: https://etcd.io/docs/current/op-guide/security/ - ## - peer: - secureTransport: false + ## Authentication parameters + ## + auth: + ## Role-based access control parameters + ## ref: https://etcd.io/docs/current/op-guide/authentication/ + ## + rbac: + ## @param auth.rbac.create Switch to enable RBAC authentication + ## + create: true + ## @param auth.rbac.allowNoneAuthentication Allow to use etcd without configuring RBAC authentication + ## + allowNoneAuthentication: true + ## @param auth.rbac.rootPassword Root user password. The root user is always `root` + ## + rootPassword: "" + ## @param auth.rbac.existingSecret Name of the existing secret containing credentials for the root user + ## + existingSecret: "" + ## @param auth.rbac.existingSecretPasswordKey Name of key containing password to be retrieved from the existing secret + ## + existingSecretPasswordKey: "" + ## Authentication token + ## ref: https://etcd.io/docs/latest/learning/design-auth-v3/#two-types-of-tokens-simple-and-jwt + ## + token: + ## @param auth.token.type Authentication token type. Allowed values: 'simple' or 'jwt' + ## ref: https://etcd.io/docs/latest/op-guide/configuration/#--auth-token + ## + type: simple + ## TLS authentication for client-to-server communications + ## ref: https://etcd.io/docs/current/op-guide/security/ + ## + client: + secureTransport: false + ## TLS authentication for server-to-server communications + ## ref: https://etcd.io/docs/current/op-guide/security/ + ## + peer: + secureTransport: false ## Enable persistence using Persistent Volume Claims ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ ## From 5612199bc7641a8c32616912fa38f167e94fd786 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 18:40:00 +0700 Subject: [PATCH 046/188] Add tolerations values --- charts/mayastor/templates/DaemonSet.yaml | 6 ++++++ .../mayastor/templates/csi-controller/Deployment.yaml | 6 ++++++ charts/mayastor/templates/csi/DaemonSet.yaml | 10 ++++++++-- charts/mayastor/templates/mcp-core/Deployment.yaml | 6 ++++++ charts/mayastor/templates/mcp-rest/Deployment.yaml | 6 ++++++ charts/mayastor/templates/msp-operator/Deployment.yaml | 6 ++++++ 6 files changed, 38 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index 2e0ec773..4f8d8771 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -35,6 +35,12 @@ spec: hostNetwork: true {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} + {{- end }} # To resolve services from mayastor's namespace dnsPolicy: ClusterFirstWithHostNet nodeSelector: diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 7c65e7fa..bbd00447 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -35,6 +35,12 @@ spec: hostNetwork: true {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} + {{- end }} initContainers: - command: - sh diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index d582fa93..a3d35c68 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -32,11 +32,17 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: - {{- include "mayastor.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "mayastor.serviceAccountName" . }} hostNetwork: true nodeSelector: kubernetes.io/arch: amd64 + {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "mayastor.serviceAccountName" . }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} + {{- end }} # NOTE: Each container must have mem/cpu limits defined in order to # belong to Guaranteed QoS class, hence can never get evicted in case of # pressure unless they exceed those limits. limits and requests must be diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index cd800f82..f79b575b 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -33,6 +33,12 @@ spec: spec: {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} + {{- end }} initContainers: - command: - sh diff --git a/charts/mayastor/templates/mcp-rest/Deployment.yaml b/charts/mayastor/templates/mcp-rest/Deployment.yaml index 9d40319c..a5b186b6 100644 --- a/charts/mayastor/templates/mcp-rest/Deployment.yaml +++ b/charts/mayastor/templates/mcp-rest/Deployment.yaml @@ -33,6 +33,12 @@ spec: spec: {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} + {{- end }} initContainers: - command: - sh diff --git a/charts/mayastor/templates/msp-operator/Deployment.yaml b/charts/mayastor/templates/msp-operator/Deployment.yaml index cb35cd33..60a5e736 100644 --- a/charts/mayastor/templates/msp-operator/Deployment.yaml +++ b/charts/mayastor/templates/msp-operator/Deployment.yaml @@ -33,6 +33,12 @@ spec: spec: {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} + {{- end }} initContainers: - command: - sh From 4f2157fa525203febd91455ad70c66a5483d9c09 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 19:20:44 +0700 Subject: [PATCH 047/188] Change csi driver --- charts/mayastor/templates/csi/DaemonSet.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index a3d35c68..c9ff2da0 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -112,10 +112,12 @@ spec: cpu: "100m" memory: "50Mi" - name: csi-driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:v2.1.0 + # image: quay.io/k8scsi/csi-node-driver-registrar:v2.1.0 + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0 args: - "--csi-address=/csi/csi.sock" - "--kubelet-registration-path=/var/lib/kubelet/plugins/mayastor.openebs.io/csi.sock" + - "--v=5" volumeMounts: - name: plugin-dir mountPath: /csi From 51482b2fc702752f1204293b0552103418b5ecfb Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 19:26:44 +0700 Subject: [PATCH 048/188] Add revisionHistoryLimit --- charts/mayastor/templates/DaemonSet.yaml | 1 + charts/mayastor/templates/csi-controller/Deployment.yaml | 1 + charts/mayastor/templates/mcp-core/Deployment.yaml | 1 + charts/mayastor/templates/mcp-rest/Deployment.yaml | 1 + charts/mayastor/templates/msp-operator/Deployment.yaml | 1 + 5 files changed, 5 insertions(+) diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index 4f8d8771..0bb4d7d4 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -12,6 +12,7 @@ metadata: annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} spec: + revisionHistoryLimit: 2 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: mayastor diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index bbd00447..b32f3f64 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -16,6 +16,7 @@ metadata: {{- end }} spec: replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: 2 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: csi-controller diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index f79b575b..a7308d4d 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -16,6 +16,7 @@ metadata: {{- end }} spec: replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: 2 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: core-agents diff --git a/charts/mayastor/templates/mcp-rest/Deployment.yaml b/charts/mayastor/templates/mcp-rest/Deployment.yaml index a5b186b6..c7b1937b 100644 --- a/charts/mayastor/templates/mcp-rest/Deployment.yaml +++ b/charts/mayastor/templates/mcp-rest/Deployment.yaml @@ -16,6 +16,7 @@ metadata: {{- end }} spec: replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: 2 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: mcp-rest diff --git a/charts/mayastor/templates/msp-operator/Deployment.yaml b/charts/mayastor/templates/msp-operator/Deployment.yaml index 60a5e736..67c93d67 100644 --- a/charts/mayastor/templates/msp-operator/Deployment.yaml +++ b/charts/mayastor/templates/msp-operator/Deployment.yaml @@ -16,6 +16,7 @@ metadata: {{- end }} spec: replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: 2 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: msp-operator From 44898364c4116fbb842467d00e2f1bc7564e614a Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 19:35:54 +0700 Subject: [PATCH 049/188] Fix core-agents --- charts/mayastor/templates/csi/DaemonSet.yaml | 1 + charts/mayastor/templates/mcp-core/Deployment.yaml | 4 ++-- charts/mayastor/values-test.yaml | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index c9ff2da0..3c9ec88c 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -12,6 +12,7 @@ metadata: annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} spec: + revisionHistoryLimit: 2 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: mayastor-csi diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/mcp-core/Deployment.yaml index a7308d4d..42b1715b 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/mcp-core/Deployment.yaml @@ -80,8 +80,8 @@ spec: args: # - "-smayastor-etcd" # - "-nmayastor-nats" - - "--nats nats://mayastor-nats:4222" - - "--store http://mayastor-etcd:2379" + - "--nats=mayastor-nats" + - "--store=mayastor-etcd" - "--request-timeout=5s" - "--cache-period=30s" {{- end }} diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 58cf4f9c..c2753136 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -35,6 +35,8 @@ nats: mayastorPools: - node: alberta-lin-k8m.jkt2.mci.net.id device: /dev/sdb + - node: einbech-lin-k8m.jkt2.mci.net.id + device: /dev/sdb - node: rachel-lin-k8w.jkt2.mci.net.id device: /dev/sdb - node: verus-lin-k8w.jkt2.mci.net.id From e432a72856620bd161f21dfc5a2d475c26932bb6 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 20:13:19 +0700 Subject: [PATCH 050/188] Add csi-socket to values --- charts/mayastor/templates/csi/DaemonSet.yaml | 43 +++++++++++--------- charts/mayastor/values.yaml | 4 +- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index 3c9ec88c..8ed7d88d 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -64,7 +64,7 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} {{- else }} args: - - "--csi-socket=/csi/csi.sock" + - "--csi-socket=$(CSI_SOCKET_ADDRESS)" - "--node-name=$(MY_NODE_NAME)" - "--grpc-endpoint=$(MY_POD_IP):10199" {{- if .Values.csi.nvme.io_timeout_enabled }} @@ -77,6 +77,8 @@ spec: securityContext: privileged: true env: + - name: CSI_SOCKET_ADDRESS + value: {{ .Values.csi.socketAddress | quote }} - name: MY_NODE_NAME valueFrom: fieldRef: @@ -91,6 +93,13 @@ spec: - name: MOAC value: "true" {{- end }} + resources: + limits: + cpu: "100m" + memory: "50Mi" + requests: + cpu: "100m" + memory: "50Mi" volumeMounts: - name: device mountPath: /dev @@ -105,25 +114,21 @@ spec: - name: kubelet-dir mountPath: /var/lib/kubelet mountPropagation: "Bidirectional" - resources: - limits: - cpu: "100m" - memory: "50Mi" - requests: - cpu: "100m" - memory: "50Mi" - name: csi-driver-registrar # image: quay.io/k8scsi/csi-node-driver-registrar:v2.1.0 image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0 args: - - "--csi-address=/csi/csi.sock" + - "--csi-address=$(CSI_SOCKET_ADDRESS)" - "--kubelet-registration-path=/var/lib/kubelet/plugins/mayastor.openebs.io/csi.sock" - "--v=5" - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration + env: + - name: CSI_SOCKET_ADDRESS + value: {{ .Values.csi.socketAddress | quote }} + # Mayastor node plugin gRPC server + ports: + - containerPort: 10199 + name: mayastor-node + protocol: TCP resources: limits: cpu: "100m" @@ -131,11 +136,11 @@ spec: requests: cpu: "100m" memory: "50Mi" - # Mayastor node plugin gRPC server - ports: - - containerPort: 10199 - name: mayastor-node - protocol: TCP + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration volumes: - name: device hostPath: diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 3d8ceb51..3e51f16f 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -349,7 +349,7 @@ csi: ## image: registry: docker.io - repository: mayadata/mayastor + repository: mayadata/mayastor-csi tag: release-1.0.2 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -369,6 +369,8 @@ csi: io_timeout: "30" io_timeout_enabled: true + socketAddress: /csi/csi.sock + csiController: enabled: true ## Mayastor Control Plane Core Agent image From a8edc02537e76514033681a7fb5c5f4348122f33 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 21:10:06 +0700 Subject: [PATCH 051/188] Fix label and selector --- charts/mayastor/templates/DaemonSet.yaml | 9 +++++---- charts/mayastor/templates/csi/DaemonSet.yaml | 1 + .../templates/msp-operator/Deployment.yaml | 14 +++++++------- charts/mayastor/values-test.yaml | 4 +++- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index 0bb4d7d4..5a58a844 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -1,10 +1,11 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ include "common.names.fullname" . }} + name: {{ printf "%s-node" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} openebs/engine: mayastor + app.kubernetes.io/component: mayastor-node {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} @@ -15,7 +16,7 @@ spec: revisionHistoryLimit: 2 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} - app.kubernetes.io/component: mayastor + app.kubernetes.io/component: mayastor-node updateStrategy: type: RollingUpdate rollingUpdate: @@ -28,7 +29,7 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} {{- end }} labels: {{- include "common.labels.standard" . | nindent 8 }} - app.kubernetes.io/component: mayastor + app.kubernetes.io/component: mayastor-node {{- if .Values.podLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} @@ -52,7 +53,7 @@ spec: image: busybox:latest command: ['sh', '-c', 'until nc -vz mayastor-nats 4222; do echo "Waiting for message bus..."; sleep 1; done;'] containers: - - name: mayastor + - name: mayastor-node image: {{ include "mayastor.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} env: diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index 8ed7d88d..f248cb4c 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -35,6 +35,7 @@ spec: spec: hostNetwork: true nodeSelector: + openebs.io/engine: mayastor kubernetes.io/arch: amd64 {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} diff --git a/charts/mayastor/templates/msp-operator/Deployment.yaml b/charts/mayastor/templates/msp-operator/Deployment.yaml index 67c93d67..0cdeac74 100644 --- a/charts/mayastor/templates/msp-operator/Deployment.yaml +++ b/charts/mayastor/templates/msp-operator/Deployment.yaml @@ -55,13 +55,6 @@ spec: name: etcd-probe containers: - name: msp-operator - resources: - limits: - cpu: 100m - memory: 32Mi - requests: - cpu: 50m - memory: 16Mi image: {{ include "mayastor.mspOperator.image" . }} imagePullPolicy: {{ .Values.mspOperator.image.pullPolicy | quote }} args: @@ -76,4 +69,11 @@ spec: value: info,msp_operator=info - name: RUST_BACKTRACE value: full + resources: + limits: + cpu: 100m + memory: 32Mi + requests: + cpu: 50m + memory: 16Mi {{- end }} \ No newline at end of file diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index c2753136..4dbb08a0 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -14,6 +14,7 @@ tolerations: etcd: clusterDomain: nebula.mci.net.id replicaCount: 1 + revisionHistoryLimit: 2 auth: rbac: create: true @@ -25,9 +26,10 @@ etcd: nats: clusterDomain: nebula.mci.net.id + replicaCount: 1 + revisionHistoryLimit: 2 auth: enabled: false - replicaCount: 1 persistence: enabled: false storageClass: openebs-hostpath From ee7eb1c2bd8249d7e02821d6404032615bb0cffc Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 21:37:49 +0700 Subject: [PATCH 052/188] Add image helper --- charts/mayastor/templates/_helper.tpl | 24 +++++++++++++++++-- charts/mayastor/templates/csi/DaemonSet.yaml | 5 ++-- charts/mayastor/values.yaml | 25 ++++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index 0c62c57a..616b4c6a 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -12,16 +12,36 @@ {{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }} {{- end -}} -{{/* Return the proper Mayastor CSI image name */}} +{{/* Return the proper Mayastor CSI Node image name */}} {{- define "mayastor.csi.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.csi.image "global" .Values.global) }} {{- end -}} -{{/* Return the proper Mayastor CSI image name */}} +{{/* Return the proper Mayastor CSI Node image name */}} +{{- define "mayastor.csiNode.driverRegistrar.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.csi.driverRegistrar.image "global" .Values.global) }} +{{- end -}} + +{{/* Return the proper Mayastor CSI Controller image name */}} {{- define "mayastor.csiController.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} {{- end -}} +{{/* Return the proper Mayastor CSI Controller Attacher image name */}} +{{- define "mayastor.csiController.attacher.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} +{{- end -}} + +{{/* Return the proper Mayastor CSI Controller Livenessprobe image name */}} +{{- define "mayastor.csiController.livenessprobe.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} +{{- end -}} + +{{/* Return the proper Mayastor CSI Controller Provisioner image name */}} +{{- define "mayastor.csiController.provisioner.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} +{{- end -}} + {{/* Return the proper Mayastor Rest image name */}} {{- define "mayastor.mcpCore.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.mcpCore.image "global" .Values.global) }} diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index f248cb4c..f4bb9e1e 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -116,8 +116,9 @@ spec: mountPath: /var/lib/kubelet mountPropagation: "Bidirectional" - name: csi-driver-registrar - # image: quay.io/k8scsi/csi-node-driver-registrar:v2.1.0 - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0 + # image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0 + image: {{ include "mayastor.csiNode.driverRegistrar.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" - "--kubelet-registration-path=/var/lib/kubelet/plugins/mayastor.openebs.io/csi.sock" diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 3e51f16f..38454fee 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -364,6 +364,12 @@ csi: ## pullSecrets: [] + driverRegistrar: + image: + registry: quay.io + repository: k8scsi/csi-node-driver-registrar + tag: v2.1.0 + nvme: # nvme_core module io timeout in seconds io_timeout: "30" @@ -399,6 +405,25 @@ csiController: ## pullSecrets: [] + attacher: + image: + registry: k8s.gcr.io + repository: sig-storage/csi-attacher + tag: v3.4.0 + + livenessprobe: + image: + registry: k8s.gcr.io + repository: sig-storage/livenessprobe + tag: v2.6.0 + + provisioner: + image: + registry: k8s.gcr.io + repository: sig-storage/csi-provisioner + tag: v3.1.0 + + etcd: enabled: true ## @param replicaCount Number of etcd replicas to deploy From 1d87e02aece09edc327a7dc51942aa142eb72613 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 21:38:30 +0700 Subject: [PATCH 053/188] Fix image helper --- charts/mayastor/templates/_helper.tpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index 616b4c6a..d0ba9726 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -29,17 +29,17 @@ {{/* Return the proper Mayastor CSI Controller Attacher image name */}} {{- define "mayastor.csiController.attacher.image" -}} - {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.csiController.attacher.image "global" .Values.global) }} {{- end -}} {{/* Return the proper Mayastor CSI Controller Livenessprobe image name */}} {{- define "mayastor.csiController.livenessprobe.image" -}} - {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.csiController.livenessprobe.image "global" .Values.global) }} {{- end -}} {{/* Return the proper Mayastor CSI Controller Provisioner image name */}} {{- define "mayastor.csiController.provisioner.image" -}} - {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.csiController.provisioner.image "global" .Values.global) }} {{- end -}} {{/* Return the proper Mayastor Rest image name */}} From ea9af2cbdba563afbd25f2ce80cd2f3b4d7665a5 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 23:16:13 +0700 Subject: [PATCH 054/188] Add CSIDriver --- charts/mayastor/templates/CSIDriver.yaml | 9 +++++++++ charts/mayastor/templates/_helper.tpl | 9 --------- charts/mayastor/templates/csi/DaemonSet.yaml | 14 +++++++++++--- charts/mayastor/values.yaml | 4 ++++ 4 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 charts/mayastor/templates/CSIDriver.yaml diff --git a/charts/mayastor/templates/CSIDriver.yaml b/charts/mayastor/templates/CSIDriver.yaml new file mode 100644 index 00000000..e6884ce3 --- /dev/null +++ b/charts/mayastor/templates/CSIDriver.yaml @@ -0,0 +1,9 @@ +{{- if .Values.csiDriver.create -}} +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: mayastor.openebs.io +spec: + podInfoOnMount: {{ .Values.csiDriver.podInfoOnMount }} + attachRequired: {{ .Values.csiDriver.attachRequired }} +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index d0ba9726..b4528a54 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -67,15 +67,6 @@ {{- include "common.images.pullSecrets" (dict "images" (list .Values.image .Values.volumePermissions.image) "global" .Values.global) -}} {{- end -}} -{{/* Enforce trailing slash to mayastorImagesPrefix or leave empty */}} -{{- define "mayastorImagesPrefix" -}} -{{- if .Values.mayastorImagesRegistry }} -{{- printf "%s/" (.Values.mayastorImagesRegistry | trimSuffix "/") }} -{{- else }} -{{- "" }} -{{- end }} -{{- end }} - {{/* Generate CPU list specification based on CPU count (-l param of mayastor) */}} {{- define "mayastorCpuSpec" -}} {{- range $i, $e := until (int .Values.mayastorCpuCount) }} diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index f4bb9e1e..3cba4d12 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -4,7 +4,7 @@ metadata: name: {{ printf "%s-csi" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - openebs/engine: mayastor + app.kubernetes.io/component: mayastor-csi {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} @@ -50,7 +50,7 @@ spec: # pressure unless they exceed those limits. limits and requests must be # the same. containers: - - name: mayastor-csi + - name: mayastor-csi-plugin {{- if .Values.diagnosticMode.enabled }} command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} {{- else if .Values.command }} @@ -71,7 +71,7 @@ spec: {{- if .Values.csi.nvme.io_timeout_enabled }} - "--nvme-core-io-timeout={{ .Values.csi.nvme.io_timeout }}" {{- end }} - - "-v" + - "--v=5" {{- end }} image: {{ include "mayastor.csi.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} @@ -83,6 +83,7 @@ spec: - name: MY_NODE_NAME valueFrom: fieldRef: + apiVersion: v1 fieldPath: spec.nodeName - name: MY_POD_IP valueFrom: @@ -126,6 +127,13 @@ spec: env: - name: CSI_SOCKET_ADDRESS value: {{ .Values.csi.socketAddress | quote }} + lifecycle: + preStop: + exec: + command: + - /bin/sh" + - -c + - rm -rf /registration/mayastor.openebs.io /registration/mayastor.openebs.io-reg.sock # Mayastor node plugin gRPC server ports: - containerPort: 10199 diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 38454fee..0597abaa 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -423,6 +423,10 @@ csiController: repository: sig-storage/csi-provisioner tag: v3.1.0 +csiDriver: + create: true + podInfoOnMount: true + attachRequired: false etcd: enabled: true From 33140c9dd5e590560abf2cd3991fa94efa9a2088 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 23:26:24 +0700 Subject: [PATCH 055/188] Change driverRegistrar image --- charts/mayastor/templates/csi/DaemonSet.yaml | 9 +++++---- charts/mayastor/values.yaml | 9 ++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index 3cba4d12..a4c4847c 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -71,7 +71,7 @@ spec: {{- if .Values.csi.nvme.io_timeout_enabled }} - "--nvme-core-io-timeout={{ .Values.csi.nvme.io_timeout }}" {{- end }} - - "--v=5" + - "-v" {{- end }} image: {{ include "mayastor.csi.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} @@ -117,7 +117,6 @@ spec: mountPath: /var/lib/kubelet mountPropagation: "Bidirectional" - name: csi-driver-registrar - # image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0 image: {{ include "mayastor.csiNode.driverRegistrar.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} args: @@ -131,9 +130,11 @@ spec: preStop: exec: command: - - /bin/sh" + - /bin/sh - -c - - rm -rf /registration/mayastor.openebs.io /registration/mayastor.openebs.io-reg.sock + - >- + echo "Deleting stale file"; + rm -rf /registration/mayastor.openebs.io /registration/mayastor.openebs.io-reg.sock; # Mayastor node plugin gRPC server ports: - containerPort: 10199 diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 0597abaa..975a7006 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -366,9 +366,12 @@ csi: driverRegistrar: image: - registry: quay.io - repository: k8scsi/csi-node-driver-registrar - tag: v2.1.0 + # registry: quay.io + # repository: k8scsi/csi-node-driver-registrar + # tag: v2.1.0 + registry: k8s.gcr.io + repository: sig-storage/csi-node-driver-registrar + tag: v2.5.0 nvme: # nvme_core module io timeout in seconds From a0e944286e9b2cee349acd19222f35c9ca957b83 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 23:42:31 +0700 Subject: [PATCH 056/188] Change CSIDriver --- charts/mayastor/templates/CSIDriver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/CSIDriver.yaml b/charts/mayastor/templates/CSIDriver.yaml index e6884ce3..508cb26f 100644 --- a/charts/mayastor/templates/CSIDriver.yaml +++ b/charts/mayastor/templates/CSIDriver.yaml @@ -2,7 +2,7 @@ apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: - name: mayastor.openebs.io + name: io.openebs.csi-mayastor spec: podInfoOnMount: {{ .Values.csiDriver.podInfoOnMount }} attachRequired: {{ .Values.csiDriver.attachRequired }} From b0a9b187b985c27ab0d477f92dcf53e5ab9850d5 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Wed, 15 Jun 2022 23:47:11 +0700 Subject: [PATCH 057/188] Add alternative image --- charts/mayastor/values.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 975a7006..a633e2ba 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -393,8 +393,10 @@ csiController: ## image: registry: docker.io - repository: mayadata/mcp-csi-controller - tag: release-1.0.2 + # repository: mayadata/mcp-csi-controller + # tag: release-1.0.2 + repository: mayadata/mayastor-csi-controller + tag: develop ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -581,8 +583,10 @@ mcpCore: ## image: registry: docker.io - repository: mayadata/mcp-core - tag: release-1.0.2 + # repository: mayadata/mcp-core + # tag: release-1.0.2 + repository: mayadata/mayastor-agent-core + tag: develop ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images From 91de6ae54f540a4d8d6e6c622e5d7e5aa102577b Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 00:00:33 +0700 Subject: [PATCH 058/188] Add switch for imageDev --- charts/mayastor/templates/_helper.tpl | 4 ++++ charts/mayastor/values.yaml | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index b4528a54..e21830e3 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -49,8 +49,12 @@ {{/* Return the proper Mayastor Rest image name */}} {{- define "mayastor.mcpRest.image" -}} +{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} + {{ include "common.images.image" (dict "imageRoot" .Values.mcpRest.imagedev "global" .Values.global) }} +{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.mcpRest.image "global" .Values.global) }} {{- end -}} +{{- end -}} {{/* Return the proper Mayastor Operator image name */}} {{- define "mayastor.mspOperator.image" -}} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index a633e2ba..b8dc29ad 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -59,6 +59,9 @@ diagnosticMode: ## args: - infinity + ## @param diagnosticMode.environment + ## + environment: development ## @section Mayastor common parameters @@ -602,6 +605,10 @@ mcpCore: mcpRest: enabled: true + imageDev: + registry: docker.io + repository: mayadata/mayastor-api-rest + tag: develop ## Mayastor Control Plane Rest image ## ref: https://hub.docker.com/r/mayadata/mcp-rest/tags ## @param image.registry Mayastor Control Plane Rest image registry From d19393877839a783b5c8cb68ee869599e6b045e9 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 00:03:38 +0700 Subject: [PATCH 059/188] Add imageDev --- charts/mayastor/values.yaml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index b8dc29ad..416e6aa0 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -341,6 +341,10 @@ service: csi: enabled: true + imageDev: + registry: docker.io + repository: mayadata/mayastor-csi-node + tag: develop ## Mayastor CSI image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry @@ -385,6 +389,10 @@ csi: csiController: enabled: true + imageDev: + registry: docker.io + repository: mayadata/mayastor-csi-controller + tag: develop ## Mayastor Control Plane Core Agent image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry @@ -396,10 +404,8 @@ csiController: ## image: registry: docker.io - # repository: mayadata/mcp-csi-controller - # tag: release-1.0.2 - repository: mayadata/mayastor-csi-controller - tag: develop + repository: mayadata/mcp-csi-controller + tag: release-1.0.2 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -575,6 +581,10 @@ nats: mcpCore: enabled: true + imageDev: + registry: docker.io + repository: mayadata/mayastor-agent-core + tag: develop ## Mayastor Control Plane Core Agent image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry @@ -586,10 +596,8 @@ mcpCore: ## image: registry: docker.io - # repository: mayadata/mcp-core - # tag: release-1.0.2 - repository: mayadata/mayastor-agent-core - tag: develop + repository: mayadata/mcp-core + tag: release-1.0.2 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images From 856309112d3cff2b9674dfb93da842d53b9634b4 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 00:08:01 +0700 Subject: [PATCH 060/188] Fix image helper --- charts/mayastor/templates/_helper.tpl | 18 +++++++++++++++++- charts/mayastor/values.yaml | 4 ++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index e21830e3..aafce0a6 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -9,13 +9,21 @@ {{/* Return the proper Mayastor image name */}} {{- define "mayastor.image" -}} +{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} + {{ include "common.images.image" (dict "imageRoot" .Values.imageDev "global" .Values.global) }} +{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }} {{- end -}} +{{- end -}} {{/* Return the proper Mayastor CSI Node image name */}} {{- define "mayastor.csi.image" -}} +{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} + {{ include "common.images.image" (dict "imageRoot" .Values.csi.imageDev "global" .Values.global) }} +{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.csi.image "global" .Values.global) }} {{- end -}} +{{- end -}} {{/* Return the proper Mayastor CSI Node image name */}} {{- define "mayastor.csiNode.driverRegistrar.image" -}} @@ -24,8 +32,12 @@ {{/* Return the proper Mayastor CSI Controller image name */}} {{- define "mayastor.csiController.image" -}} +{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} + {{ include "common.images.image" (dict "imageRoot" .Values.csiController.imageDev "global" .Values.global) }} +{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} {{- end -}} +{{- end -}} {{/* Return the proper Mayastor CSI Controller Attacher image name */}} {{- define "mayastor.csiController.attacher.image" -}} @@ -44,13 +56,17 @@ {{/* Return the proper Mayastor Rest image name */}} {{- define "mayastor.mcpCore.image" -}} +{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} + {{ include "common.images.image" (dict "imageRoot" .Values.mcpCore.imageDev "global" .Values.global) }} +{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.mcpCore.image "global" .Values.global) }} {{- end -}} +{{- end -}} {{/* Return the proper Mayastor Rest image name */}} {{- define "mayastor.mcpRest.image" -}} {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} - {{ include "common.images.image" (dict "imageRoot" .Values.mcpRest.imagedev "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.mcpRest.imageDev "global" .Values.global) }} {{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.mcpRest.image "global" .Values.global) }} {{- end -}} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 416e6aa0..930638c1 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -66,6 +66,10 @@ diagnosticMode: ## @section Mayastor common parameters ## Mayastor image +imageDev: + registry: docker.io + repository: mayadata/mayastor-csi-node + tag: develop ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry ## @param image.repository Mayastor Control Plane Rest image repository From be7fe4f14b9dd6932064a88a8d923c8b24493b55 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 00:30:07 +0700 Subject: [PATCH 061/188] test: diagnosticMode --- charts/mayastor/templates/DaemonSet.yaml | 46 +++++++++++++++--------- charts/mayastor/templates/_helper.tpl | 6 +++- charts/mayastor/values-test.yaml | 3 +- charts/mayastor/values.yaml | 8 +++-- 4 files changed, 42 insertions(+), 21 deletions(-) diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index 5a58a844..e8cf72c2 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -34,7 +34,12 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: + # To resolve services from mayastor's namespace + dnsPolicy: ClusterFirstWithHostNet hostNetwork: true + nodeSelector: + openebs.io/engine: mayastor + kubernetes.io/arch: amd64 {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- if .Values.tolerations }} @@ -43,11 +48,6 @@ spec: {{- if .Values.topologySpreadConstraints }} topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} {{- end }} - # To resolve services from mayastor's namespace - dnsPolicy: ClusterFirstWithHostNet - nodeSelector: - openebs.io/engine: mayastor - kubernetes.io/arch: amd64 initContainers: - name: message-bus-probe image: busybox:latest @@ -71,6 +71,19 @@ spec: value: info,mayastor={{ .Values.mayastorLogLevel }} - name: RUST_BACKTRACE value: full + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} + {{- else if .Values.command }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} + {{- else }} + command: + - mayastor + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if .Values.args }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} + {{- else }} args: # The -l argument accepts cpu-list. Indexing starts at zero. # For example -l 1,2,10-20 means use core 1, 2, 10 to 20. @@ -82,19 +95,9 @@ spec: - "-y/var/local/mayastor/config.yaml" - "-l{{ include "mayastorCpuSpec" . }}" - "-pmayastor-etcd" - command: - - mayastor + {{- end }} securityContext: privileged: true - volumeMounts: - - name: device - mountPath: /dev - - name: udev - mountPath: /run/udev - - name: dshm - mountPath: /dev/shm - - name: configlocation - mountPath: /var/local/mayastor/ resources: # NOTE: Each container must have mem/cpu limits defined in order to # belong to Guaranteed QoS class, hence can never get evicted in case of @@ -111,12 +114,21 @@ spec: - containerPort: 10124 name: mayastor protocol: TCP + volumeMounts: + - name: device + mountPath: /dev + - name: run-udev + mountPath: /run/udev + - name: dshm + mountPath: /dev/shm + - name: configlocation + mountPath: /var/local/mayastor/ volumes: - name: device hostPath: path: /dev type: Directory - - name: udev + - name: run-udev hostPath: path: /run/udev type: Directory diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index aafce0a6..c2743c87 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -54,7 +54,7 @@ {{ include "common.images.image" (dict "imageRoot" .Values.csiController.provisioner.image "global" .Values.global) }} {{- end -}} -{{/* Return the proper Mayastor Rest image name */}} +{{/* Return the proper Mayastor Core Agent image name */}} {{- define "mayastor.mcpCore.image" -}} {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} {{ include "common.images.image" (dict "imageRoot" .Values.mcpCore.imageDev "global" .Values.global) }} @@ -74,8 +74,12 @@ {{/* Return the proper Mayastor Operator image name */}} {{- define "mayastor.mspOperator.image" -}} +{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} + {{ include "common.images.image" (dict "imageRoot" .Values.mspOperator.imageDev "global" .Values.global) }} +{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.mspOperator.image "global" .Values.global) }} {{- end -}} +{{- end -}} {{/* Return the proper image name (for the init container volume-permissions image) */}} {{- define "mayastor.volumePermissions.image" -}} diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 4dbb08a0..f64c36b4 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -1,7 +1,8 @@ clusterDomain: nebula.mci.net.id diagnosticMode: - enabled: false + enabled: true + environment: development tolerations: - effect: NoSchedule diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 930638c1..9a3dc45c 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -59,9 +59,9 @@ diagnosticMode: ## args: - infinity - ## @param diagnosticMode.environment + ## @param diagnosticMode.environment (enter: development) ## - environment: development + environment: "" ## @section Mayastor common parameters @@ -718,6 +718,10 @@ mcpRest: mspOperator: enabled: true + imageDev: + registry: docker.io + repository: mayadata/mayastor-operator-diskpool + tag: develop ## Mayastor Control Plane MSP Operator image ## ref: https://hub.docker.com/r/mayadata/mcp-msp-operator/tags ## @param image.registry Mayastor Control Plane MSP Operator image registry From ca2dec8c5cc93650742569cf3c5a8bf5ee197699 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 00:38:42 +0700 Subject: [PATCH 062/188] test: disabled diagnosticMode --- charts/mayastor/templates/csi/DaemonSet.yaml | 4 ++++ charts/mayastor/values-test.yaml | 2 +- charts/mayastor/values.yaml | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index a4c4847c..07121525 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -57,7 +57,11 @@ spec: command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} {{- else }} command: + {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} + - csi-node + {{- else -}} - mayastor-csi + {{- end -}} {{- end }} {{- if .Values.diagnosticMode.enabled }} args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index f64c36b4..683abef2 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -1,7 +1,7 @@ clusterDomain: nebula.mci.net.id diagnosticMode: - enabled: true + enabled: false environment: development tolerations: diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 9a3dc45c..9a5c32e8 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -68,8 +68,8 @@ diagnosticMode: ## Mayastor image imageDev: registry: docker.io - repository: mayadata/mayastor-csi-node - tag: develop + repository: mayadata/mayastor + tag: release-1.0.2 ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry ## @param image.repository Mayastor Control Plane Rest image repository From de7d123ab2cd4911fddbcf03c4bbeae17b269558 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 00:43:13 +0700 Subject: [PATCH 063/188] Fix bracket --- charts/mayastor/templates/csi/DaemonSet.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index 07121525..6e7fc0fe 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -57,11 +57,11 @@ spec: command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} {{- else }} command: - {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} + {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") }} - csi-node - {{- else -}} + {{- else }} - mayastor-csi - {{- end -}} + {{- end }} {{- end }} {{- if .Values.diagnosticMode.enabled }} args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} From 65cd9c506c3d1b083b260f2fb8cdf9f004ad9b96 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 00:55:41 +0700 Subject: [PATCH 064/188] Add ClusterRoe --- charts/mayastor/templates/ClusterRole.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/charts/mayastor/templates/ClusterRole.yaml b/charts/mayastor/templates/ClusterRole.yaml index 8ef0b989..98dfcdf1 100644 --- a/charts/mayastor/templates/ClusterRole.yaml +++ b/charts/mayastor/templates/ClusterRole.yaml @@ -55,4 +55,11 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] + #Custom Role +- apiGroups: [""] + resources: ["secrets","namespaces"] + verbs: ["get", "list"] +- apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "get", "list", "watch" ] {{- end }} \ No newline at end of file From e54bf1549ceb1bd8079ded8df1bd3e2978109796 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 01:51:34 +0700 Subject: [PATCH 065/188] Revert etcd version --- charts/mayastor/Chart.yaml | 2 +- charts/mayastor/templates/ClusterRole.yaml | 24 ++++++++++++++++------ charts/mayastor/values.yaml | 3 ++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/charts/mayastor/Chart.yaml b/charts/mayastor/Chart.yaml index fa43a123..473f45f4 100644 --- a/charts/mayastor/Chart.yaml +++ b/charts/mayastor/Chart.yaml @@ -13,7 +13,7 @@ dependencies: repository: https://charts.bitnami.com/bitnami tags: - bitnami-etcd - version: 8.x.x + version: 6.x.x - name: nats condition: nats.enabled repository: https://charts.bitnami.com/bitnami diff --git a/charts/mayastor/templates/ClusterRole.yaml b/charts/mayastor/templates/ClusterRole.yaml index 98dfcdf1..0d866aff 100644 --- a/charts/mayastor/templates/ClusterRole.yaml +++ b/charts/mayastor/templates/ClusterRole.yaml @@ -10,6 +10,7 @@ rules: - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["create", "list"] + # must read mayastor pools info - apiGroups: ["openebs.io"] resources: ["mayastorpools"] @@ -18,6 +19,16 @@ rules: - apiGroups: ["openebs.io"] resources: ["mayastorpools/status"] verbs: ["update", "patch"] + + # must read diskpool info +- apiGroups: ["openebs.io"] + resources: ["diskpools"] + verbs: ["get", "list", "watch", "update", "replace", "patch"] + # must update diskpool status +- apiGroups: ["openebs.io"] + resources: ["diskpools/status"] + verbs: ["update", "patch"] + # external provisioner & attacher - apiGroups: [""] resources: ["persistentvolumes"] @@ -25,6 +36,7 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] + # external provisioner - apiGroups: [""] resources: ["persistentvolumeclaims"] @@ -44,6 +56,7 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] + # external attacher - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] @@ -51,15 +64,14 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] + # CSI nodes must be listed - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - #Custom Role + + # get kube-system namespace to retrieve Uid - apiGroups: [""] - resources: ["secrets","namespaces"] - verbs: ["get", "list"] -- apiGroups: [ "" ] - resources: [ "pods" ] - verbs: [ "get", "list", "watch" ] + resources: ["namespaces"] + verbs: ["get"] {{- end }} \ No newline at end of file diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 9a5c32e8..1f3eee55 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -463,7 +463,8 @@ etcd: rbac: ## @param auth.rbac.create Switch to enable RBAC authentication ## - create: true + #create: true + enabled: false ## @param auth.rbac.allowNoneAuthentication Allow to use etcd without configuring RBAC authentication ## allowNoneAuthentication: true From 78e51ebf5e960e6beb163dd09540d5322acb44fb Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 01:54:23 +0700 Subject: [PATCH 066/188] Commit --- charts/mayastor/templates/csi/DaemonSet.yaml | 6 +++--- charts/mayastor/values.yaml | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi/DaemonSet.yaml index 6e7fc0fe..6db2bfe2 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi/DaemonSet.yaml @@ -4,7 +4,7 @@ metadata: name: {{ printf "%s-csi" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: mayastor-csi + app.kubernetes.io/component: csi-node {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} @@ -15,7 +15,7 @@ spec: revisionHistoryLimit: 2 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} - app.kubernetes.io/component: mayastor-csi + app.kubernetes.io/component: csi-node updateStrategy: type: RollingUpdate rollingUpdate: @@ -28,7 +28,7 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} {{- end }} labels: {{- include "common.labels.standard" . | nindent 8 }} - app.kubernetes.io/component: mayastor-csi + app.kubernetes.io/component: csi-node {{- if .Values.podLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 1f3eee55..a2d2864f 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -454,6 +454,8 @@ etcd: ## Kubernetes Cluster Domain ## clusterDomain: cluster.local + podLabels: + app.kubernetes.io/component: etcd ## Authentication parameters ## auth: @@ -546,6 +548,8 @@ nats: ## Kubernetes Cluster Domain ## clusterDomain: cluster.local + podLabels: + app.kubernetes.io/component: nats ## Enable persistence using Persistent Volume Claims ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ ## From 2e24c75d85c9ef11f6e586bbe66f766a9bf2e289 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 03:19:56 +0700 Subject: [PATCH 067/188] Fix broken template --- charts/mayastor/templates/DaemonSet.yaml | 18 ++--- charts/mayastor/templates/_helper.tpl | 45 +++++++---- .../{mcp-core => agent-core}/Deployment.yaml | 47 +++++------ .../templates/agent-core/Service.yaml | 62 ++++++++++++++ .../{mcp-rest => api-rest}/Deployment.yaml | 30 +++---- .../{mcp-rest => api-rest}/Service.yaml | 44 +++++----- .../templates/csi-controller/Deployment.yaml | 27 ++++--- .../{csi => csi-node}/DaemonSet.yaml | 38 ++++----- charts/mayastor/values.yaml | 81 +++++++++++++++++-- 9 files changed, 262 insertions(+), 130 deletions(-) rename charts/mayastor/templates/{mcp-core => agent-core}/Deployment.yaml (73%) create mode 100644 charts/mayastor/templates/agent-core/Service.yaml rename charts/mayastor/templates/{mcp-rest => api-rest}/Deployment.yaml (71%) rename charts/mayastor/templates/{mcp-rest => api-rest}/Service.yaml (61%) rename charts/mayastor/templates/{csi => csi-node}/DaemonSet.yaml (83%) diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index e8cf72c2..c7411b2f 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -1,11 +1,11 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ printf "%s-node" (include "common.names.fullname" .) }} + name: {{ printf "%s" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - openebs/engine: mayastor - app.kubernetes.io/component: mayastor-node + openebs.io/engine: io-engine + app.kubernetes.io/component: mayastor {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} @@ -16,7 +16,7 @@ spec: revisionHistoryLimit: 2 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} - app.kubernetes.io/component: mayastor-node + app.kubernetes.io/component: mayastor updateStrategy: type: RollingUpdate rollingUpdate: @@ -29,7 +29,7 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} {{- end }} labels: {{- include "common.labels.standard" . | nindent 8 }} - app.kubernetes.io/component: mayastor-node + app.kubernetes.io/component: mayastor {{- if .Values.podLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} @@ -37,10 +37,10 @@ spec: # To resolve services from mayastor's namespace dnsPolicy: ClusterFirstWithHostNet hostNetwork: true - nodeSelector: - openebs.io/engine: mayastor - kubernetes.io/arch: amd64 {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- if .Values.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} @@ -53,7 +53,7 @@ spec: image: busybox:latest command: ['sh', '-c', 'until nc -vz mayastor-nats 4222; do echo "Waiting for message bus..."; sleep 1; done;'] containers: - - name: mayastor-node + - name: mayastor image: {{ include "mayastor.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} env: diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index c2743c87..b745471d 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -1,3 +1,16 @@ + +{{- define "mayastor.agentCore.fullname" -}} + {{- printf "%s-agent-core" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end -}} + +{{- define "mayastor.apiRest.fullname" -}} + {{- printf "%s-api-rest" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end -}} + +{{- define "mayastor.etcd.fullname" -}} + {{- printf "%s-etcd" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end -}} + {{/* Create the name of the service account to use for the deployment */}} {{- define "mayastor.serviceAccountName" -}} {{- if .Values.serviceAccount.create -}} @@ -16,20 +29,15 @@ {{- end -}} {{- end -}} -{{/* Return the proper Mayastor CSI Node image name */}} -{{- define "mayastor.csi.image" -}} +{{/* Return the proper Mayastor Agent Core image name */}} +{{- define "mayastor.agentCore.image" -}} {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} - {{ include "common.images.image" (dict "imageRoot" .Values.csi.imageDev "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.agentCore.imageDev "global" .Values.global) }} {{- else -}} - {{ include "common.images.image" (dict "imageRoot" .Values.csi.image "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.agentCore.image "global" .Values.global) }} {{- end -}} {{- end -}} -{{/* Return the proper Mayastor CSI Node image name */}} -{{- define "mayastor.csiNode.driverRegistrar.image" -}} - {{ include "common.images.image" (dict "imageRoot" .Values.csi.driverRegistrar.image "global" .Values.global) }} -{{- end -}} - {{/* Return the proper Mayastor CSI Controller image name */}} {{- define "mayastor.csiController.image" -}} {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} @@ -54,21 +62,26 @@ {{ include "common.images.image" (dict "imageRoot" .Values.csiController.provisioner.image "global" .Values.global) }} {{- end -}} -{{/* Return the proper Mayastor Core Agent image name */}} -{{- define "mayastor.mcpCore.image" -}} +{{/* Return the proper Mayastor CSI Node image name */}} +{{- define "mayastor.csiNode.image" -}} {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} - {{ include "common.images.image" (dict "imageRoot" .Values.mcpCore.imageDev "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.csiNode.imageDev "global" .Values.global) }} {{- else -}} - {{ include "common.images.image" (dict "imageRoot" .Values.mcpCore.image "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.csiNode.image "global" .Values.global) }} {{- end -}} {{- end -}} +{{/* Return the proper Mayastor CSI Node image name */}} +{{- define "mayastor.csiNode.driverRegistrar.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.csiNode.driverRegistrar.image "global" .Values.global) }} +{{- end -}} + {{/* Return the proper Mayastor Rest image name */}} -{{- define "mayastor.mcpRest.image" -}} +{{- define "mayastor.apiRest.image" -}} {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} - {{ include "common.images.image" (dict "imageRoot" .Values.mcpRest.imageDev "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.apiRest.imageDev "global" .Values.global) }} {{- else -}} - {{ include "common.images.image" (dict "imageRoot" .Values.mcpRest.image "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.apiRest.image "global" .Values.global) }} {{- end -}} {{- end -}} diff --git a/charts/mayastor/templates/mcp-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml similarity index 73% rename from charts/mayastor/templates/mcp-core/Deployment.yaml rename to charts/mayastor/templates/agent-core/Deployment.yaml index 42b1715b..e77a7a35 100644 --- a/charts/mayastor/templates/mcp-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -1,13 +1,13 @@ -## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/core-agents-deployment.yaml +## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/agent-core-deployment.yaml -{{- if .Values.mcpCore.enabled }} +{{- if .Values.agentCore.enabled }} apiVersion: apps/v1 kind: Deployment metadata: - name: {{ printf "%s-core-agents" (include "common.names.fullname" .) }} + name: {{ include "mayastor.agentCore.fullname" . }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: core-agents + app.kubernetes.io/component: agent-core {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} @@ -19,7 +19,7 @@ spec: revisionHistoryLimit: 2 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} - app.kubernetes.io/component: core-agents + app.kubernetes.io/component: agent-core template: metadata: {{- if .Values.podAnnotations }} @@ -27,7 +27,7 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} {{- end }} labels: {{- include "common.labels.standard" . | nindent 8 }} - app.kubernetes.io/component: core-agents + app.kubernetes.io/component: agent-core {{- if .Values.podLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} @@ -46,19 +46,7 @@ spec: - -c - >- trap "exit 1" TERM; - until nc -vz mayastor-nats 4222; - do - echo "Waiting for nats..."; - sleep 1; - done; - image: busybox:latest - name: nats-probe - - command: - - sh - - -c - - >- - trap "exit 1" TERM; - until nc -vz mayastor-etcd 2379; + until nc -vzw 5 {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.port }}; do echo "Waiting for etcd..."; sleep 1; @@ -66,7 +54,8 @@ spec: image: busybox:latest name: etcd-probe containers: - - name: core + - name: agent-core + image: {{ include "mayastor.agentCore.image" . }} {{- if .Values.diagnosticMode.enabled }} command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} {{- else if .Values.command }} @@ -80,10 +69,11 @@ spec: args: # - "-smayastor-etcd" # - "-nmayastor-nats" - - "--nats=mayastor-nats" - - "--store=mayastor-etcd" + # - "--nats={{ $natsName }}" + - "--store={{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" - "--request-timeout=5s" - "--cache-period=30s" + - "--grpc-server-addr=https://0.0.0.0:{{ .Values.agentCore.containerPorts.port }}" {{- end }} env: - name: MY_POD_NAME @@ -95,11 +85,18 @@ spec: fieldRef: fieldPath: metadata.namespace - name: RUST_LOG - value: info + value: {{ .Values.agentCore.logLevel }} - name: RUST_BACKTRACE + {{- if .Values.diagnosticMode.enabled }} value: full - image: {{ include "mayastor.mcpCore.image" . }} - imagePullPolicy: {{ .Values.mcpCore.image.pullPolicy | quote }} + {{- else }} + value: "1" + {{- end }} + imagePullPolicy: {{ .Values.agentCore.image.pullPolicy | quote }} + ports: + - name: grpc + containerPort: {{ .Values.agentCore.containerPorts.grpc }} + protocol: TCP resources: limits: cpu: 1000m diff --git a/charts/mayastor/templates/agent-core/Service.yaml b/charts/mayastor/templates/agent-core/Service.yaml new file mode 100644 index 00000000..7d3be058 --- /dev/null +++ b/charts/mayastor/templates/agent-core/Service.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mayastor.agentCore.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: agent-core + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if or .Values.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.service.annotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.annotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.annotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.agentCore.service.type }} + {{- if and (eq .Values.agentCore.service.type "LoadBalancer") (not (empty .Values.service.allocateLoadBalancerNodePorts)) }} + allocateLoadBalancerNodePorts: {{ .Values.service.allocateLoadBalancerNodePorts }} + {{- end }} + {{- if and .Values.service.clusterIP (eq .Values.agentCore.service.type "ClusterIP") }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if and .Values.service.externalTrafficPolicy (or (eq .Values.agentCore.service.type "LoadBalancer") (eq .Values.agentCore.service.type "NodePort")) }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | quote }} + {{- end }} + ipFamilyPolicy: {{ .Values.service.ipFamilyPolicy }} + {{- if and (eq .Values.agentCore.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerClass)) }} + loadBalancerClass: {{ .Values.service.loadBalancerClass }} + {{- end }} + {{- if (and (eq .Values.agentCore.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP))) }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if and (eq .Values.agentCore.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerSourceRanges)) }} + loadBalancerSourceRanges: {{ .Values.service.loadBalancerSourceRanges }} + {{- end }} + {{- if .Values.service.sessionAffinity }} + sessionAffinity: {{ .Values.service.sessionAffinity }} + {{- end }} + {{- if .Values.service.sessionAffinityConfig }} + sessionAffinityConfig: {{- include "common.tplvalues.render" (dict "value" .Values.service.sessionAffinityConfig "context" $) | nindent 4 }} + {{- end }} + ports: + - name: grpc + port: {{ .Values.agentCore.service.ports.grpc }} + protocol: TCP + targetPort: {{ .Values.agentCore.containerPorts.grpc }} + {{- if (and (or (eq .Values.agentCore.service.type "NodePort") (eq .Values.agentCore.service.type "LoadBalancer")) .Values.agentCore.service.nodePorts.grpc) }} + nodePort: {{ coalesce .Values.agentCore.service.nodePorts.http .Values.agentCore.service.nodePort }} + {{- else if eq .Values.agentCore.service.type "ClusterIP" }} + nodePort: null + {{- end }} + selector: {{ include "common.labels.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: agent-core +--- \ No newline at end of file diff --git a/charts/mayastor/templates/mcp-rest/Deployment.yaml b/charts/mayastor/templates/api-rest/Deployment.yaml similarity index 71% rename from charts/mayastor/templates/mcp-rest/Deployment.yaml rename to charts/mayastor/templates/api-rest/Deployment.yaml index c7b1937b..0563966f 100644 --- a/charts/mayastor/templates/mcp-rest/Deployment.yaml +++ b/charts/mayastor/templates/api-rest/Deployment.yaml @@ -1,13 +1,13 @@ ## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/rest-deployment.yaml -{{- if .Values.mcpRest.enabled }} +{{- if .Values.apiRest.enabled }} apiVersion: apps/v1 kind: Deployment metadata: - name: {{ printf "%s-rest" (include "common.names.fullname" .) }} + name: {{ printf "%s-api-rest" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: mcp-rest + app.kubernetes.io/component: api-rest {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} @@ -19,7 +19,7 @@ spec: revisionHistoryLimit: 2 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} - app.kubernetes.io/component: mcp-rest + app.kubernetes.io/component: api-rest template: metadata: {{- if .Values.podAnnotations }} @@ -27,7 +27,7 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} {{- end }} labels: {{- include "common.labels.standard" . | nindent 8 }} - app.kubernetes.io/component: mcp-rest + app.kubernetes.io/component: api-rest {{- if .Values.podLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} @@ -44,13 +44,7 @@ spec: - command: - sh - -c - - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep 1; done; - image: busybox:latest - name: nats-probe - - command: - - sh - - -c - - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; sleep 1; done; + - trap "exit 1" TERM; until nc -vz {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.port }}; do echo "Waiting for etcd..."; sleep 1; done; image: busybox:latest name: etcd-probe containers: @@ -62,20 +56,20 @@ spec: requests: cpu: 50m memory: 32Mi - image: {{ include "mayastor.mcpRest.image" . }} - imagePullPolicy: {{ .Values.mcpRest.image.pullPolicy | quote }} + image: {{ include "mayastor.apiRest.image" . }} + imagePullPolicy: {{ .Values.apiRest.image.pullPolicy | quote }} args: - "--dummy-certificates" - "--no-auth" - - "-nmayastor-nats" - - "--http=0.0.0.0:8081" + - "--http=0.0.0.0:{{ .Values.apiRest.service.ports.https }}" - "--request-timeout=5s" + - "--core-grpc=https://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.ports.grpc }}" ports: - name: http - containerPort: {{ .Values.mcpRest.containerPorts.http }} + containerPort: {{ .Values.apiRest.containerPorts.http }} protocol: TCP - name: https - containerPort: {{ .Values.mcpRest.containerPorts.https }} + containerPort: {{ .Values.apiRest.containerPorts.https }} protocol: TCP env: - name: RUST_LOG diff --git a/charts/mayastor/templates/mcp-rest/Service.yaml b/charts/mayastor/templates/api-rest/Service.yaml similarity index 61% rename from charts/mayastor/templates/mcp-rest/Service.yaml rename to charts/mayastor/templates/api-rest/Service.yaml index 27ba44d6..78ee011b 100644 --- a/charts/mayastor/templates/mcp-rest/Service.yaml +++ b/charts/mayastor/templates/api-rest/Service.yaml @@ -1,10 +1,10 @@ apiVersion: v1 kind: Service metadata: - name: {{ printf "%s-rest" (include "common.names.fullname" .) }} + name: {{ printf "%s-api-rest" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: mcp-rest + app.kubernetes.io/component: api-rest {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} @@ -21,24 +21,24 @@ metadata: {{- end }} {{- end }} spec: - type: {{ .Values.mcpRest.service.type }} - {{- if and (eq .Values.mcpRest.service.type "LoadBalancer") (not (empty .Values.service.allocateLoadBalancerNodePorts)) }} + type: {{ .Values.apiRest.service.type }} + {{- if and (eq .Values.apiRest.service.type "LoadBalancer") (not (empty .Values.service.allocateLoadBalancerNodePorts)) }} allocateLoadBalancerNodePorts: {{ .Values.service.allocateLoadBalancerNodePorts }} {{- end }} - {{- if and .Values.service.clusterIP (eq .Values.mcpRest.service.type "ClusterIP") }} + {{- if and .Values.service.clusterIP (eq .Values.apiRest.service.type "ClusterIP") }} clusterIP: {{ .Values.service.clusterIP }} {{- end }} - {{- if and .Values.service.externalTrafficPolicy (or (eq .Values.mcpRest.service.type "LoadBalancer") (eq .Values.mcpRest.service.type "NodePort")) }} + {{- if and .Values.service.externalTrafficPolicy (or (eq .Values.apiRest.service.type "LoadBalancer") (eq .Values.apiRest.service.type "NodePort")) }} externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | quote }} {{- end }} ipFamilyPolicy: {{ .Values.service.ipFamilyPolicy }} - {{- if and (eq .Values.mcpRest.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerClass)) }} + {{- if and (eq .Values.apiRest.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerClass)) }} loadBalancerClass: {{ .Values.service.loadBalancerClass }} {{- end }} - {{- if (and (eq .Values.mcpRest.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP))) }} + {{- if (and (eq .Values.apiRest.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP))) }} loadBalancerIP: {{ .Values.service.loadBalancerIP }} {{- end }} - {{- if and (eq .Values.mcpRest.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerSourceRanges)) }} + {{- if and (eq .Values.apiRest.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerSourceRanges)) }} loadBalancerSourceRanges: {{ .Values.service.loadBalancerSourceRanges }} {{- end }} {{- if .Values.service.sessionAffinity }} @@ -49,25 +49,25 @@ spec: {{- end }} ports: - name: http - port: {{ .Values.mcpRest.service.ports.http }} - protocol: UDP - targetPort: {{ .Values.mcpRest.containerPorts.http }} - {{- if (and (or (eq .Values.mcpRest.service.type "NodePort") (eq .Values.mcpRest.service.type "LoadBalancer")) .Values.mcpRest.service.nodePorts.http) }} - nodePort: {{ coalesce .Values.mcpRest.service.nodePorts.http .Values.mcpRest.service.nodePort }} - {{- else if eq .Values.mcpRest.service.type "ClusterIP" }} + port: {{ .Values.apiRest.service.ports.http }} + protocol: TCP + targetPort: {{ .Values.apiRest.containerPorts.http }} + {{- if (and (or (eq .Values.apiRest.service.type "NodePort") (eq .Values.apiRest.service.type "LoadBalancer")) .Values.apiRest.service.nodePorts.http) }} + nodePort: {{ coalesce .Values.apiRest.service.nodePorts.http .Values.apiRest.service.nodePort }} + {{- else if eq .Values.apiRest.service.type "ClusterIP" }} nodePort: null {{- end }} - {{- if .Values.mcpRest.tls.enabled }} + {{- if .Values.apiRest.tls.enabled }} - name: https - port: {{ .Values.mcpRest.service.ports.https }} + port: {{ .Values.apiRest.service.ports.https }} protocol: TCP - targetPort: {{ .Values.mcpRest.containerPorts.https }} - {{- if (and (or (eq .Values.mcpRest.service.type "NodePort") (eq .Values.mcpRest.service.type "LoadBalancer")) .Values.mcpRest.service.nodePorts.https) }} - nodePort: {{ .Values.mcpRest.service.nodePorts.https }} - {{- else if eq .Values.mcpRest.service.type "ClusterIP" }} + targetPort: {{ .Values.apiRest.containerPorts.https }} + {{- if (and (or (eq .Values.apiRest.service.type "NodePort") (eq .Values.apiRest.service.type "LoadBalancer")) .Values.apiRest.service.nodePorts.https) }} + nodePort: {{ .Values.apiRest.service.nodePorts.https }} + {{- else if eq .Values.apiRest.service.type "ClusterIP" }} nodePort: null {{- end }} {{- end }} selector: {{ include "common.labels.matchLabels" . | nindent 4 }} - app.kubernetes.io/component: mcp-rest + app.kubernetes.io/component: api-rest --- \ No newline at end of file diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index b32f3f64..0a97f061 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -43,18 +43,20 @@ spec: topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} {{- end }} initContainers: - - command: + - name: api-rest-probe + image: busybox:latest + command: - sh + args: - -c - - >- - trap "exit 1" TERM; - until nc -vz mayastor-rest 8081; + - | + trap "exit 1" TERM + until nc -vz {{ include "mayastor.apiRest.fullname" . }} 8081 do - echo "Waiting for REST API endpoint to become available"; - sleep 1; - done; - image: busybox:latest - name: rest-probe + date + echo "Waiting for REST API endpoint to become available" + sleep 1 + done containers: - name: csi-provisioner image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0 @@ -95,10 +97,13 @@ spec: imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} args: - "--csi-socket=/var/lib/csi/sockets/pluginproxy/csi.sock" - - "--rest-endpoint=http://mayastor-rest:8081" + - "--rest-endpoint=http://{{ include "mayastor.apiRest.fullname" . }}:8081" + {{- range $key, $val := .Values.nodeSelector }} + - "--io-engine-selector={{ $key }}:{{ $val }}" + {{- end }} env: - name: RUST_LOG - value: info + value: {{ .Values.csiController.logLevel }} - name: RUST_BACKTRACE value: full volumeMounts: diff --git a/charts/mayastor/templates/csi/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml similarity index 83% rename from charts/mayastor/templates/csi/DaemonSet.yaml rename to charts/mayastor/templates/csi-node/DaemonSet.yaml index 6db2bfe2..6ffa4d5e 100644 --- a/charts/mayastor/templates/csi/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -1,9 +1,10 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ printf "%s-csi" (include "common.names.fullname" .) }} + name: {{ printf "%s-csi-node" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} + openebs.io/engine: io-engine app.kubernetes.io/component: csi-node {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} @@ -34,9 +35,9 @@ spec: {{- end }} spec: hostNetwork: true - nodeSelector: - openebs.io/engine: mayastor - kubernetes.io/arch: amd64 + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- if .Values.tolerations }} @@ -57,11 +58,7 @@ spec: command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} {{- else }} command: - {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") }} - csi-node - {{- else }} - - mayastor-csi - {{- end }} {{- end }} {{- if .Values.diagnosticMode.enabled }} args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} @@ -72,18 +69,19 @@ spec: - "--csi-socket=$(CSI_SOCKET_ADDRESS)" - "--node-name=$(MY_NODE_NAME)" - "--grpc-endpoint=$(MY_POD_IP):10199" - {{- if .Values.csi.nvme.io_timeout_enabled }} - - "--nvme-core-io-timeout={{ .Values.csi.nvme.io_timeout }}" + {{- if .Values.csiNode.nvme.io_timeout_enabled }} + - "--nvme-core-io-timeout={{ .Values.csiNode.nvme.io_timeout }}" {{- end }} + - "--nvme-nr-io-queues=2" - "-v" {{- end }} - image: {{ include "mayastor.csi.image" . }} + image: {{ include "mayastor.csiNode.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} securityContext: privileged: true env: - name: CSI_SOCKET_ADDRESS - value: {{ .Values.csi.socketAddress | quote }} + value: {{ .Values.csiNode.socketAddress | quote }} - name: MY_NODE_NAME valueFrom: fieldRef: @@ -93,6 +91,8 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: RUST_LOG + value: {{ .Values.csiNode.logLevel }} - name: RUST_BACKTRACE value: full {{- if .Values.moac }} @@ -113,8 +113,6 @@ spec: mountPath: /sys - name: run-udev mountPath: /run/udev - - name: host-root - mountPath: /host - name: plugin-dir mountPath: /csi - name: kubelet-dir @@ -125,11 +123,11 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy | quote }} args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/mayastor.openebs.io/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/io.openebs.mayastor/csi.sock" - "--v=5" env: - name: CSI_SOCKET_ADDRESS - value: {{ .Values.csi.socketAddress | quote }} + value: {{ .Values.csiNode.socketAddress | quote }} lifecycle: preStop: exec: @@ -138,7 +136,7 @@ spec: - -c - >- echo "Deleting stale file"; - rm -rf /registration/mayastor.openebs.io /registration/mayastor.openebs.io-reg.sock; + rm -rf /registration/io.openebs.mayastor /registration/io.openebs.mayastor-reg.sock; # Mayastor node plugin gRPC server ports: - containerPort: 10199 @@ -169,17 +167,13 @@ spec: hostPath: path: /run/udev type: Directory - - name: host-root - hostPath: - path: / - type: Directory - name: registration-dir hostPath: path: /var/lib/kubelet/plugins_registry/ type: Directory - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/mayastor.openebs.io/ + path: /var/lib/kubelet/plugins/io.openebs.mayastor/ type: DirectoryOrCreate - name: kubelet-dir hostPath: diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index a2d2864f..196eef69 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -152,7 +152,9 @@ affinity: {} ## @param nodeSelector Node labels for Mayastor pods assignment ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ ## -nodeSelector: {} +nodeSelector: + openebs.io/engine: mayastor + kubernetes.io/arch: amd64 ## @param tolerations Tolerations for Mayastor pods assignment ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## @@ -343,8 +345,9 @@ service: ## timeoutSeconds: 300 sessionAffinityConfig: {} -csi: +csiNode: enabled: true + logLevel: info imageDev: registry: docker.io repository: mayadata/mayastor-csi-node @@ -393,6 +396,7 @@ csi: csiController: enabled: true + logLevel: info imageDev: registry: docker.io repository: mayadata/mayastor-csi-controller @@ -541,7 +545,7 @@ etcd: podAntiAffinityPreset: "hard" nats: - enabled: true + enabled: false ## Number of replicas ## replicaCount: 1 @@ -588,8 +592,9 @@ nats: metrics: enabled: false -mcpCore: +agentCore: enabled: true + logLevel: info imageDev: registry: docker.io repository: mayadata/mayastor-agent-core @@ -619,8 +624,69 @@ mcpCore: ## - myRegistryKeySecretName ## pullSecrets: [] + ## @param agentCore.containerPorts.http Mayastor Agent Core GRPC container port + ## + containerPorts: + grpc: 50051 + service: + ## @param service.type Mayastor Control Plane Rest Kubernetes service type + ## + type: ClusterIP + ## @param service.ports.auth Mayastor Control Plane Rest Kubernetes service port + ## + ports: + grpc: 50051 + ## @param service.nodePorts.mysql Mayastor Control Plane Rest Kubernetes service node port + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + grpc: "" + ## @param service.clusterIP Mayastor Control Plane Rest Kubernetes service clusterIP IP + ## + clusterIP: "" + ## @param service.loadBalancerIP Mayastor Control Plane Rest loadBalancerIP if service type is `LoadBalancer` + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + loadBalancerIP: "" + ## @param service.ipFamilyPolicy Mayastor Control Plane Rest Kubernetes service ipFamilyPolicy policy + ## + ipFamilyPolicy: SingleStack + ## @param service.externalTrafficPolicy Enable client source IP preservation + ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + ## @param service.allocateLoadBalancerNodePorts Allow users to disable node ports for Service Type=LoadBalancer. This is useful for + ## bare metal / on-prem environments that rely on VIP based LB implementations. + allocateLoadBalancerNodePorts: false + ## @param service.loadBalancerClass Enables to use a load balancer implementation other than the cloud provider default. + ## https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-class + ## + loadBalancerClass: "" + ## @param service.loadBalancerSourceRanges Address that are allowed when Mayastor Control Plane Rest service is LoadBalancer + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## E.g. + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## @param service.extraPorts Extra ports to expose (normally used with the `sidecar` value) + ## + extraPorts: [] + ## @param service.annotations Provide any additional annotations which may be required + ## + annotations: {} + ## @param service.sessionAffinity Session Affinity for Kubernetes service, can be "None" or "ClientIP" + ## If "ClientIP", consecutive client requests will be directed to the same Pod + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + ## + sessionAffinity: None + ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity + ## sessionAffinityConfig: + ## clientIP: + ## timeoutSeconds: 300 + sessionAffinityConfig: {} -mcpRest: +apiRest: enabled: true imageDev: registry: docker.io @@ -651,8 +717,8 @@ mcpRest: ## - myRegistryKeySecretName ## pullSecrets: [] - ## @param mcpRest.containerPorts.auth Mayastor Control Plane Rest HTTP container port - ## @param mcpRest.containerPorts.acct Mayastor Control Plane Rest HTTPS container port + ## @param mcpRest.containerPorts.http Mayastor Control Plane Rest HTTP container port + ## @param mcpRest.containerPorts.https Mayastor Control Plane Rest HTTPS container port ## containerPorts: http: 8080 @@ -670,6 +736,7 @@ mcpRest: ports: http: 8080 https: 8081 + grpc: 50051 ## @param service.nodePorts.mysql Mayastor Control Plane Rest Kubernetes service node port ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport ## From 21158f343274602e40ba388ea4456887462f6d5b Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 03:34:37 +0700 Subject: [PATCH 068/188] Fix many things --- charts/mayastor/templates/_helper.tpl | 14 +- .../templates/api-rest/Deployment.yaml | 2 +- .../mayastor/templates/api-rest/Service.yaml | 2 +- .../mayastor/templates/metrics/DaemonSet.yaml | 124 ++++++++++++++++++ .../mayastor/templates/metrics/Service.yaml | 63 +++++++++ .../Deployment.yaml | 26 ++-- charts/mayastor/values.yaml | 4 +- 7 files changed, 213 insertions(+), 22 deletions(-) create mode 100644 charts/mayastor/templates/metrics/DaemonSet.yaml create mode 100644 charts/mayastor/templates/metrics/Service.yaml rename charts/mayastor/templates/{msp-operator => operator}/Deployment.yaml (75%) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index b745471d..07d8330e 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -11,6 +11,14 @@ {{- printf "%s-etcd" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} {{- end -}} +{{- define "mayastor.metrics.fullname" -}} + {{- printf "%s-metrics" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end -}} + +{{- define "mayastor.operator.fullname" -}} + {{- printf "%s-operator" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end -}} + {{/* Create the name of the service account to use for the deployment */}} {{- define "mayastor.serviceAccountName" -}} {{- if .Values.serviceAccount.create -}} @@ -86,11 +94,11 @@ {{- end -}} {{/* Return the proper Mayastor Operator image name */}} -{{- define "mayastor.mspOperator.image" -}} +{{- define "mayastor.operator.image" -}} {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} - {{ include "common.images.image" (dict "imageRoot" .Values.mspOperator.imageDev "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.operator.imageDev "global" .Values.global) }} {{- else -}} - {{ include "common.images.image" (dict "imageRoot" .Values.mspOperator.image "global" .Values.global) }} + {{ include "common.images.image" (dict "imageRoot" .Values.operator.image "global" .Values.global) }} {{- end -}} {{- end -}} diff --git a/charts/mayastor/templates/api-rest/Deployment.yaml b/charts/mayastor/templates/api-rest/Deployment.yaml index 0563966f..8a43ecb9 100644 --- a/charts/mayastor/templates/api-rest/Deployment.yaml +++ b/charts/mayastor/templates/api-rest/Deployment.yaml @@ -4,7 +4,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ printf "%s-api-rest" (include "common.names.fullname" .) }} + name: {{ include "mayastor.apiRest.fullname" . }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} app.kubernetes.io/component: api-rest diff --git a/charts/mayastor/templates/api-rest/Service.yaml b/charts/mayastor/templates/api-rest/Service.yaml index 78ee011b..767c7752 100644 --- a/charts/mayastor/templates/api-rest/Service.yaml +++ b/charts/mayastor/templates/api-rest/Service.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ printf "%s-api-rest" (include "common.names.fullname" .) }} + name: {{ include "mayastor.apiRest.fullname" . }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} app.kubernetes.io/component: api-rest diff --git a/charts/mayastor/templates/metrics/DaemonSet.yaml b/charts/mayastor/templates/metrics/DaemonSet.yaml new file mode 100644 index 00000000..7ae32cf9 --- /dev/null +++ b/charts/mayastor/templates/metrics/DaemonSet.yaml @@ -0,0 +1,124 @@ +{{- if .Values.metrics.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ .Release.Name }}-io-engine + labels: + openebs.io/engine: io-engine +spec: + selector: + matchLabels: + app: io-engine + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + minReadySeconds: 10 + template: + metadata: + labels: + app: io-engine + openebs.io/logging: "true" + spec: + imagePullSecrets: + {{- include "base_pull_secrets" . }} + hostNetwork: true + # To resolve services in the namespace + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: {{- .Values.mayastor.nodeSelector | toYaml | nindent 8 }} + initContainers: + {{- include "base_init_containers" . }} + containers: + - name: metrics-exporter-pool + image: "{{ .Values.image.registry }}/{{ .Values.image.repo }}/mayastor-metrics-exporter-pool:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + command: + - metrics-exporter-pool + ports: + - containerPort: 9502 + protocol: TCP + name: metrics + - name: io-engine + image: "{{ .Values.image.registry }}/{{ .Values.image.repo }}/mayastor-io-engine:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: RUST_LOG + value: {{ .Values.mayastor.logLevel }} + - name: NVME_QPAIR_CONNECT_ASYNC + value: "true" + - name: NVMF_TCP_MAX_QUEUE_DEPTH + value: "32" + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + args: + # The -l argument accepts cpu-list. Indexing starts at zero. + # For example -l 1,2,10-20 means use core 1, 2, 10 to 20. + # Note: Ensure that the CPU resources are updated accordingly. + # If you use 2 CPUs, the CPU: field should also read 2. + - "-g$(MY_POD_IP)" + - "-Rhttps://{{ .Release.Name }}-agent-core:50051" + - "-y/var/local/io-engine/config.yaml" + - "-l{{ include "cpuFlag" . }}" + - "-p={{ .Release.Name }}-etcd:{{ .Values.etcd.service.port }}" + command: + - io-engine + securityContext: + privileged: true + volumeMounts: + - name: device + mountPath: /dev + - name: udev + mountPath: /run/udev + - name: dshm + mountPath: /dev/shm + - name: configlocation + mountPath: /var/local/io-engine/ + resources: + limits: + cpu: {{ .Values.mayastor.resources.limits.cpu | default .Values.mayastor.cpuCount | quote }} + memory: {{ .Values.mayastor.resources.limits.memory | quote }} + hugepages-2Mi: {{ .Values.mayastor.resources.limits.hugepages2Mi | quote }} + requests: + cpu: {{ .Values.mayastor.resources.requests.cpu | default .Values.mayastor.cpuCount | quote }} + memory: {{ .Values.mayastor.resources.requests.memory | quote }} + hugepages-2Mi: {{ .Values.mayastor.resources.requests.hugepages2Mi | quote }} + ports: + - containerPort: 10124 + protocol: TCP + name: io-engine + volumes: + - name: device + hostPath: + path: /dev + type: Directory + - name: udev + hostPath: + path: /run/udev + type: Directory + - name: dshm + emptyDir: + medium: Memory + sizeLimit: "1Gi" + - name: hugepage + emptyDir: + medium: HugePages + - name: configlocation + hostPath: + path: /var/local/io-engine/ + type: DirectoryOrCreate +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/metrics/Service.yaml b/charts/mayastor/templates/metrics/Service.yaml new file mode 100644 index 00000000..fbf19cc6 --- /dev/null +++ b/charts/mayastor/templates/metrics/Service.yaml @@ -0,0 +1,63 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mayastor.metrics.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: metrics + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if or .Values.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.service.annotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.annotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.annotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.metrics.service.type }} + {{- if and (eq .Values.metrics.service.type "LoadBalancer") (not (empty .Values.service.allocateLoadBalancerNodePorts)) }} + allocateLoadBalancerNodePorts: {{ .Values.service.allocateLoadBalancerNodePorts }} + {{- end }} + {{- if and .Values.service.clusterIP (eq .Values.metrics.service.type "ClusterIP") }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if and .Values.service.externalTrafficPolicy (or (eq .Values.metrics.service.type "LoadBalancer") (eq .Values.metrics.service.type "NodePort")) }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | quote }} + {{- end }} + ipFamilyPolicy: {{ .Values.service.ipFamilyPolicy }} + {{- if and (eq .Values.metrics.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerClass)) }} + loadBalancerClass: {{ .Values.service.loadBalancerClass }} + {{- end }} + {{- if (and (eq .Values.metrics.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP))) }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if and (eq .Values.metrics.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerSourceRanges)) }} + loadBalancerSourceRanges: {{ .Values.service.loadBalancerSourceRanges }} + {{- end }} + {{- if .Values.service.sessionAffinity }} + sessionAffinity: {{ .Values.service.sessionAffinity }} + {{- end }} + {{- if .Values.service.sessionAffinityConfig }} + sessionAffinityConfig: {{- include "common.tplvalues.render" (dict "value" .Values.service.sessionAffinityConfig "context" $) | nindent 4 }} + {{- end }} + ports: + - name: tcp-metrics + port: {{ .Values.metrics.service.ports.http }} + protocol: TCP + targetPort: {{ .Values.metrics.containerPorts.http }} + {{- if (and (or (eq .Values.metrics.service.type "NodePort") (eq .Values.metrics.service.type "LoadBalancer")) .Values.metrics.service.nodePorts.http) }} + nodePort: {{ coalesce .Values.metrics.service.nodePorts.http .Values.metrics.service.nodePort }} + {{- else if eq .Values.metrics.service.type "ClusterIP" }} + nodePort: null + {{- end }} + selector: {{ include "common.labels.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: metrics +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/msp-operator/Deployment.yaml b/charts/mayastor/templates/operator/Deployment.yaml similarity index 75% rename from charts/mayastor/templates/msp-operator/Deployment.yaml rename to charts/mayastor/templates/operator/Deployment.yaml index 0cdeac74..9edcec5c 100644 --- a/charts/mayastor/templates/msp-operator/Deployment.yaml +++ b/charts/mayastor/templates/operator/Deployment.yaml @@ -1,13 +1,13 @@ ##ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/msp-deployment.yaml -{{- if .Values.mspOperator.enabled }} +{{- if .Values.operator.enabled }} apiVersion: apps/v1 kind: Deployment metadata: - name: {{ printf "%s-msp-operator" (include "common.names.fullname" .) }} + name: {{ include "mayastor.operator.fullname" . }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: msp-operator + app.kubernetes.io/component: operator {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} {{- end }} @@ -16,10 +16,10 @@ metadata: {{- end }} spec: replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: 2 + revisionHistoryLimit: 1 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} - app.kubernetes.io/component: msp-operator + app.kubernetes.io/component: operator template: metadata: {{- if .Values.podAnnotations }} @@ -27,7 +27,7 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} {{- end }} labels: {{- include "common.labels.standard" . | nindent 8 }} - app.kubernetes.io/component: msp-operator + app.kubernetes.io/component: operator {{- if .Values.podLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} @@ -41,12 +41,6 @@ spec: topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} {{- end }} initContainers: - - command: - - sh - - -c - - trap "exit 1" TERM; until nc -vz mayastor-nats 4222; do echo "Waiting for nats..."; sleep 1; done; - image: busybox:latest - name: nats-probe - command: - sh - -c @@ -54,11 +48,11 @@ spec: image: busybox:latest name: etcd-probe containers: - - name: msp-operator - image: {{ include "mayastor.mspOperator.image" . }} - imagePullPolicy: {{ .Values.mspOperator.image.pullPolicy | quote }} + - name: operator + image: {{ include "mayastor.operator.image" . }} + imagePullPolicy: {{ .Values.operator.image.pullPolicy | quote }} args: - - "-e http://mayastor-rest:8081" + - "-e http://{{ include "mayastor.apiRest.fullname" . }}:{{ .Values.apiRest.containerPorts.https }}" - "--interval=30s" env: - name: MY_POD_NAME diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 196eef69..ef6d401f 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -788,7 +788,7 @@ apiRest: ## timeoutSeconds: 300 sessionAffinityConfig: {} -mspOperator: +operator: enabled: true imageDev: registry: docker.io @@ -870,3 +870,5 @@ rbac: ## rules: [] +metrics: + enabled: false \ No newline at end of file From 83fa22c05ee66ee38a3bf9cb45bdc14f4d2db8f5 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 03:36:38 +0700 Subject: [PATCH 069/188] Remove unused variable --- charts/mayastor/templates/agent-core/Deployment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index e77a7a35..472c9dc5 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -69,7 +69,6 @@ spec: args: # - "-smayastor-etcd" # - "-nmayastor-nats" - # - "--nats={{ $natsName }}" - "--store={{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" - "--request-timeout=5s" - "--cache-period=30s" From 002543bfbf67d86cd1d223cf34f32d8a31eaa0cf Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 03:55:02 +0700 Subject: [PATCH 070/188] Fix Mayastor DS --- charts/mayastor/templates/DaemonSet.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/templates/DaemonSet.yaml index c7411b2f..5cd7a408 100644 --- a/charts/mayastor/templates/DaemonSet.yaml +++ b/charts/mayastor/templates/DaemonSet.yaml @@ -48,10 +48,6 @@ spec: {{- if .Values.topologySpreadConstraints }} topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} {{- end }} - initContainers: - - name: message-bus-probe - image: busybox:latest - command: ['sh', '-c', 'until nc -vz mayastor-nats 4222; do echo "Waiting for message bus..."; sleep 1; done;'] containers: - name: mayastor image: {{ include "mayastor.image" . }} @@ -91,7 +87,6 @@ spec: # If you use 2 CPUs, the CPU: field should also read 2. - "-N$(MY_NODE_NAME)" - "-g$(MY_POD_IP)" - - "-nmayastor-nats" - "-y/var/local/mayastor/config.yaml" - "-l{{ include "mayastorCpuSpec" . }}" - "-pmayastor-etcd" From d75f64b91fb0ff1f9baca7cf6e868c8eab5124ee Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 04:00:42 +0700 Subject: [PATCH 071/188] Fix etcd ports --- charts/mayastor/values.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index ef6d401f..3dac24a4 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -527,22 +527,23 @@ etcd: ## # name: {{ include "mayastor.serviceAccountName" $ }} name: mayastor - ## Init containers parameters: ## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. ## volumePermissions: enabled: true - ## Set to true if you would like to see extra information on logs ## debug: false - ## Pod anti-affinity preset ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity ## Allowed values: soft, hard ## podAntiAffinityPreset: "hard" + # etcd service parameters defines how the etcd service is exposed + service: + # etcd client port + port: 2379 nats: enabled: false From dfe770e3cf48329011458f2af3488221b9afe9db Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 04:12:21 +0700 Subject: [PATCH 072/188] Fix agentCore --- charts/mayastor/templates/agent-core/Deployment.yaml | 5 ++--- charts/mayastor/templates/operator/Deployment.yaml | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index 472c9dc5..b5e022f5 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -48,6 +48,7 @@ spec: trap "exit 1" TERM; until nc -vzw 5 {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.port }}; do + date; echo "Waiting for etcd..."; sleep 1; done; @@ -67,12 +68,10 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} {{- else }} args: - # - "-smayastor-etcd" - # - "-nmayastor-nats" - "--store={{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" - "--request-timeout=5s" - "--cache-period=30s" - - "--grpc-server-addr=https://0.0.0.0:{{ .Values.agentCore.containerPorts.port }}" + - "--grpc-server-addr=https://0.0.0.0:{{ .Values.agentCore.containerPorts.grpc }}" {{- end }} env: - name: MY_POD_NAME diff --git a/charts/mayastor/templates/operator/Deployment.yaml b/charts/mayastor/templates/operator/Deployment.yaml index 9edcec5c..23d318ef 100644 --- a/charts/mayastor/templates/operator/Deployment.yaml +++ b/charts/mayastor/templates/operator/Deployment.yaml @@ -44,7 +44,7 @@ spec: - command: - sh - -c - - trap "exit 1" TERM; until nc -vz mayastor-etcd 2379; do echo "Waiting for etcd..."; sleep 1; done; + - trap "exit 1" TERM; until nc -vzw 5 mayastor-etcd 2379; do date; echo "Waiting for etcd..."; sleep 1; done; image: busybox:latest name: etcd-probe containers: From c348e1d16e1ae6930078fe2cc18d2e4a009233f6 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 04:19:10 +0700 Subject: [PATCH 073/188] Fix etcd --- charts/mayastor/values.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 3dac24a4..235ddd6c 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -460,6 +460,10 @@ etcd: clusterDomain: cluster.local podLabels: app.kubernetes.io/component: etcd + client: + secureTransport: false + peer: + secureTransport: false ## Authentication parameters ## auth: From 793c4c33b3897156f023cd511bc305ae55b2864e Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 04:19:28 +0700 Subject: [PATCH 074/188] Fix etcd --- charts/mayastor/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 235ddd6c..f63244ed 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -459,7 +459,7 @@ etcd: ## clusterDomain: cluster.local podLabels: - app.kubernetes.io/component: etcd + app.kubernetes.io/component: etcd client: secureTransport: false peer: From bf659d25048103a65da404747946177ed05dd598 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 04:22:31 +0700 Subject: [PATCH 075/188] Fix etcd --- charts/mayastor/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index f63244ed..e2528055 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -464,6 +464,8 @@ etcd: secureTransport: false peer: secureTransport: false + removeMemberOnContainerTermination: false + initialClusterState: "new" ## Authentication parameters ## auth: From a0de0c85714177f4b495874180001a9da90a2351 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 06:42:16 +0700 Subject: [PATCH 076/188] test: podAnnotations --- charts/mayastor/values-test.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 683abef2..d724cdce 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -43,4 +43,7 @@ mayastorPools: - node: rachel-lin-k8w.jkt2.mci.net.id device: /dev/sdb - node: verus-lin-k8w.jkt2.mci.net.id - device: /dev/sdb \ No newline at end of file + device: /dev/sdb + +podAnnotations: + v1.multus-cni.io/default-network: kube-public/calico-ipv4-only \ No newline at end of file From b59c0e0a6dd72a564e425117d26fb60d9cff44ad Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 06:48:17 +0700 Subject: [PATCH 077/188] test: DiskPool --- .../mayastor/templates/{MayastorPool.yaml => DiskPool.yaml} | 4 ++-- charts/mayastor/values-test.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename charts/mayastor/templates/{MayastorPool.yaml => DiskPool.yaml} (80%) diff --git a/charts/mayastor/templates/MayastorPool.yaml b/charts/mayastor/templates/DiskPool.yaml similarity index 80% rename from charts/mayastor/templates/MayastorPool.yaml rename to charts/mayastor/templates/DiskPool.yaml index 6d6f8592..91844c92 100644 --- a/charts/mayastor/templates/MayastorPool.yaml +++ b/charts/mayastor/templates/DiskPool.yaml @@ -1,8 +1,8 @@ ##ref: https://github.com/openebs/mayastor/blob/master/deploy/pool.yaml -{{- range .Values.mayastorPools }} +{{- range .Values.diskPools }} apiVersion: "openebs.io/v1alpha1" -kind: MayastorPool +kind: DiskPool metadata: name: pool-on-{{ .node }} namespace: {{ include "common.names.namespace" $ | quote }} diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index d724cdce..c008ca33 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -35,7 +35,7 @@ nats: enabled: false storageClass: openebs-hostpath -mayastorPools: +diskPools: - node: alberta-lin-k8m.jkt2.mci.net.id device: /dev/sdb - node: einbech-lin-k8m.jkt2.mci.net.id From 73798a7d844c044e770372308f2b409049b47d30 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 10:29:50 +0700 Subject: [PATCH 078/188] Add ioEngine --- .../{templates => archive}/DaemonSet.yaml | 0 charts/mayastor/templates/_helper.tpl | 17 +- .../templates/api-rest/Deployment.yaml | 2 +- .../templates/csi-node/DaemonSet.yaml | 8 +- .../templates/io-engine/DaemonSet.yaml | 163 ++++++++++++++ .../mayastor/templates/metrics/DaemonSet.yaml | 124 ---------- .../mayastor/templates/metrics/Service.yaml | 8 +- .../templates/metrics/ServiceMonitor.yaml | 49 ++++ charts/mayastor/values.yaml | 213 ++++++++++++++---- 9 files changed, 406 insertions(+), 178 deletions(-) rename charts/mayastor/{templates => archive}/DaemonSet.yaml (100%) create mode 100644 charts/mayastor/templates/io-engine/DaemonSet.yaml delete mode 100644 charts/mayastor/templates/metrics/DaemonSet.yaml create mode 100644 charts/mayastor/templates/metrics/ServiceMonitor.yaml diff --git a/charts/mayastor/templates/DaemonSet.yaml b/charts/mayastor/archive/DaemonSet.yaml similarity index 100% rename from charts/mayastor/templates/DaemonSet.yaml rename to charts/mayastor/archive/DaemonSet.yaml diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index 07d8330e..06a9b3e9 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -37,6 +37,12 @@ {{- end -}} {{- end -}} +{{/* Return the proper Mayastor I/O Engine image name */}} +{{- define "mayastor.ioEngine.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.ioEngine.image "global" .Values.global) }} +{{- end -}} +{{- end -}} + {{/* Return the proper Mayastor Agent Core image name */}} {{- define "mayastor.agentCore.image" -}} {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} @@ -93,6 +99,11 @@ {{- end -}} {{- end -}} +{{/* Return the proper image name (for the init container volume-permissions image) */}} +{{- define "mayastor.metrics.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.metrics.image "global" .Values.global) }} +{{- end -}} + {{/* Return the proper Mayastor Operator image name */}} {{- define "mayastor.operator.image" -}} {{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} @@ -113,10 +124,10 @@ {{- end -}} {{/* Generate CPU list specification based on CPU count (-l param of mayastor) */}} -{{- define "mayastorCpuSpec" -}} -{{- range $i, $e := until (int .Values.mayastorCpuCount) }} +{{- define "mayastor.cpuFlag" -}} +{{- range $i, $e := until (int .Values.ioEngine.cpuCount) }} {{- if gt $i 0 }} -{{- printf "," }} + {{- printf "," }} {{- end }} {{- printf "%d" (add $i 1) }} {{- end }} diff --git a/charts/mayastor/templates/api-rest/Deployment.yaml b/charts/mayastor/templates/api-rest/Deployment.yaml index 8a43ecb9..1d46d807 100644 --- a/charts/mayastor/templates/api-rest/Deployment.yaml +++ b/charts/mayastor/templates/api-rest/Deployment.yaml @@ -61,7 +61,7 @@ spec: args: - "--dummy-certificates" - "--no-auth" - - "--http=0.0.0.0:{{ .Values.apiRest.service.ports.https }}" + - "--http=0.0.0.0:{{ .Values.apiRest.containerPorts.https }}" - "--request-timeout=5s" - "--core-grpc=https://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.ports.grpc }}" ports: diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 6ffa4d5e..357cedc8 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -91,10 +91,14 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - - name: RUST_LOG - value: {{ .Values.csiNode.logLevel }} - name: RUST_BACKTRACE + {{- if .Values.diagnosticMode.enabled }} value: full + {{- else }} + value: "1" + {{- end }} + - name: RUST_LOG + value: {{ .Values.csiNode.logLevel }} {{- if .Values.moac }} - name: MOAC value: "true" diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml new file mode 100644 index 00000000..853e96c4 --- /dev/null +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -0,0 +1,163 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ printf "%s-io-engine" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: io-engine + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: io-engine + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + minReadySeconds: 10 + template: + metadata: + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "common.labels.standard" . | nindent 8 }} + app.kubernetes.io/component: io-engine + openebs.io/logging: "true" + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + spec: + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.nodeSelector "context" $) | nindent 8 }} + {{- end }} + initContainers: + - name: agent-core-probe + image: busybox:latest + command: + - sh + args: + - -c + - trap "exit 1" TERM; until nc -vzw 5 {{ include "mayastor.agentCore.fullname" . }} {{ .Values.agentCore.service.port.grpc }}; do date; echo "Waiting for agent-core-grpc services..."; sleep 1; done; + - name: etcd-probe + image: busybox:latest + command: + - sh + args: + - -c + - trap "exit 1" TERM; until nc -vzw 5 {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.port }}; do date; echo "Waiting for etcd..."; sleep 1; done; + containers: + - name: io-engine + image: {{ include "mayastor.ioEngine.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} + {{- else if .Values.command }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} + {{- else }} + command: + - io-engine + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if .Values.args }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} + {{- else }} + args: + # The -l argument accepts cpu-list. Indexing starts at zero. + # For example -l 1,2,10-20 means use core 1, 2, 10 to 20. + # Note: Ensure that the CPU resources are updated accordingly. + # If you use 2 CPUs, the CPU: field should also read 2. + - "-g $(MY_POD_IP)" + - "-l {{ include "mayastor.ioEngine.cpuFlag" . }}" + - "-p {{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" + - "-R https://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.port.grpc }}" + - "-y /var/local/io-engine/config.yaml" + {{- else }} + env: + - name: NVME_QPAIR_CONNECT_ASYNC + value: "true" + - name: NVMF_TCP_MAX_QUEUE_DEPTH + value: "32" + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: RUST_BACKTRACE + {{- if .Values.diagnosticMode.enabled }} + value: full + {{- else }} + value: "1" + {{- end }} + - name: RUST_LOG + value: {{ .Values.ioEngine.logLevel }} + securityContext: + privileged: true + volumeMounts: + - name: device + mountPath: /dev + - name: udev + mountPath: /run/udev + - name: dshm + mountPath: /dev/shm + - name: configlocation + mountPath: /var/local/io-engine/ + {{- if .Values.resources }} + resources: {{ toYaml .Values.resources | nindent 12 }} + {{- end }} + ports: + - containerPort: 10124 + protocol: TCP + name: io-engine + {{- if .Values.metrics.enabled }} + - name: metrics-exporter-pool + image: {{ include "mayastor.apiRest.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + command: + - metrics-exporter-pool + ports: + - name: http-metrics + containerPort: 9502 + protocol: TCP + {{- end }} + volumes: + - name: device + hostPath: + path: /dev + type: Directory + - name: udev + hostPath: + path: /run/udev + type: Directory + - name: dshm + emptyDir: + medium: Memory + sizeLimit: "1Gi" + - name: hugepage + emptyDir: + medium: HugePages + - name: configlocation + hostPath: + path: /var/local/io-engine/ + type: DirectoryOrCreate \ No newline at end of file diff --git a/charts/mayastor/templates/metrics/DaemonSet.yaml b/charts/mayastor/templates/metrics/DaemonSet.yaml deleted file mode 100644 index 7ae32cf9..00000000 --- a/charts/mayastor/templates/metrics/DaemonSet.yaml +++ /dev/null @@ -1,124 +0,0 @@ -{{- if .Values.metrics.enabled }} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ .Release.Name }}-io-engine - labels: - openebs.io/engine: io-engine -spec: - selector: - matchLabels: - app: io-engine - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - minReadySeconds: 10 - template: - metadata: - labels: - app: io-engine - openebs.io/logging: "true" - spec: - imagePullSecrets: - {{- include "base_pull_secrets" . }} - hostNetwork: true - # To resolve services in the namespace - dnsPolicy: ClusterFirstWithHostNet - nodeSelector: {{- .Values.mayastor.nodeSelector | toYaml | nindent 8 }} - initContainers: - {{- include "base_init_containers" . }} - containers: - - name: metrics-exporter-pool - image: "{{ .Values.image.registry }}/{{ .Values.image.repo }}/mayastor-metrics-exporter-pool:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - command: - - metrics-exporter-pool - ports: - - containerPort: 9502 - protocol: TCP - name: metrics - - name: io-engine - image: "{{ .Values.image.registry }}/{{ .Values.image.repo }}/mayastor-io-engine:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: RUST_LOG - value: {{ .Values.mayastor.logLevel }} - - name: NVME_QPAIR_CONNECT_ASYNC - value: "true" - - name: NVMF_TCP_MAX_QUEUE_DEPTH - value: "32" - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - args: - # The -l argument accepts cpu-list. Indexing starts at zero. - # For example -l 1,2,10-20 means use core 1, 2, 10 to 20. - # Note: Ensure that the CPU resources are updated accordingly. - # If you use 2 CPUs, the CPU: field should also read 2. - - "-g$(MY_POD_IP)" - - "-Rhttps://{{ .Release.Name }}-agent-core:50051" - - "-y/var/local/io-engine/config.yaml" - - "-l{{ include "cpuFlag" . }}" - - "-p={{ .Release.Name }}-etcd:{{ .Values.etcd.service.port }}" - command: - - io-engine - securityContext: - privileged: true - volumeMounts: - - name: device - mountPath: /dev - - name: udev - mountPath: /run/udev - - name: dshm - mountPath: /dev/shm - - name: configlocation - mountPath: /var/local/io-engine/ - resources: - limits: - cpu: {{ .Values.mayastor.resources.limits.cpu | default .Values.mayastor.cpuCount | quote }} - memory: {{ .Values.mayastor.resources.limits.memory | quote }} - hugepages-2Mi: {{ .Values.mayastor.resources.limits.hugepages2Mi | quote }} - requests: - cpu: {{ .Values.mayastor.resources.requests.cpu | default .Values.mayastor.cpuCount | quote }} - memory: {{ .Values.mayastor.resources.requests.memory | quote }} - hugepages-2Mi: {{ .Values.mayastor.resources.requests.hugepages2Mi | quote }} - ports: - - containerPort: 10124 - protocol: TCP - name: io-engine - volumes: - - name: device - hostPath: - path: /dev - type: Directory - - name: udev - hostPath: - path: /run/udev - type: Directory - - name: dshm - emptyDir: - medium: Memory - sizeLimit: "1Gi" - - name: hugepage - emptyDir: - medium: HugePages - - name: configlocation - hostPath: - path: /var/local/io-engine/ - type: DirectoryOrCreate -{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/metrics/Service.yaml b/charts/mayastor/templates/metrics/Service.yaml index fbf19cc6..8335c5d2 100644 --- a/charts/mayastor/templates/metrics/Service.yaml +++ b/charts/mayastor/templates/metrics/Service.yaml @@ -50,11 +50,11 @@ spec: {{- end }} ports: - name: tcp-metrics - port: {{ .Values.metrics.service.ports.http }} + port: {{ .Values.metrics.service.ports.metrics }} protocol: TCP - targetPort: {{ .Values.metrics.containerPorts.http }} - {{- if (and (or (eq .Values.metrics.service.type "NodePort") (eq .Values.metrics.service.type "LoadBalancer")) .Values.metrics.service.nodePorts.http) }} - nodePort: {{ coalesce .Values.metrics.service.nodePorts.http .Values.metrics.service.nodePort }} + targetPort: {{ .Values.metrics.containerPorts.metrics }} + {{- if (and (or (eq .Values.metrics.service.type "NodePort") (eq .Values.metrics.service.type "LoadBalancer")) .Values.metrics.service.nodePorts.metrics) }} + nodePort: {{ coalesce .Values.metrics.service.nodePorts.metrics .Values.metrics.service.nodePort }} {{- else if eq .Values.metrics.service.type "ClusterIP" }} nodePort: null {{- end }} diff --git a/charts/mayastor/templates/metrics/ServiceMonitor.yaml b/charts/mayastor/templates/metrics/ServiceMonitor.yaml new file mode 100644 index 00000000..fc2f9ee4 --- /dev/null +++ b/charts/mayastor/templates/metrics/ServiceMonitor.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ printf "%s-exporter" (include "common.names.fullname" .) }} + namespace: {{ default (include "common.names.namespace" .) .Values.metrics.serviceMonitor.namespace | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.labels }} + {{- include "common.tplvalues.render" (dict "value" .Values.metrics.serviceMonitor.labels "context" $) | nindent 4 }} + {{- end }} + app.kubernetes.io/component: metrics + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + {{- if .Values.metrics.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.metrics.serviceMonitor.jobLabel }} + {{- end }} + selector: + matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} + {{- if .Values.metrics.serviceMonitor.selector }} + {{- include "common.tplvalues.render" (dict "value" .Values.metrics.serviceMonitor.selector "context" $) | nindent 6 }} + {{- end }} + app.kubernetes.io/component: metrics + endpoints: + - port: http-metrics + path: /metrics + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.relabelings }} + relabelings: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.serviceMonitor.relabelings "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.serviceMonitor.metricRelabelings "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.honorLabels }} + honorLabels: {{ .Values.metrics.serviceMonitor.honorLabels }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "common.names.namespace" . | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index e2528055..75ad45b0 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -65,11 +65,7 @@ diagnosticMode: ## @section Mayastor common parameters -## Mayastor image -imageDev: - registry: docker.io - repository: mayadata/mayastor - tag: release-1.0.2 +## Mayastor io-engine image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry ## @param image.repository Mayastor Control Plane Rest image repository @@ -80,8 +76,8 @@ imageDev: ## image: registry: docker.io - repository: mayadata/mayastor - tag: release-1.0.2 + repository: mayastor-io-engine + tag: develop ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -345,14 +341,14 @@ service: ## timeoutSeconds: 300 sessionAffinityConfig: {} -csiNode: +csiController: enabled: true logLevel: info imageDev: registry: docker.io - repository: mayadata/mayastor-csi-node + repository: mayadata/mayastor-csi-controller tag: develop - ## Mayastor CSI image + ## Mayastor Control Plane Core Agent image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry ## @param image.repository Mayastor Control Plane Rest image repository @@ -363,7 +359,7 @@ csiNode: ## image: registry: docker.io - repository: mayadata/mayastor-csi + repository: mayadata/mcp-csi-controller tag: release-1.0.2 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -378,30 +374,37 @@ csiNode: ## pullSecrets: [] - driverRegistrar: + attacher: image: - # registry: quay.io - # repository: k8scsi/csi-node-driver-registrar - # tag: v2.1.0 registry: k8s.gcr.io - repository: sig-storage/csi-node-driver-registrar - tag: v2.5.0 + repository: sig-storage/csi-attacher + tag: v3.4.0 - nvme: - # nvme_core module io timeout in seconds - io_timeout: "30" - io_timeout_enabled: true + livenessprobe: + image: + registry: k8s.gcr.io + repository: sig-storage/livenessprobe + tag: v2.6.0 - socketAddress: /csi/csi.sock + provisioner: + image: + registry: k8s.gcr.io + repository: sig-storage/csi-provisioner + tag: v3.1.0 -csiController: +csiDriver: + create: true + podInfoOnMount: true + attachRequired: false + +csiNode: enabled: true logLevel: info imageDev: registry: docker.io - repository: mayadata/mayastor-csi-controller + repository: mayadata/mayastor-csi-node tag: develop - ## Mayastor Control Plane Core Agent image + ## Mayastor CSI image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry ## @param image.repository Mayastor Control Plane Rest image repository @@ -412,7 +415,7 @@ csiController: ## image: registry: docker.io - repository: mayadata/mcp-csi-controller + repository: mayadata/mayastor-csi tag: release-1.0.2 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -427,28 +430,23 @@ csiController: ## pullSecrets: [] - attacher: + driverRegistrar: image: + # registry: quay.io + # repository: k8scsi/csi-node-driver-registrar + # tag: v2.1.0 registry: k8s.gcr.io - repository: sig-storage/csi-attacher - tag: v3.4.0 + repository: sig-storage/csi-node-driver-registrar + tag: v2.5.0 - livenessprobe: - image: - registry: k8s.gcr.io - repository: sig-storage/livenessprobe - tag: v2.6.0 + nvme: + # nvme_core module io timeout in seconds + io_timeout: "30" + io_timeout_enabled: true + + socketAddress: /csi/csi.sock - provisioner: - image: - registry: k8s.gcr.io - repository: sig-storage/csi-provisioner - tag: v3.1.0 -csiDriver: - create: true - podInfoOnMount: true - attachRequired: false etcd: enabled: true @@ -795,6 +793,131 @@ apiRest: ## timeoutSeconds: 300 sessionAffinityConfig: {} +ioEngine: + ## Mayastor I/O Engine image + ## ref: https://hub.docker.com/r/mayadata/mcp-msp-operator/tags + ## @param ioEngine.image.registry Mayastor Control Plane MSP Operator image registry + ## @param ioEngine.image.repository Mayastor Control Plane MSP Operator image repository + ## @param ioEngine.image.tag Mayastor Control Plane MSP Operator image tag (immutable tags are recommended) + ## @param ioEngine.image.pullPolicy Mayastor Control Plane MSP Operator image pull policy + ## @param ioEngine.image.pullSecrets Specify docker-registry secret names as an array + ## @param ioEngine.image.debug Specify if debug logs should be enabled + ## + image: + registry: docker.io + repository: mayadata/mayastor-io-engine + tag: develop + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param ioEngine.replicaCount Number of Mayastor I/O Engine replicas to deploy + ## + replicaCount: 1 + ## @param ioEngine.containerPorts.http Mayastor I/O Engine HTTP container port + ## @param ioEngine.containerPorts.https Mayastor I/O Engine HTTPS container port + ## + containerPorts: + grpc: 10124 + ## Configure extra options for Mayastor I/O Engine containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## @param ioEngine.livenessProbe.enabled Enable livenessProbe on Mayastor I/O Engine containers + ## @param ioEngine.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param ioEngine.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param ioEngine.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param ioEngine.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param ioEngine.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: false + initialDelaySeconds: foo + periodSeconds: bar + timeoutSeconds: foo + failureThreshold: bar + successThreshold: foo + ## @param ioEngine.readinessProbe.enabled Enable readinessProbe on Mayastor I/O Engine containers + ## @param ioEngine.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param ioEngine.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param ioEngine.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param ioEngine.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param ioEngine.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: false + initialDelaySeconds: foo + periodSeconds: bar + timeoutSeconds: foo + failureThreshold: bar + successThreshold: foo + ## @param ioEngine.startupProbe.enabled Enable startupProbe on Mayastor I/O Engine containers + ## @param ioEngine.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe + ## @param ioEngine.startupProbe.periodSeconds Period seconds for startupProbe + ## @param ioEngine.startupProbe.timeoutSeconds Timeout seconds for startupProbe + ## @param ioEngine.startupProbe.failureThreshold Failure threshold for startupProbe + ## @param ioEngine.startupProbe.successThreshold Success threshold for startupProbe + ## + startupProbe: + enabled: false + initialDelaySeconds: foo + periodSeconds: bar + timeoutSeconds: foo + failureThreshold: bar + successThreshold: foo + ## @param ioEngine.customLivenessProbe Custom livenessProbe that overrides the default one + ## + customLivenessProbe: {} + ## @param ioEngine.customReadinessProbe Custom readinessProbe that overrides the default one + ## + customReadinessProbe: {} + ## @param ioEngine.customStartupProbe Custom startupProbe that overrides the default one + ## + customStartupProbe: {} + ## Mayastor I/O Engine resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## @param ioEngine.resources.limits The resources limits for the Mayastor I/O Engine containers + ## @param ioEngine.resources.requests The requested resources for the Mayastor I/O Engine containers + ## + resources: + limits: + cpu: 2 + memory: 1Gi + hugepages-2Mi: 2Gi + requests: + cpu: 2 + memory: 1Gi + hugepages-2Mi: 2Gi + ## Configure Pods Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + ## @param ioEngine.podSecurityContext.enabled Enabled Mayastor I/O Engine pods' Security Context + ## @param ioEngine.podSecurityContext.fsGroup Set Mayastor I/O Engine pod's Security Context fsGroup + ## + podSecurityContext: + enabled: false + fsGroup: 1001 + ## Configure Container Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## @param ioEngine.containerSecurityContext.enabled Enabled Mayastor I/O Engine containers' Security Context + ## @param ioEngine.containerSecurityContext.runAsUser Set Mayastor I/O Engine containers' Security Context runAsUser + ## @param ioEngine.containerSecurityContext.runAsNonRoot Set Mayastor I/O Engine containers' Security Context runAsNonRoot + ## @param ioEngine.containerSecurityContext.readOnlyRootFilesystem Set Mayastor I/O Engine containers' Security Context runAsNonRoot + ## + containerSecurityContext: + enabled: false + runAsUser: 1001 + runAsNonRoot: true + readOnlyRootFilesystem: false + + logLevel: info + cpuCount: "" + operator: enabled: true imageDev: @@ -878,4 +1001,6 @@ rbac: rules: [] metrics: - enabled: false \ No newline at end of file + enabled: false + serviceMonitor: + enabled: false \ No newline at end of file From 27829d1dc2639f5a5a339cc187d59ee06333d8fe Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 10:37:08 +0700 Subject: [PATCH 079/188] Fix ioEngine DS typo --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 6 +++--- charts/mayastor/values.yaml | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 853e96c4..7d2381a5 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -81,7 +81,7 @@ spec: - "-p {{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" - "-R https://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.port.grpc }}" - "-y /var/local/io-engine/config.yaml" - {{- else }} + {{- end }} env: - name: NVME_QPAIR_CONNECT_ASYNC value: "true" @@ -118,9 +118,9 @@ spec: resources: {{ toYaml .Values.resources | nindent 12 }} {{- end }} ports: - - containerPort: 10124 + - name: io-engine + containerPort: {{ .Values.ioEngine.containerPorts.io-engine }} protocol: TCP - name: io-engine {{- if .Values.metrics.enabled }} - name: metrics-exporter-pool image: {{ include "mayastor.apiRest.image" . }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 75ad45b0..d95e6830 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -446,8 +446,6 @@ csiNode: socketAddress: /csi/csi.sock - - etcd: enabled: true ## @param replicaCount Number of etcd replicas to deploy @@ -826,7 +824,7 @@ ioEngine: ## @param ioEngine.containerPorts.https Mayastor I/O Engine HTTPS container port ## containerPorts: - grpc: 10124 + io-engine: 10124 ## Configure extra options for Mayastor I/O Engine containers' liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes ## @param ioEngine.livenessProbe.enabled Enable livenessProbe on Mayastor I/O Engine containers From 444d35d41eefc639344d2b8c1184729e77ea47e0 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 11:16:41 +0700 Subject: [PATCH 080/188] Fix containerPort --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 6 +++--- charts/mayastor/values.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 7d2381a5..af3396db 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -118,9 +118,9 @@ spec: resources: {{ toYaml .Values.resources | nindent 12 }} {{- end }} ports: - - name: io-engine - containerPort: {{ .Values.ioEngine.containerPorts.io-engine }} - protocol: TCP + - name: io-engine + containerPort: {{ .Values.ioEngine.containerPorts.ioEngine }} + protocol: TCP {{- if .Values.metrics.enabled }} - name: metrics-exporter-pool image: {{ include "mayastor.apiRest.image" . }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index d95e6830..6271890c 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -65,7 +65,7 @@ diagnosticMode: ## @section Mayastor common parameters -## Mayastor io-engine image +## Mayastor image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry ## @param image.repository Mayastor Control Plane Rest image repository @@ -824,7 +824,7 @@ ioEngine: ## @param ioEngine.containerPorts.https Mayastor I/O Engine HTTPS container port ## containerPorts: - io-engine: 10124 + ioEngine: 10124 ## Configure extra options for Mayastor I/O Engine containers' liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes ## @param ioEngine.livenessProbe.enabled Enable livenessProbe on Mayastor I/O Engine containers From a3d71472dae009b1ca1278d5fd9b81fd21e4c947 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 11:18:37 +0700 Subject: [PATCH 081/188] FIx ioEngine helper image --- charts/mayastor/templates/_helper.tpl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index 06a9b3e9..e29a0ca2 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -37,11 +37,7 @@ {{- end -}} {{- end -}} -{{/* Return the proper Mayastor I/O Engine image name */}} -{{- define "mayastor.ioEngine.image" -}} - {{ include "common.images.image" (dict "imageRoot" .Values.ioEngine.image "global" .Values.global) }} -{{- end -}} -{{- end -}} + {{/* Return the proper Mayastor Agent Core image name */}} {{- define "mayastor.agentCore.image" -}} @@ -99,6 +95,11 @@ {{- end -}} {{- end -}} +{{/* Return the proper Mayastor I/O Engine image name */}} +{{- define "mayastor.ioEngine.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.ioEngine.image "global" .Values.global) }} +{{- end -}} + {{/* Return the proper image name (for the init container volume-permissions image) */}} {{- define "mayastor.metrics.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.metrics.image "global" .Values.global) }} From a264326a9472bba50fe95bccdb9ca69f7a692a4c Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 11:21:45 +0700 Subject: [PATCH 082/188] Fix ioEngine initContainers ports --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 2 +- charts/mayastor/values.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index af3396db..7389733f 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -46,7 +46,7 @@ spec: - sh args: - -c - - trap "exit 1" TERM; until nc -vzw 5 {{ include "mayastor.agentCore.fullname" . }} {{ .Values.agentCore.service.port.grpc }}; do date; echo "Waiting for agent-core-grpc services..."; sleep 1; done; + - trap "exit 1" TERM; until nc -vzw 5 {{ include "mayastor.agentCore.fullname" . }} {{ .Values.agentCore.service.ports.grpc }}; do date; echo "Waiting for agent-core-grpc services..."; sleep 1; done; - name: etcd-probe image: busybox:latest command: diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 6271890c..7b005002 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -739,7 +739,6 @@ apiRest: ports: http: 8080 https: 8081 - grpc: 50051 ## @param service.nodePorts.mysql Mayastor Control Plane Rest Kubernetes service node port ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport ## From 9a36b55f8a84b8c9e179419af65db7b3894ab8bf Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 11:24:49 +0700 Subject: [PATCH 083/188] Fix ioEngine cpuFlag --- charts/mayastor/templates/_helper.tpl | 2 +- charts/mayastor/templates/io-engine/DaemonSet.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index e29a0ca2..1efe2e7f 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -128,7 +128,7 @@ {{- define "mayastor.cpuFlag" -}} {{- range $i, $e := until (int .Values.ioEngine.cpuCount) }} {{- if gt $i 0 }} - {{- printf "," }} + {{- printf "," }} {{- end }} {{- printf "%d" (add $i 1) }} {{- end }} diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 7389733f..d15aa072 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -77,7 +77,7 @@ spec: # Note: Ensure that the CPU resources are updated accordingly. # If you use 2 CPUs, the CPU: field should also read 2. - "-g $(MY_POD_IP)" - - "-l {{ include "mayastor.ioEngine.cpuFlag" . }}" + - "-l {{ include "mayastor.cpuFlag" . }}" - "-p {{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" - "-R https://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.port.grpc }}" - "-y /var/local/io-engine/config.yaml" From bb05196b20f991c8c5f73e013337324f29cda2c4 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 11:33:34 +0700 Subject: [PATCH 084/188] Add ioEngine securityContext --- .../templates/io-engine/DaemonSet.yaml | 29 ++++++++++--------- charts/mayastor/values.yaml | 6 ++-- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index d15aa072..2cb42984 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -79,14 +79,10 @@ spec: - "-g $(MY_POD_IP)" - "-l {{ include "mayastor.cpuFlag" . }}" - "-p {{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" - - "-R https://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.port.grpc }}" + - "-R https://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.ports.grpc }}" - "-y /var/local/io-engine/config.yaml" {{- end }} env: - - name: NVME_QPAIR_CONNECT_ASYNC - value: "true" - - name: NVMF_TCP_MAX_QUEUE_DEPTH - value: "32" - name: MY_NODE_NAME valueFrom: fieldRef: @@ -95,6 +91,10 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: NVME_QPAIR_CONNECT_ASYNC + value: "true" + - name: NVMF_TCP_MAX_QUEUE_DEPTH + value: "32" - name: RUST_BACKTRACE {{- if .Values.diagnosticMode.enabled }} value: full @@ -103,8 +103,16 @@ spec: {{- end }} - name: RUST_LOG value: {{ .Values.ioEngine.logLevel }} - securityContext: - privileged: true + ports: + - name: io-engine + containerPort: {{ .Values.ioEngine.containerPorts.ioEngine }} + protocol: TCP + {{- if .Values.ioEngine.resources }} + resources: {{ toYaml .Values.ioEngine.resources | nindent 12 }} + {{- end }} + {{- if .Values.ioEngine.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.ioEngine.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: device mountPath: /dev @@ -114,13 +122,6 @@ spec: mountPath: /dev/shm - name: configlocation mountPath: /var/local/io-engine/ - {{- if .Values.resources }} - resources: {{ toYaml .Values.resources | nindent 12 }} - {{- end }} - ports: - - name: io-engine - containerPort: {{ .Values.ioEngine.containerPorts.ioEngine }} - protocol: TCP {{- if .Values.metrics.enabled }} - name: metrics-exporter-pool image: {{ include "mayastor.apiRest.image" . }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 7b005002..ea4c29a6 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -907,10 +907,8 @@ ioEngine: ## @param ioEngine.containerSecurityContext.readOnlyRootFilesystem Set Mayastor I/O Engine containers' Security Context runAsNonRoot ## containerSecurityContext: - enabled: false - runAsUser: 1001 - runAsNonRoot: true - readOnlyRootFilesystem: false + enabled: true + privileged: true logLevel: info cpuCount: "" From be1bb5aaa66002a8096ec3fee24bafc10aefd089 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 11:41:05 +0700 Subject: [PATCH 085/188] Add affinity --- .../mayastor/templates/io-engine/DaemonSet.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 2cb42984..95349190 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -33,12 +33,27 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" ( dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} dnsPolicy: ClusterFirstWithHostNet hostNetwork: true {{- include "mayastor.imagePullSecrets" . | nindent 6 }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.nodeSelector "context" $) | nindent 8 }} {{- end }} + serviceAccountName: {{ include "mayastor.serviceAccountName" . }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} + {{- end }} initContainers: - name: agent-core-probe image: busybox:latest From 35bf44e6b4e6bbbaadd10f57563434a7624ae8f1 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 11:44:56 +0700 Subject: [PATCH 086/188] Fix ioEngine container args --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 95349190..d2a0d0a3 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -92,7 +92,9 @@ spec: # Note: Ensure that the CPU resources are updated accordingly. # If you use 2 CPUs, the CPU: field should also read 2. - "-g $(MY_POD_IP)" - - "-l {{ include "mayastor.cpuFlag" . }}" + {{- if .Values.ioEngine.cpuCount }} + - "-l {{ include "mayastor.cpuFlag" . | quote }}" + {{- end }} - "-p {{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" - "-R https://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.ports.grpc }}" - "-y /var/local/io-engine/config.yaml" From e96ba787352e5c779a5e64e372d4dfd7386bbdab Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 12:10:26 +0700 Subject: [PATCH 087/188] Add affinity, simplify image helper --- charts/mayastor/templates/_helper.tpl | 26 ------- .../templates/io-engine/DaemonSet.yaml | 22 +++--- charts/mayastor/values-test.yaml | 19 +++--- charts/mayastor/values.yaml | 67 ++++++++++--------- 4 files changed, 54 insertions(+), 80 deletions(-) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index 1efe2e7f..91e08248 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -30,32 +30,18 @@ {{/* Return the proper Mayastor image name */}} {{- define "mayastor.image" -}} -{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} - {{ include "common.images.image" (dict "imageRoot" .Values.imageDev "global" .Values.global) }} -{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }} {{- end -}} -{{- end -}} - - {{/* Return the proper Mayastor Agent Core image name */}} {{- define "mayastor.agentCore.image" -}} -{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} - {{ include "common.images.image" (dict "imageRoot" .Values.agentCore.imageDev "global" .Values.global) }} -{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.agentCore.image "global" .Values.global) }} {{- end -}} -{{- end -}} {{/* Return the proper Mayastor CSI Controller image name */}} {{- define "mayastor.csiController.image" -}} -{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} - {{ include "common.images.image" (dict "imageRoot" .Values.csiController.imageDev "global" .Values.global) }} -{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} {{- end -}} -{{- end -}} {{/* Return the proper Mayastor CSI Controller Attacher image name */}} {{- define "mayastor.csiController.attacher.image" -}} @@ -74,12 +60,8 @@ {{/* Return the proper Mayastor CSI Node image name */}} {{- define "mayastor.csiNode.image" -}} -{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} - {{ include "common.images.image" (dict "imageRoot" .Values.csiNode.imageDev "global" .Values.global) }} -{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.csiNode.image "global" .Values.global) }} {{- end -}} -{{- end -}} {{/* Return the proper Mayastor CSI Node image name */}} {{- define "mayastor.csiNode.driverRegistrar.image" -}} @@ -88,12 +70,8 @@ {{/* Return the proper Mayastor Rest image name */}} {{- define "mayastor.apiRest.image" -}} -{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} - {{ include "common.images.image" (dict "imageRoot" .Values.apiRest.imageDev "global" .Values.global) }} -{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.apiRest.image "global" .Values.global) }} {{- end -}} -{{- end -}} {{/* Return the proper Mayastor I/O Engine image name */}} {{- define "mayastor.ioEngine.image" -}} @@ -107,12 +85,8 @@ {{/* Return the proper Mayastor Operator image name */}} {{- define "mayastor.operator.image" -}} -{{- if or (eq .Values.diagnosticMode.environment "development") (eq .Values.diagnosticMode.environment "dev") -}} - {{ include "common.images.image" (dict "imageRoot" .Values.operator.imageDev "global" .Values.global) }} -{{- else -}} {{ include "common.images.image" (dict "imageRoot" .Values.operator.image "global" .Values.global) }} {{- end -}} -{{- end -}} {{/* Return the proper image name (for the init container volume-permissions image) */}} {{- define "mayastor.volumePermissions.image" -}} diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index d2a0d0a3..5cad43a5 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -33,8 +33,8 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: - {{- if .Values.affinity }} - affinity: {{- include "common.tplvalues.render" ( dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- if .Values.ioEngine.affinity }} + affinity: {{- include "common.tplvalues.render" ( dict "value" .Values.ioEngine.affinity "context" $) | nindent 8 }} {{- else }} affinity: podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "context" $) | nindent 10 }} @@ -44,31 +44,31 @@ spec: dnsPolicy: ClusterFirstWithHostNet hostNetwork: true {{- include "mayastor.imagePullSecrets" . | nindent 6 }} - {{- if .Values.nodeSelector }} + {{- if .Values.ioEngine.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.nodeSelector "context" $) | nindent 8 }} {{- end }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} - {{- if .Values.tolerations }} - tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} + {{- if .Values.ioEngine.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.tolerations "context" .) | nindent 8 }} {{- end }} - {{- if .Values.topologySpreadConstraints }} - topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} + {{- if .Values.ioEngine.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.topologySpreadConstraints "context" .) | nindent 8 }} {{- end }} initContainers: - - name: agent-core-probe + - name: etcd-probe image: busybox:latest command: - sh args: - -c - - trap "exit 1" TERM; until nc -vzw 5 {{ include "mayastor.agentCore.fullname" . }} {{ .Values.agentCore.service.ports.grpc }}; do date; echo "Waiting for agent-core-grpc services..."; sleep 1; done; - - name: etcd-probe + - trap "exit 1" TERM; until nc -vzw 5 {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.port }}; do date; echo "Waiting for etcd..."; sleep 1; done; + - name: agent-core-probe image: busybox:latest command: - sh args: - -c - - trap "exit 1" TERM; until nc -vzw 5 {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.port }}; do date; echo "Waiting for etcd..."; sleep 1; done; + - trap "exit 1" TERM; until nc -vzw 5 {{ include "mayastor.agentCore.fullname" . }} {{ .Values.agentCore.service.ports.grpc }}; do date; echo "Waiting for agent-core-grpc services..."; sleep 1; done; containers: - name: io-engine image: {{ include "mayastor.ioEngine.image" . }} diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index c008ca33..ee4506d0 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -25,16 +25,6 @@ etcd: enabled: true storageClass: openebs-hostpath -nats: - clusterDomain: nebula.mci.net.id - replicaCount: 1 - revisionHistoryLimit: 2 - auth: - enabled: false - persistence: - enabled: false - storageClass: openebs-hostpath - diskPools: - node: alberta-lin-k8m.jkt2.mci.net.id device: /dev/sdb @@ -45,5 +35,14 @@ diskPools: - node: verus-lin-k8w.jkt2.mci.net.id device: /dev/sdb +ioEngine: + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + podAnnotations: v1.multus-cni.io/default-network: kube-public/calico-ipv4-only \ No newline at end of file diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index ea4c29a6..0af2b1c6 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -59,9 +59,6 @@ diagnosticMode: ## args: - infinity - ## @param diagnosticMode.environment (enter: development) - ## - environment: "" ## @section Mayastor common parameters @@ -344,10 +341,6 @@ service: csiController: enabled: true logLevel: info - imageDev: - registry: docker.io - repository: mayadata/mayastor-csi-controller - tag: develop ## Mayastor Control Plane Core Agent image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry @@ -359,8 +352,8 @@ csiController: ## image: registry: docker.io - repository: mayadata/mcp-csi-controller - tag: release-1.0.2 + repository: mayadata/mayastor-csi-controller + tag: develop ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -400,10 +393,6 @@ csiDriver: csiNode: enabled: true logLevel: info - imageDev: - registry: docker.io - repository: mayadata/mayastor-csi-node - tag: develop ## Mayastor CSI image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry @@ -415,8 +404,8 @@ csiNode: ## image: registry: docker.io - repository: mayadata/mayastor-csi - tag: release-1.0.2 + repository: mayadata/mayastor-csi-node + tag: develop ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -598,10 +587,6 @@ nats: agentCore: enabled: true logLevel: info - imageDev: - registry: docker.io - repository: mayadata/mayastor-agent-core - tag: develop ## Mayastor Control Plane Core Agent image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry @@ -613,8 +598,8 @@ agentCore: ## image: registry: docker.io - repository: mayadata/mcp-core - tag: release-1.0.2 + repository: mayadata/mayastor-agent-core + tag: develop ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -691,10 +676,6 @@ agentCore: apiRest: enabled: true - imageDev: - registry: docker.io - repository: mayadata/mayastor-api-rest - tag: develop ## Mayastor Control Plane Rest image ## ref: https://hub.docker.com/r/mayadata/mcp-rest/tags ## @param image.registry Mayastor Control Plane Rest image registry @@ -706,8 +687,8 @@ apiRest: ## image: registry: docker.io - repository: mayadata/mcp-rest - tag: release-1.0.2 + repository: mayadata/mayastor-api-rest + tag: develop ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -909,16 +890,36 @@ ioEngine: containerSecurityContext: enabled: true privileged: true + ## @param affinity Affinity for Mayastor I/O Engine pods assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set + ## + affinity: {} + ## @param nodeSelector Node labels for Mayastor I/O Engine pods assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: + openebs.io/engine: mayastor + kubernetes.io/arch: amd64 + ## @param tolerations Tolerations for Mayastor I/O Engine pods assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @param topologySpreadConstraints Topology Spread Constraints for Mayastor I/O Engine pods assignment + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## E.g. + ## topologySpreadConstraints: + ## - maxSkew: 1 + ## topologyKey: topology.kubernetes.io/zone + ## whenUnsatisfiable: DoNotSchedule + ## + topologySpreadConstraints: {} logLevel: info cpuCount: "" operator: enabled: true - imageDev: - registry: docker.io - repository: mayadata/mayastor-operator-diskpool - tag: develop ## Mayastor Control Plane MSP Operator image ## ref: https://hub.docker.com/r/mayadata/mcp-msp-operator/tags ## @param image.registry Mayastor Control Plane MSP Operator image registry @@ -930,8 +931,8 @@ operator: ## image: registry: docker.io - repository: mayadata/mcp-msp-operator - tag: release-1.0.2 + repository: mayadata/mayastor-operator-diskpool + tag: develop ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images From 305e56d28264c6a93435e72f5db176aee52cd7b2 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 12:29:34 +0700 Subject: [PATCH 088/188] Upgrade etcd dependencies to v8.x.x --- charts/mayastor/Chart.yaml | 3 ++- charts/mayastor/templates/_helper.tpl | 10 +++++----- charts/mayastor/templates/api-rest/Deployment.yaml | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/charts/mayastor/Chart.yaml b/charts/mayastor/Chart.yaml index 473f45f4..04d782f8 100644 --- a/charts/mayastor/Chart.yaml +++ b/charts/mayastor/Chart.yaml @@ -13,7 +13,8 @@ dependencies: repository: https://charts.bitnami.com/bitnami tags: - bitnami-etcd - version: 6.x.x + # version: 6.x.x + version: 8.x.x - name: nats condition: nats.enabled repository: https://charts.bitnami.com/bitnami diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index 91e08248..f8f21672 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -38,6 +38,11 @@ {{ include "common.images.image" (dict "imageRoot" .Values.agentCore.image "global" .Values.global) }} {{- end -}} +{{/* Return the proper Mayastor Rest image name */}} +{{- define "mayastor.apiRest.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.apiRest.image "global" .Values.global) }} +{{- end -}} + {{/* Return the proper Mayastor CSI Controller image name */}} {{- define "mayastor.csiController.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.csiController.image "global" .Values.global) }} @@ -68,11 +73,6 @@ {{ include "common.images.image" (dict "imageRoot" .Values.csiNode.driverRegistrar.image "global" .Values.global) }} {{- end -}} -{{/* Return the proper Mayastor Rest image name */}} -{{- define "mayastor.apiRest.image" -}} - {{ include "common.images.image" (dict "imageRoot" .Values.apiRest.image "global" .Values.global) }} -{{- end -}} - {{/* Return the proper Mayastor I/O Engine image name */}} {{- define "mayastor.ioEngine.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.ioEngine.image "global" .Values.global) }} diff --git a/charts/mayastor/templates/api-rest/Deployment.yaml b/charts/mayastor/templates/api-rest/Deployment.yaml index 1d46d807..1492ac16 100644 --- a/charts/mayastor/templates/api-rest/Deployment.yaml +++ b/charts/mayastor/templates/api-rest/Deployment.yaml @@ -48,7 +48,7 @@ spec: image: busybox:latest name: etcd-probe containers: - - name: rest + - name: api-rest resources: limits: cpu: 100m From 76285eef533ba5c98ad9f4d4e01f8e314f448427 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 12:34:19 +0700 Subject: [PATCH 089/188] Fix agentCore --- charts/mayastor/Chart.yaml | 6 ------ charts/mayastor/templates/agent-core/Deployment.yaml | 2 +- charts/mayastor/values.yaml | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/charts/mayastor/Chart.yaml b/charts/mayastor/Chart.yaml index 04d782f8..f8f49e16 100644 --- a/charts/mayastor/Chart.yaml +++ b/charts/mayastor/Chart.yaml @@ -15,12 +15,6 @@ dependencies: - bitnami-etcd # version: 6.x.x version: 8.x.x - - name: nats - condition: nats.enabled - repository: https://charts.bitnami.com/bitnami - tags: - - bitnami-nats - version: 7.x.x description: Mayastor is a cloud-native declarative data plane written in Rust. Our goal is to abstract storage resources and their differences through the data plane such that users only need to supply the what and do not have to worry about diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index b5e022f5..60da0844 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -88,7 +88,7 @@ spec: {{- if .Values.diagnosticMode.enabled }} value: full {{- else }} - value: "1" + value: full {{- end }} imagePullPolicy: {{ .Values.agentCore.image.pullPolicy | quote }} ports: diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 0af2b1c6..f2370821 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -586,7 +586,7 @@ nats: agentCore: enabled: true - logLevel: info + logLevel: debug ## Mayastor Control Plane Core Agent image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags ## @param image.registry Mayastor Control Plane Rest image registry From c0f72dfc838066b92de0b6230d86d3331d70f839 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 12:38:51 +0700 Subject: [PATCH 090/188] Fix etcd --- charts/mayastor/values-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index ee4506d0..b4bf8034 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -19,6 +19,7 @@ etcd: auth: rbac: create: true + enabled: true allowNoneAuthentication: true rootPassword: password persistence: From 2291ebef389a2772995361c558832562e2db8192 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 12:43:30 +0700 Subject: [PATCH 091/188] Change etcd auth --- charts/mayastor/values-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index b4bf8034..25439892 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -18,8 +18,8 @@ etcd: revisionHistoryLimit: 2 auth: rbac: - create: true - enabled: true + create: false + enabled: false allowNoneAuthentication: true rootPassword: password persistence: From 3107d3863a0dfd2a4e31893e62b368c904ada7a4 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 13:02:58 +0700 Subject: [PATCH 092/188] Fix ioEngine args --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 5cad43a5..492f71b6 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -91,13 +91,13 @@ spec: # For example -l 1,2,10-20 means use core 1, 2, 10 to 20. # Note: Ensure that the CPU resources are updated accordingly. # If you use 2 CPUs, the CPU: field should also read 2. - - "-g $(MY_POD_IP)" + - "-g$(MY_POD_IP)" {{- if .Values.ioEngine.cpuCount }} - - "-l {{ include "mayastor.cpuFlag" . | quote }}" + - "-l{{ include "mayastor.cpuFlag" . | quote }}" {{- end }} - - "-p {{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" - - "-R https://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.ports.grpc }}" - - "-y /var/local/io-engine/config.yaml" + - "-p{{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" + - "-Rhttps://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.ports.grpc }}" + - "-y/var/local/io-engine/config.yaml" {{- end }} env: - name: MY_NODE_NAME From 494edf41e8e8307cc303a46735f23b30f1d59e77 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 13:23:03 +0700 Subject: [PATCH 093/188] test: logLevel debug --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 10 ++++++++-- charts/mayastor/values-test.yaml | 4 +--- charts/mayastor/values.yaml | 5 +++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 492f71b6..2c8372ff 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -59,9 +59,15 @@ spec: image: busybox:latest command: - sh - args: - -c - - trap "exit 1" TERM; until nc -vzw 5 {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.port }}; do date; echo "Waiting for etcd..."; sleep 1; done; + - | + trap "exit 1" TERM + until nc -vzw 5 {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.port }} + do + date + echo "Waiting for etcd..." + sleep 1 + done - name: agent-core-probe image: busybox:latest command: diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 25439892..445a93c3 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -37,6 +37,7 @@ diskPools: device: /dev/sdb ioEngine: + logLevel: debug tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane @@ -44,6 +45,3 @@ ioEngine: - effect: NoSchedule key: node-role.kubernetes.io/master operator: Exists - -podAnnotations: - v1.multus-cni.io/default-network: kube-public/calico-ipv4-only \ No newline at end of file diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index f2370821..6d436c88 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -772,6 +772,8 @@ apiRest: sessionAffinityConfig: {} ioEngine: + logLevel: info + cpuCount: "" ## Mayastor I/O Engine image ## ref: https://hub.docker.com/r/mayadata/mcp-msp-operator/tags ## @param ioEngine.image.registry Mayastor Control Plane MSP Operator image registry @@ -915,8 +917,7 @@ ioEngine: ## topologySpreadConstraints: {} - logLevel: info - cpuCount: "" + operator: enabled: true From 8dea1e107d4b2ec86f323e443f9f97582d67b56f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 13:31:46 +0700 Subject: [PATCH 094/188] Fix diskpool --- charts/mayastor/values-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 445a93c3..bd01b4cc 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -32,7 +32,7 @@ diskPools: - node: einbech-lin-k8m.jkt2.mci.net.id device: /dev/sdb - node: rachel-lin-k8w.jkt2.mci.net.id - device: /dev/sdb + device: /dev/sda - node: verus-lin-k8w.jkt2.mci.net.id device: /dev/sdb From 66cb0f28d060cf6ce208e3f911cf54e18df02caf Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Thu, 16 Jun 2022 13:36:45 +0700 Subject: [PATCH 095/188] Remove crds --- charts/mayastor/{ => archive}/crds/MayastorPool.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename charts/mayastor/{ => archive}/crds/MayastorPool.yaml (100%) diff --git a/charts/mayastor/crds/MayastorPool.yaml b/charts/mayastor/archive/crds/MayastorPool.yaml similarity index 100% rename from charts/mayastor/crds/MayastorPool.yaml rename to charts/mayastor/archive/crds/MayastorPool.yaml From e0f976e3c544ff669aea34df4c2a74a66d289442 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Fri, 17 Jun 2022 20:29:45 +0700 Subject: [PATCH 096/188] Change ioEngine debug --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 2c8372ff..be8f1c01 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -122,7 +122,7 @@ spec: {{- if .Values.diagnosticMode.enabled }} value: full {{- else }} - value: "1" + value: full {{- end }} - name: RUST_LOG value: {{ .Values.ioEngine.logLevel }} From 470ba84c30818f94e2d82c802d518919c82a3e6f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Fri, 17 Jun 2022 21:16:51 +0700 Subject: [PATCH 097/188] Add test diskPool --- charts/mayastor/values-test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index bd01b4cc..7a8ba719 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -2,7 +2,6 @@ clusterDomain: nebula.mci.net.id diagnosticMode: enabled: false - environment: development tolerations: - effect: NoSchedule @@ -31,6 +30,8 @@ diskPools: device: /dev/sdb - node: einbech-lin-k8m.jkt2.mci.net.id device: /dev/sdb + - node: lutie-lin-k8m.jkt2.mci.net.id + device: /dev/sdb - node: rachel-lin-k8w.jkt2.mci.net.id device: /dev/sda - node: verus-lin-k8w.jkt2.mci.net.id From 0d304287a7a3e09effc04b7a97d67d1343baeaf3 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Fri, 17 Jun 2022 21:26:53 +0700 Subject: [PATCH 098/188] test: diskPool --- charts/mayastor/values-test.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 7a8ba719..a3ed1649 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -28,14 +28,14 @@ etcd: diskPools: - node: alberta-lin-k8m.jkt2.mci.net.id device: /dev/sdb - - node: einbech-lin-k8m.jkt2.mci.net.id - device: /dev/sdb + # - node: einbech-lin-k8m.jkt2.mci.net.id + # device: /dev/sdb - node: lutie-lin-k8m.jkt2.mci.net.id device: /dev/sdb - node: rachel-lin-k8w.jkt2.mci.net.id device: /dev/sda - - node: verus-lin-k8w.jkt2.mci.net.id - device: /dev/sdb + # - node: verus-lin-k8w.jkt2.mci.net.id + # device: /dev/sdb ioEngine: logLevel: debug From 450ad565b48310eed29e1f6f32d83fa5ef0db76d Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 09:55:18 +0700 Subject: [PATCH 099/188] csi-controller add node selector --- .../{templates => archive}/CSIDriver.yaml | 0 .../templates/csi-controller/Deployment.yaml | 34 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) rename charts/mayastor/{templates => archive}/CSIDriver.yaml (100%) diff --git a/charts/mayastor/templates/CSIDriver.yaml b/charts/mayastor/archive/CSIDriver.yaml similarity index 100% rename from charts/mayastor/templates/CSIDriver.yaml rename to charts/mayastor/archive/CSIDriver.yaml diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 0a97f061..5148af84 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -35,6 +35,9 @@ spec: dnsPolicy: ClusterFirstWithHostNet hostNetwork: true {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + {{- if .Values.ioEngine.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.nodeSelector "context" $) | nindent 8 }} + {{- end }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- if .Values.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} @@ -51,10 +54,9 @@ spec: - -c - | trap "exit 1" TERM - until nc -vz {{ include "mayastor.apiRest.fullname" . }} 8081 + until nc -vz {{ include "mayastor.apiRest.fullname" . }} {{ .Values.apiRest.service.ports.https }} do - date - echo "Waiting for REST API endpoint to become available" + echo "$(date) Waiting for REST API endpoint to become available" sleep 1 done containers: @@ -86,26 +88,30 @@ spec: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-controller - resources: - limits: - cpu: 32m - memory: 128Mi - requests: - cpu: 16m - memory: 64Mi image: {{ include "mayastor.csiController.image" . }} imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} args: - "--csi-socket=/var/lib/csi/sockets/pluginproxy/csi.sock" - - "--rest-endpoint=http://{{ include "mayastor.apiRest.fullname" . }}:8081" - {{- range $key, $val := .Values.nodeSelector }} + - "--rest-endpoint=http://{{ include "mayastor.apiRest.fullname" . }}:{{ .Values.apiRest.service.ports.https }}" + {{- range $key, $val := .Values.ioEngine.nodeSelector }} - "--io-engine-selector={{ $key }}:{{ $val }}" {{- end }} env: - - name: RUST_LOG - value: {{ .Values.csiController.logLevel }} - name: RUST_BACKTRACE + {{- if .Values.diagnosticMode.enabled }} value: full + {{- else }} + value: full + {{- end }} + - name: RUST_LOG + value: {{ .Values.csiController.logLevel }} + resources: + limits: + cpu: 32m + memory: 128Mi + requests: + cpu: 16m + memory: 64Mi volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ From 635de1a0d38cdf120c13d1ac44af5132cbd23beb Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 11:31:38 +0700 Subject: [PATCH 100/188] Change ioEngine logLevel --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 4 ++++ charts/mayastor/values-test.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index be8f1c01..735c6e27 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -125,7 +125,11 @@ spec: value: full {{- end }} - name: RUST_LOG + {{- if .Values.diagnosticMode.enabled }} + value: debug + {{- else }} value: {{ .Values.ioEngine.logLevel }} + {{- end }} ports: - name: io-engine containerPort: {{ .Values.ioEngine.containerPorts.ioEngine }} diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index a3ed1649..96e94dad 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -38,7 +38,7 @@ diskPools: # device: /dev/sdb ioEngine: - logLevel: debug + logLevel: info tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane From 1585c2163ac5e7056b8db3268c650e5527866086 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 13:57:30 +0700 Subject: [PATCH 101/188] Add metrics components --- .../templates/csi-controller/Deployment.yaml | 2 +- .../templates/csi-node/DaemonSet.yaml | 2 +- .../templates/io-engine/DaemonSet.yaml | 28 +- .../templates/metrics/PrometheusRule.yaml | 26 ++ charts/mayastor/values.yaml | 296 ++++++++++++++++-- 5 files changed, 310 insertions(+), 44 deletions(-) create mode 100644 charts/mayastor/templates/metrics/PrometheusRule.yaml diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 5148af84..ed0a0d8d 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -54,7 +54,7 @@ spec: - -c - | trap "exit 1" TERM - until nc -vz {{ include "mayastor.apiRest.fullname" . }} {{ .Values.apiRest.service.ports.https }} + until nc -vzw 5 {{ include "mayastor.apiRest.fullname" . }} {{ .Values.apiRest.service.ports.https }} do echo "$(date) Waiting for REST API endpoint to become available" sleep 1 diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 357cedc8..0c2733ff 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -51,7 +51,7 @@ spec: # pressure unless they exceed those limits. limits and requests must be # the same. containers: - - name: mayastor-csi-plugin + - name: csi-plugin {{- if .Values.diagnosticMode.enabled }} command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} {{- else if .Values.command }} diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 735c6e27..6e7048c9 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -62,7 +62,7 @@ spec: - -c - | trap "exit 1" TERM - until nc -vzw 5 {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.port }} + until nc -vzw 5 {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.ports.client }} do date echo "Waiting for etcd..." @@ -74,7 +74,14 @@ spec: - sh args: - -c - - trap "exit 1" TERM; until nc -vzw 5 {{ include "mayastor.agentCore.fullname" . }} {{ .Values.agentCore.service.ports.grpc }}; do date; echo "Waiting for agent-core-grpc services..."; sleep 1; done; + - |- + trap "exit 1" TERM + until nc -vzw 5 {{ include "mayastor.agentCore.fullname" . }} {{ .Values.agentCore.service.ports.grpc }} + do + date + echo "Waiting for agent-core-grpc services..." + sleep 1 + done containers: - name: io-engine image: {{ include "mayastor.ioEngine.image" . }} @@ -101,7 +108,7 @@ spec: {{- if .Values.ioEngine.cpuCount }} - "-l{{ include "mayastor.cpuFlag" . | quote }}" {{- end }} - - "-p{{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" + - "-p{{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.ports.client }}" - "-Rhttps://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.ports.grpc }}" - "-y/var/local/io-engine/config.yaml" {{- end }} @@ -151,8 +158,10 @@ spec: mountPath: /var/local/io-engine/ {{- if .Values.metrics.enabled }} - name: metrics-exporter-pool - image: {{ include "mayastor.apiRest.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + image: {{ .Values.metrics.image }} + imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }} + command: + - metrics-exporter-pool env: - name: MY_NODE_NAME valueFrom: @@ -162,12 +171,13 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - command: - - metrics-exporter-pool ports: - - name: http-metrics - containerPort: 9502 + - name: metrics + containerPort: {{ .Values.metrics.containerPorts.metrics }} protocol: TCP + {{- if .Values.metrics.resources }} + resources: {{ toYaml .Values.metrics.resources | nindent 12 }} + {{- end }} {{- end }} volumes: - name: device diff --git a/charts/mayastor/templates/metrics/PrometheusRule.yaml b/charts/mayastor/templates/metrics/PrometheusRule.yaml new file mode 100644 index 00000000..085ad4b2 --- /dev/null +++ b/charts/mayastor/templates/metrics/PrometheusRule.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.metrics.enabled .Values.metrics.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "common.names.fullname" . }} + {{- if .Values.metrics.prometheusRule.namespace }} + namespace: {{ .Values.metrics.prometheusRule.namespace }} + {{- else }} + namespace: {{ include "common.names.namespace" . | quote }} + {{- end }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: metrics + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.metrics.prometheusRule.additionalLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + groups: + - name: {{ include "common.names.fullname" . }} + rules: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.prometheusRule.rules "context" $ ) | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 6d436c88..ea0d8edb 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -393,14 +393,14 @@ csiDriver: csiNode: enabled: true logLevel: info - ## Mayastor CSI image - ## ref: https://hub.docker.com/r/mayadata/mayastor/tags - ## @param image.registry Mayastor Control Plane Rest image registry - ## @param image.repository Mayastor Control Plane Rest image repository - ## @param image.tag Mayastor Control Plane Rest image tag (immutable tags are recommended) - ## @param image.pullPolicy Mayastor Control Plane Rest image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - ## @param image.debug Specify if debug logs should be enabled + ## Mayastor CSI Node image + ## ref: https://hub.docker.com/r/mayadata/mayastor-csi-node/tags + ## @param csiNode.image.registry Mayastor CSI Node image registry + ## @param csiNode.image.repository Mayastor CSI Node image repository + ## @param csiNode.image.tag Mayastor CSI Node image tag (immutable tags are recommended) + ## @param csiNode.image.pullPolicy Mayastor CSI Node image pull policy + ## @param csiNode.image.pullSecrets Specify docker-registry secret names as an array + ## @param csiNode.image.debug Specify if debug logs should be enabled ## image: registry: docker.io @@ -427,6 +427,18 @@ csiNode: registry: k8s.gcr.io repository: sig-storage/csi-node-driver-registrar tag: v2.5.0 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] nvme: # nvme_core module io timeout in seconds @@ -435,22 +447,15 @@ csiNode: socketAddress: /csi/csi.sock +## @section etcd parameters +## etcd: - enabled: true - ## @param replicaCount Number of etcd replicas to deploy + ## @param etcd.enabled Enable etcd service ## - replicaCount: 1 - ## Kubernetes Cluster Domain + enabled: true + ## @param etcd.clusterDomain Default Kubernetes cluster domain ## clusterDomain: cluster.local - podLabels: - app.kubernetes.io/component: etcd - client: - secureTransport: false - peer: - secureTransport: false - removeMemberOnContainerTermination: false - initialClusterState: "new" ## Authentication parameters ## auth: @@ -458,27 +463,27 @@ etcd: ## ref: https://etcd.io/docs/current/op-guide/authentication/ ## rbac: - ## @param auth.rbac.create Switch to enable RBAC authentication + ## @param etcd.auth.rbac.create Switch to enable RBAC authentication ## #create: true enabled: false - ## @param auth.rbac.allowNoneAuthentication Allow to use etcd without configuring RBAC authentication + ## @param etcd.auth.rbac.allowNoneAuthentication Allow to use etcd without configuring RBAC authentication ## allowNoneAuthentication: true - ## @param auth.rbac.rootPassword Root user password. The root user is always `root` + ## @param etcd.auth.rbac.rootPassword Root user password. The root user is always `root` ## rootPassword: "" - ## @param auth.rbac.existingSecret Name of the existing secret containing credentials for the root user + ## @param etcd.auth.rbac.existingSecret Name of the existing secret containing credentials for the root user ## existingSecret: "" - ## @param auth.rbac.existingSecretPasswordKey Name of key containing password to be retrieved from the existing secret + ## @param etcd.auth.rbac.existingSecretPasswordKey Name of key containing password to be retrieved from the existing secret ## existingSecretPasswordKey: "" ## Authentication token ## ref: https://etcd.io/docs/latest/learning/design-auth-v3/#two-types-of-tokens-simple-and-jwt ## token: - ## @param auth.token.type Authentication token type. Allowed values: 'simple' or 'jwt' + ## @param etcd.auth.token.type Authentication token type. Allowed values: 'simple' or 'jwt' ## ref: https://etcd.io/docs/latest/op-guide/configuration/#--auth-token ## type: simple @@ -492,7 +497,26 @@ etcd: ## peer: secureTransport: false - ## Enable persistence using Persistent Volume Claims + ## @param etcd.initialClusterState Initial cluster state. Allowed values: 'new' or 'existing' + ## If this values is not set, the default values below are set: + ## - 'new': when installing the chart ('helm install ...') + ## - 'existing': when upgrading the chart ('helm upgrade ...') + ## + initialClusterState: "" + ## @param etcd.removeMemberOnContainerTermination Use a PreStop hook to remove the etcd members from the etcd cluster on container termination + ## they the containers are terminated + ## NOTE: Ignored if lifecycleHooks is set or replicaCount=1 + ## + removeMemberOnContainerTermination: false + ## @param etcd.replicaCount Number of etcd replicas to deploy + ## + replicaCount: 1 + ## @param etcd.podLabels [object] Extra labels for etcd pods + ## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + ## + podLabels: + app.kubernetes.io/component: etcd + ## Enable etcd.persistence using Persistent Volume Claims ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ ## persistence: @@ -511,13 +535,13 @@ etcd: ## size: 2Gi serviceAccount: - ## @param serviceAccount.create Enable/disable service account creation + ## @param etcd.serviceAccount.create Enable/disable service account creation ## - create: false + create: true ## @param serviceAccount.name Name of the service account to create or use ## # name: {{ include "mayastor.serviceAccountName" $ }} - name: mayastor + name: "" ## Init containers parameters: ## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. ## @@ -531,10 +555,17 @@ etcd: ## Allowed values: soft, hard ## podAntiAffinityPreset: "hard" - # etcd service parameters defines how the etcd service is exposed + + ## @section Traffic exposure parameters + ## service: - # etcd client port port: 2379 + ## @param etcd.service.ports.client etcd client port + ## @param etcd.service.ports.peer etcd peer port + ## + ports: + client: 2379 + peer: 2380 nats: enabled: false @@ -997,7 +1028,206 @@ rbac: ## rules: [] + +## @section Metrics parameters +## + metrics: + ## @param metrics.enabled Start a side-car prometheus exporter + ## enabled: false + ## @param metrics.image.registry Exporter image registry + ## @param metrics.image.repository Exporter image repository + ## @param metrics.image.tag Exporter image tag (immutable tags are recommended) + ## @param metrics.image.pullPolicy Exporter image pull policy + ## @param metrics.image.pullSecrets Specify docker-registry secret names as an array + ## + image: + registry: docker.io + repository: mayadata/mayastor-metrics-exporter-pool + tag: develop + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param metrics.annotations [object] Annotations for the Exporter pod + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9502" + ## Mayastor metrics container ports to open + ## @param containerPorts.metrics Exporter port to expose at container level + ## + containerPorts: + metrics: 9502 + ## Mayastor metrics container Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## @param metrics.containerSecurityContext.enabled Enable security context for Mayastor metrics container + ## Example: + ## containerSecurityContext: + ## enabled: true + ## capabilities: + ## drop: ["NET_RAW"] + ## readOnlyRootFilesystem: true + ## + containerSecurityContext: + enabled: false + ## Mayastor Prometheus exporter resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param metrics.resources.limits The resources limits for MariaDB prometheus exporter containers + ## @param metrics.resources.requests The requested resources for MariaDB prometheus exporter containers + ## + resources: + ## Example: + ## limits: + ## cpu: 100m + ## memory: 256Mi + ## + limits: {} + ## Examples: + ## requests: + ## cpu: 100m + ## memory: 256Mi + ## + requests: {} + ## Configure extra options for liveness probe + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## @param metrics.livenessProbe.enabled Enable livenessProbe + ## @param metrics.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param metrics.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param metrics.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param metrics.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param metrics.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: true + initialDelaySeconds: 120 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + ## Configure extra options for readiness probe + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## @param metrics.readinessProbe.enabled Enable readinessProbe + ## @param metrics.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param metrics.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param metrics.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param metrics.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param metrics.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + ## Prometheus Service Monitor + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + podMonitor: + ## @param metrics.podMonitor.enabled Create PodMonitor Resource for scraping metrics using PrometheusOperator + ## + enabled: false + ## @param metrics.podMonitor.namespace Namespace in which Prometheus is running + ## + namespace: monitoring + ## @param metrics.podMonitor.interval Specify the interval at which metrics should be scraped + ## + interval: 30s + ## @param metrics.podMonitor.scrapeTimeout Specify the timeout after which the scrape is ended + ## + scrapeTimeout: 30s + ## @param metrics.podMonitor.additionalLabels [object] Additional labels that can be used so PodMonitors will be discovered by Prometheus + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#prometheusspec + ## + additionalLabels: {} + ## @param metrics.podMonitor.scheme Scheme to use for scraping + ## + scheme: http + ## @param metrics.podMonitor.tlsConfig [object] TLS configuration used for scrape endpoints used by Prometheus + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#tlsconfig + ## e.g: + ## tlsConfig: + ## ca: + ## secret: + ## name: existingSecretName + ## + tlsConfig: {} + ## @param metrics.podMonitor.relabelings [array] Prometheus relabeling rules + ## + relabelings: [] + ## Prometheus Service Monitor + ## ref: https://github.com/coreos/prometheus-operator + ## serviceMonitor: - enabled: false \ No newline at end of file + ## @param metrics.serviceMonitor.enabled Create ServiceMonitor Resource for scraping metrics using PrometheusOperator + ## + enabled: false + ## @param metrics.serviceMonitor.namespace Namespace which Prometheus is running in + ## + namespace: "" + ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in prometheus. + ## + jobLabel: "" + ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped + ## + interval: 30s + ## @param metrics.serviceMonitor.scrapeTimeout Specify the timeout after which the scrape is ended + ## e.g: + ## scrapeTimeout: 30s + ## + scrapeTimeout: "" + ## @param metrics.serviceMonitor.relabelings RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## + relabelings: [] + ## @param metrics.serviceMonitor.metricRelabelings MetricRelabelConfigs to apply to samples before ingestion + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + ## @param metrics.serviceMonitor.honorLabels honorLabels chooses the metric's labels on collisions with target labels + ## + honorLabels: false + ## @param metrics.serviceMonitor.selector ServiceMonitor selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + ## selector: + ## prometheus: my-prometheus + ## + selector: {} + ## @param metrics.serviceMonitor.labels Extra labels for the ServiceMonitor + ## + labels: {} + ## Prometheus Operator PrometheusRule configuration + ## + prometheusRule: + ## @param metrics.prometheusRule.enabled Create a Prometheus Operator PrometheusRule (also requires `metrics.enabled` to be `true` and `metrics.prometheusRule.rules`) + ## + enabled: false + ## @param metrics.prometheusRule.namespace Namespace for the PrometheusRule Resource (defaults to the Release Namespace) + ## + namespace: "" + ## @param metrics.prometheusRule.additionalLabels Additional labels that can be used so PrometheusRule will be discovered by Prometheus + ## + additionalLabels: {} + ## @param metrics.prometheusRule.rules Prometheus Rule definitions + # - alert: ETCD has no leader + # annotations: + # summary: "ETCD has no leader" + # description: "pod {{`{{`}} $labels.pod {{`}}`}} state error, can't connect leader" + # for: 1m + # expr: etcd_server_has_leader == 0 + # labels: + # severity: critical + # group: PaaS + ## + rules: [] \ No newline at end of file From 938835f5fb53bbcf89126177004bba382a2c912e Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 14:29:11 +0700 Subject: [PATCH 102/188] Add resources --- .../templates/csi-controller/Deployment.yaml | 13 ++- .../templates/csi-node/DaemonSet.yaml | 6 +- .../templates/io-engine/DaemonSet.yaml | 28 ++++-- charts/mayastor/values.yaml | 93 +++++++++++++++++-- 4 files changed, 115 insertions(+), 25 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index ed0a0d8d..af7f3d2a 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -84,6 +84,9 @@ spec: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} + {{- if .Values.csiController.attacher.resources }} + resources: {{ toYaml .Values.csiController.attacher.resources | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -105,13 +108,9 @@ spec: {{- end }} - name: RUST_LOG value: {{ .Values.csiController.logLevel }} - resources: - limits: - cpu: 32m - memory: 128Mi - requests: - cpu: 16m - memory: 64Mi + {{- if .Values.csiController.resources }} + resources: {{ toYaml .Values.csiController.resources | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 0c2733ff..46413359 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -138,9 +138,9 @@ spec: command: - /bin/sh - -c - - >- - echo "Deleting stale file"; - rm -rf /registration/io.openebs.mayastor /registration/io.openebs.mayastor-reg.sock; + - |- + echo "Deleting stale file" + rm -rf /registration/io.openebs.mayastor /registration/io.openebs.mayastor-reg.sock # Mayastor node plugin gRPC server ports: - containerPort: 10199 diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 6e7048c9..1def8397 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -163,18 +163,30 @@ spec: command: - metrics-exporter-pool env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP ports: - name: metrics containerPort: {{ .Values.metrics.containerPorts.metrics }} protocol: TCP + {{- if .Values.metrics.livenessProbe.enabled }} + livenessProbe: {{- omit .Values.metrics.livenessProbe "enabled" | toYaml | nindent 12 }} + httpGet: + path: /metrics + port: metrics + {{- end }} + {{- if .Values.metrics.readinessProbe.enabled }} + readinessProbe: {{- omit .Values.metrics.readinessProbe "enabled" | toYaml | nindent 12 }} + httpGet: + path: /metrics + port: metrics + {{- end }} {{- if .Values.metrics.resources }} resources: {{ toYaml .Values.metrics.resources | nindent 12 }} {{- end }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index ea0d8edb..21b65677 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -341,12 +341,12 @@ service: csiController: enabled: true logLevel: info - ## Mayastor Control Plane Core Agent image + ## Mayastor CSI Controller image ## ref: https://hub.docker.com/r/mayadata/mayastor/tags - ## @param image.registry Mayastor Control Plane Rest image registry - ## @param image.repository Mayastor Control Plane Rest image repository - ## @param image.tag Mayastor Control Plane Rest image tag (immutable tags are recommended) - ## @param image.pullPolicy Mayastor Control Plane Rest image pull policy + ## @param image.registry Mayastor CSI Controller image registry + ## @param image.repository Mayastor CSI Controller image repository + ## @param image.tag Mayastor CSI Controller image tag (immutable tags are recommended) + ## @param image.pullPolicy Mayastor CSI Controller image pull policy ## @param image.pullSecrets Specify docker-registry secret names as an array ## @param image.debug Specify if debug logs should be enabled ## @@ -366,24 +366,102 @@ csiController: ## - myRegistryKeySecretName ## pullSecrets: [] + ## CSI controller resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## @param controller.resources.limits The resources limits for CSI controller containers + ## @param controller.resources.requests The requested resources for CSI controller containers + ## + resources: + limits: + cpu: 32m + memory: 128Mi + requests: + cpu: 16m + memory: 64Mi + ## @section Kubernetes CSI Driver Parameters + ## ref: https://kubernetes-csi.github.io/docs/introduction.html + + ## CSI external-attacher + ## ref: https://github.com/kubernetes-csi/external-attacher + ## attacher: image: registry: k8s.gcr.io repository: sig-storage/csi-attacher tag: v3.4.0 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## CSI attacher resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## @param attacher.resources.limits The resources limits for CSI attacher containers + ## @param attacher.resources.requests The requested resources for CSI attacher containers + ## + resources: + ## Example: + ## limits: + ## cpu: 100m + ## memory: 256Mi + ## + limits: {} + ## Examples: + ## requests: + ## cpu: 100m + ## memory: 256Mi + ## + requests: {} + ## CSI livenessprobe + ## ref: https://github.com/kubernetes-csi/livenessprobe + ## livenessprobe: image: registry: k8s.gcr.io repository: sig-storage/livenessprobe tag: v2.6.0 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## CSI external-provisioner + ## ref: https://github.com/kubernetes-csi/external-provisioner + ## provisioner: image: registry: k8s.gcr.io repository: sig-storage/csi-provisioner tag: v3.1.0 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] csiDriver: create: true @@ -419,6 +497,9 @@ csiNode: ## pullSecrets: [] + ## CSI node-driver-registrar + ## ref: https://github.com/kubernetes-csi/node-driver-registrar + ## driverRegistrar: image: # registry: quay.io @@ -948,8 +1029,6 @@ ioEngine: ## topologySpreadConstraints: {} - - operator: enabled: true ## Mayastor Control Plane MSP Operator image From 6014494f24a6d98f5dac96a84c39559f1ff6fbbb Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 14:52:06 +0700 Subject: [PATCH 103/188] Add external-provisioner env --- charts/mayastor/templates/ClusterRole.yaml | 9 --------- .../templates/csi-controller/Deployment.yaml | 20 +++++++++++++++++-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/charts/mayastor/templates/ClusterRole.yaml b/charts/mayastor/templates/ClusterRole.yaml index 0d866aff..230ea579 100644 --- a/charts/mayastor/templates/ClusterRole.yaml +++ b/charts/mayastor/templates/ClusterRole.yaml @@ -11,15 +11,6 @@ rules: resources: ["customresourcedefinitions"] verbs: ["create", "list"] - # must read mayastor pools info -- apiGroups: ["openebs.io"] - resources: ["mayastorpools"] - verbs: ["get", "list", "watch", "update", "replace", "patch"] - # must update mayastor pools status -- apiGroups: ["openebs.io"] - resources: ["mayastorpools/status"] - verbs: ["update", "patch"] - # must read diskpool info - apiGroups: ["openebs.io"] resources: ["diskpools"] diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index af7f3d2a..1b60d63a 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -63,14 +63,22 @@ spec: - name: csi-provisioner image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0 args: - - "--v=2" - "--csi-address=$(ADDRESS)" - "--feature-gates=Topology=true" - "--strict-topology=false" - "--default-fstype=ext4" + - "--v=2" env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} volumeMounts: - name: socket-dir @@ -78,11 +86,19 @@ spec: - name: csi-attacher image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 args: - - "--v=2" - "--csi-address=$(ADDRESS)" + - "--v=2" env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} {{- if .Values.csiController.attacher.resources }} resources: {{ toYaml .Values.csiController.attacher.resources | nindent 12 }} From 36a97b1d4448dda7fd4c8a6a02f91f511c83e33c Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 14:56:30 +0700 Subject: [PATCH 104/188] Fix csi-external env --- charts/mayastor/templates/csi-controller/Deployment.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 1b60d63a..58765203 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -73,11 +73,11 @@ spec: value: /var/lib/csi/sockets/pluginproxy/csi.sock - name: NAMESPACE valueFrom: - fieldRef: + fieldRef: fieldPath: metadata.namespace - name: POD_NAME valueFrom: - fieldRef: + fieldRef: fieldPath: metadata.name imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} volumeMounts: @@ -93,11 +93,11 @@ spec: value: /var/lib/csi/sockets/pluginproxy/csi.sock - name: NAMESPACE valueFrom: - fieldRef: + fieldRef: fieldPath: metadata.namespace - name: POD_NAME valueFrom: - fieldRef: + fieldRef: fieldPath: metadata.name imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} {{- if .Values.csiController.attacher.resources }} From 84b3b9d0c982eba74ad32bffc142bab5812a3d4d Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 14:57:28 +0700 Subject: [PATCH 105/188] Fix csi-external env --- charts/mayastor/templates/csi-controller/Deployment.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 58765203..70e4c613 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -91,14 +91,6 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} {{- if .Values.csiController.attacher.resources }} resources: {{ toYaml .Values.csiController.attacher.resources | nindent 12 }} From afab0a86051b8ca24b907f7d5167bcbfab667c16 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 15:26:51 +0700 Subject: [PATCH 106/188] Add csi-provioner metrics --- charts/mayastor/templates/csi-controller/Deployment.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 70e4c613..6c3746aa 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -68,6 +68,9 @@ spec: - "--strict-topology=false" - "--default-fstype=ext4" - "--v=2" + {{- if .Values.csiController.attacher.metrics.enabled }} + - "--http-endpoint=http://0.0.0.0:{{ .Values.csiController.attacher.containerPorts.metrics }}" + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -80,6 +83,10 @@ spec: fieldRef: fieldPath: metadata.name imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} + ports: + - name: csi-provisioner + containerPort: {{ .Values.csiController.attacher.containerPorts.metrics }} + protocol: TCP volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ From 052f439ec17b6c0357aa41a0fed35534a9bbc7f8 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 15:30:51 +0700 Subject: [PATCH 107/188] Add values for csi-provisioner metrics --- charts/mayastor/values.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 21b65677..4c7e9ae0 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -402,6 +402,10 @@ csiController: ## - myRegistryKeySecretName ## pullSecrets: [] + ## @param agentCore.containerPorts.http Mayastor Agent Core GRPC container port + ## + containerPorts: + metrics: 9601 ## CSI attacher resource requests and limits ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ ## @param attacher.resources.limits The resources limits for CSI attacher containers @@ -420,6 +424,8 @@ csiController: ## memory: 256Mi ## requests: {} + metrics: + enabled: true ## CSI livenessprobe ## ref: https://github.com/kubernetes-csi/livenessprobe @@ -463,11 +469,6 @@ csiController: ## pullSecrets: [] -csiDriver: - create: true - podInfoOnMount: true - attachRequired: false - csiNode: enabled: true logLevel: info From 99a77fa7c3dfcf4a8dc55808972d29e4ad924ccd Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 15:33:20 +0700 Subject: [PATCH 108/188] csi-provisioner: add node name --- charts/mayastor/templates/csi-controller/Deployment.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 6c3746aa..d556b515 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -82,6 +82,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} ports: - name: csi-provisioner From 345d18252a0d682431e60811022fd1ba4eaabfd0 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 15:37:26 +0700 Subject: [PATCH 109/188] Disable metrics --- charts/mayastor/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 4c7e9ae0..e4c758e8 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -425,7 +425,7 @@ csiController: ## requests: {} metrics: - enabled: true + enabled: false ## CSI livenessprobe ## ref: https://github.com/kubernetes-csi/livenessprobe From 7652fed87a4f1c4f57397f95429bc07413dc67e0 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 15:40:05 +0700 Subject: [PATCH 110/188] Fix http-endpoint --- charts/mayastor/templates/csi-controller/Deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index d556b515..10085b31 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -69,7 +69,7 @@ spec: - "--default-fstype=ext4" - "--v=2" {{- if .Values.csiController.attacher.metrics.enabled }} - - "--http-endpoint=http://0.0.0.0:{{ .Values.csiController.attacher.containerPorts.metrics }}" + - "--http-endpoint=0.0.0.0:{{ .Values.csiController.attacher.containerPorts.metrics }}" {{- end }} env: - name: ADDRESS From e09e8bba6e2018e63580c3627d373d459e927feb Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 15:41:01 +0700 Subject: [PATCH 111/188] Fix http endpoint --- charts/mayastor/templates/csi-controller/Deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 10085b31..49df525e 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -67,9 +67,9 @@ spec: - "--feature-gates=Topology=true" - "--strict-topology=false" - "--default-fstype=ext4" - - "--v=2" + - "--v=5" {{- if .Values.csiController.attacher.metrics.enabled }} - - "--http-endpoint=0.0.0.0:{{ .Values.csiController.attacher.containerPorts.metrics }}" + - "--http-endpoint=:{{ .Values.csiController.attacher.containerPorts.metrics }}" {{- end }} env: - name: ADDRESS From c6066c5510379ee8034ca9c158c14d2199222b50 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 15:50:47 +0700 Subject: [PATCH 112/188] Add csiNode tolerations --- .../templates/csi-node/DaemonSet.yaml | 12 +++--- charts/mayastor/values-test.yaml | 9 +++++ charts/mayastor/values.yaml | 40 +++++++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 46413359..8f18469c 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -35,16 +35,16 @@ spec: {{- end }} spec: hostNetwork: true - {{- if .Values.nodeSelector }} - nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- if .Values.csiNode.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.csiNode.nodeSelector "context" $) | nindent 8 }} {{- end }} {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} - {{- if .Values.tolerations }} - tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} + {{- if .Values.csiNode.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.csiNode.tolerations "context" .) | nindent 8 }} {{- end }} - {{- if .Values.topologySpreadConstraints }} - topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} + {{- if .Values.csiNode.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.csiNode.topologySpreadConstraints "context" .) | nindent 8 }} {{- end }} # NOTE: Each container must have mem/cpu limits defined in order to # belong to Guaranteed QoS class, hence can never get evicted in case of diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 96e94dad..357ca794 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -11,6 +11,15 @@ tolerations: key: node-role.kubernetes.io/master operator: Exists +csiNode: + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + etcd: clusterDomain: nebula.mci.net.id replicaCount: 1 diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index e4c758e8..feb4b290 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -497,6 +497,46 @@ csiNode: ## - myRegistryKeySecretName ## pullSecrets: [] + ## Configure Pods Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + ## @param ioEngine.podSecurityContext.enabled Enabled Mayastor I/O Engine pods' Security Context + ## @param ioEngine.podSecurityContext.fsGroup Set Mayastor I/O Engine pod's Security Context fsGroup + ## + podSecurityContext: + enabled: false + fsGroup: 1001 + ## Configure Container Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## @param ioEngine.containerSecurityContext.enabled Enabled Mayastor I/O Engine containers' Security Context + ## @param ioEngine.containerSecurityContext.runAsUser Set Mayastor I/O Engine containers' Security Context runAsUser + ## @param ioEngine.containerSecurityContext.runAsNonRoot Set Mayastor I/O Engine containers' Security Context runAsNonRoot + ## @param ioEngine.containerSecurityContext.readOnlyRootFilesystem Set Mayastor I/O Engine containers' Security Context runAsNonRoot + ## + containerSecurityContext: + enabled: false + privileged: true + ## @param affinity Affinity for Mayastor I/O Engine pods assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set + ## + affinity: {} + ## @param nodeSelector Node labels for Mayastor I/O Engine pods assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## @param tolerations Tolerations for Mayastor I/O Engine pods assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @param topologySpreadConstraints Topology Spread Constraints for Mayastor I/O Engine pods assignment + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## E.g. + ## topologySpreadConstraints: + ## - maxSkew: 1 + ## topologyKey: topology.kubernetes.io/zone + ## whenUnsatisfiable: DoNotSchedule + ## + topologySpreadConstraints: {} ## CSI node-driver-registrar ## ref: https://github.com/kubernetes-csi/node-driver-registrar From 92402db4954edeb0f3798a7ba57d8c3390534660 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 20:47:07 +0700 Subject: [PATCH 113/188] Add image helper --- charts/mayastor/templates/DiskPool.yaml | 6 +- charts/mayastor/templates/StorageClass.yaml | 8 +- .../templates/csi-controller/Deployment.yaml | 18 +- .../templates/csi-node/DaemonSet.yaml | 210 +++++++++--------- charts/mayastor/values-test.yaml | 22 +- charts/mayastor/values.yaml | 48 ++-- 6 files changed, 168 insertions(+), 144 deletions(-) diff --git a/charts/mayastor/templates/DiskPool.yaml b/charts/mayastor/templates/DiskPool.yaml index 91844c92..b4222c5f 100644 --- a/charts/mayastor/templates/DiskPool.yaml +++ b/charts/mayastor/templates/DiskPool.yaml @@ -1,6 +1,7 @@ -##ref: https://github.com/openebs/mayastor/blob/master/deploy/pool.yaml +## ref: https://github.com/openebs/mayastor/blob/master/deploy/pool.yaml -{{- range .Values.diskPools }} +{{- if .Values.diskPools.create }} +{{- range .Values.diskPools.pools }} apiVersion: "openebs.io/v1alpha1" kind: DiskPool metadata: @@ -10,4 +11,5 @@ spec: node: {{ .node }} disks: ["{{ .device }}"] --- +{{- end }} {{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/StorageClass.yaml b/charts/mayastor/templates/StorageClass.yaml index 0e8d33cf..378e7353 100644 --- a/charts/mayastor/templates/StorageClass.yaml +++ b/charts/mayastor/templates/StorageClass.yaml @@ -6,9 +6,9 @@ parameters: repl: '1' protocol: 'nvmf' ioTimeout: '60' - local: 'true' + local: 'false' # It is recommended to use xfs for Mayastor - # fsType: 'xfs' + fsType: 'xfs' provisioner: io.openebs.csi-mayastor volumeBindingMode: WaitForFirstConsumer --- @@ -20,9 +20,9 @@ parameters: repl: '3' protocol: 'nvmf' ioTimeout: '60' - local: 'true' + local: 'false' # It is recommended to use xfs for Mayastor - # fsType: 'xfs' + fsType: 'xfs' provisioner: io.openebs.csi-mayastor volumeBindingMode: WaitForFirstConsumer --- \ No newline at end of file diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 49df525e..ec0098f1 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -1,4 +1,4 @@ -##ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/csi-deployment.yaml +## ref: https://raw.githubusercontent.com/openebs/mayastor-control-plane/master/deploy/csi-deployment.yaml {{- if .Values.csiController.enabled }} apiVersion: apps/v1 @@ -61,7 +61,8 @@ spec: done containers: - name: csi-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0 + image: {{ include "mayastor.csiController.provisioner.image" . }} + imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} args: - "--csi-address=$(ADDRESS)" - "--feature-gates=Topology=true" @@ -86,7 +87,6 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} ports: - name: csi-provisioner containerPort: {{ .Values.csiController.attacher.containerPorts.metrics }} @@ -95,14 +95,22 @@ spec: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 + image: {{ include "mayastor.csiController.attacher.image" . }} + imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} args: - "--csi-address=$(ADDRESS)" - "--v=2" env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} + - name: RUST_BACKTRACE + {{- if .Values.diagnosticMode.enabled }} + value: full + {{- else }} + value: full + {{- end }} + - name: RUST_LOG + value: {{ .Values.csiController.logLevel }} {{- if .Values.csiController.attacher.resources }} resources: {{ toYaml .Values.csiController.attacher.resources | nindent 12 }} {{- end }} diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 8f18469c..a80ab352 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -4,7 +4,6 @@ metadata: name: {{ printf "%s-csi-node" (include "common.names.fullname" .) }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} - openebs.io/engine: io-engine app.kubernetes.io/component: csi-node {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} @@ -51,113 +50,114 @@ spec: # pressure unless they exceed those limits. limits and requests must be # the same. containers: - - name: csi-plugin - {{- if .Values.diagnosticMode.enabled }} - command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} - {{- else if .Values.command }} - command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} - {{- else }} - command: - - csi-node - {{- end }} - {{- if .Values.diagnosticMode.enabled }} - args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} - {{- else if .Values.args }} - args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} - {{- else }} - args: - - "--csi-socket=$(CSI_SOCKET_ADDRESS)" - - "--node-name=$(MY_NODE_NAME)" - - "--grpc-endpoint=$(MY_POD_IP):10199" - {{- if .Values.csiNode.nvme.io_timeout_enabled }} - - "--nvme-core-io-timeout={{ .Values.csiNode.nvme.io_timeout }}" + - name: csi-plugin + image: {{ include "mayastor.csiNode.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} + {{- else if .Values.command }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }} + {{- else }} + command: + - csi-node {{- end }} - - "--nvme-nr-io-queues=2" - - "-v" - {{- end }} - image: {{ include "mayastor.csiNode.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy | quote }} - securityContext: - privileged: true - env: - - name: CSI_SOCKET_ADDRESS - value: {{ .Values.csiNode.socketAddress | quote }} - - name: MY_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: RUST_BACKTRACE - {{- if .Values.diagnosticMode.enabled }} - value: full - {{- else }} - value: "1" + {{- if .Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if .Values.args }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} + {{- else }} + args: + - "--csi-socket=$(CSI_SOCKET_ADDRESS)" + - "--node-name=$(MY_NODE_NAME)" + - "--grpc-endpoint=$(MY_POD_IP):10199" + {{- if .Values.csiNode.nvme.io_timeout_enabled }} + - "--nvme-core-io-timeout={{ .Values.csiNode.nvme.io_timeout }}" {{- end }} - - name: RUST_LOG - value: {{ .Values.csiNode.logLevel }} - {{- if .Values.moac }} - - name: MOAC - value: "true" + - "--nvme-nr-io-queues=2" + - "-v" + {{- end }} + {{- if .Values.csiNode.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.csiNode.containerSecurityContext "enabled" | toYaml | nindent 12 }} {{- end }} - resources: - limits: - cpu: "100m" - memory: "50Mi" - requests: - cpu: "100m" - memory: "50Mi" - volumeMounts: - - name: device - mountPath: /dev - - name: sys - mountPath: /sys - - name: run-udev - mountPath: /run/udev - - name: plugin-dir - mountPath: /csi - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" - - name: csi-driver-registrar - image: {{ include "mayastor.csiNode.driverRegistrar.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy | quote }} - args: - - "--csi-address=$(CSI_SOCKET_ADDRESS)" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/io.openebs.mayastor/csi.sock" - - "--v=5" - env: - - name: CSI_SOCKET_ADDRESS - value: {{ .Values.csiNode.socketAddress | quote }} - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - |- - echo "Deleting stale file" - rm -rf /registration/io.openebs.mayastor /registration/io.openebs.mayastor-reg.sock - # Mayastor node plugin gRPC server - ports: - - containerPort: 10199 - name: mayastor-node - protocol: TCP - resources: - limits: - cpu: "100m" - memory: "50Mi" - requests: - cpu: "100m" - memory: "50Mi" - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration + env: + - name: CSI_SOCKET_ADDRESS + value: {{ .Values.csiNode.socketAddress | quote }} + - name: MY_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: RUST_BACKTRACE + {{- if .Values.diagnosticMode.enabled }} + value: full + {{- else }} + value: "1" + {{- end }} + - name: RUST_LOG + {{- if .Values.diagnosticMode.enabled }} + value: debug + {{- else }} + value: {{ .Values.csiNode.logLevel }} + {{- end }} + resources: + limits: + cpu: "100m" + memory: "50Mi" + requests: + cpu: "100m" + memory: "50Mi" + volumeMounts: + - name: device + mountPath: /dev + - name: sys + mountPath: /sys + - name: run-udev + mountPath: /run/udev + - name: plugin-dir + mountPath: /csi + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: csi-driver-registrar + image: {{ include "mayastor.csiNode.driverRegistrar.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + args: + - "--csi-address=$(CSI_SOCKET_ADDRESS)" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/io.openebs.mayastor/csi.sock" + - "--v=5" + env: + - name: CSI_SOCKET_ADDRESS + value: {{ .Values.csiNode.socketAddress | quote }} + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - |- + echo "Deleting stale file" + rm -rf /registration/io.openebs.mayastor /registration/io.openebs.mayastor-reg.sock + # Mayastor node plugin gRPC server + ports: + - containerPort: 10199 + name: csi-driver + protocol: TCP + resources: + limits: + cpu: "100m" + memory: "50Mi" + requests: + cpu: "100m" + memory: "50Mi" + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration volumes: - name: device hostPath: diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 357ca794..201d6ba9 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -35,16 +35,18 @@ etcd: storageClass: openebs-hostpath diskPools: - - node: alberta-lin-k8m.jkt2.mci.net.id - device: /dev/sdb - # - node: einbech-lin-k8m.jkt2.mci.net.id - # device: /dev/sdb - - node: lutie-lin-k8m.jkt2.mci.net.id - device: /dev/sdb - - node: rachel-lin-k8w.jkt2.mci.net.id - device: /dev/sda - # - node: verus-lin-k8w.jkt2.mci.net.id - # device: /dev/sdb + create: true + pools: + - node: alberta-lin-k8m.jkt2.mci.net.id + device: /dev/sdb + # - node: einbech-lin-k8m.jkt2.mci.net.id + # device: /dev/sdb + - node: lutie-lin-k8m.jkt2.mci.net.id + device: /dev/sdb + - node: rachel-lin-k8w.jkt2.mci.net.id + device: /dev/sda + # - node: verus-lin-k8w.jkt2.mci.net.id + # device: /dev/sdb ioEngine: logLevel: info diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index feb4b290..f2a25c82 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -499,22 +499,34 @@ csiNode: pullSecrets: [] ## Configure Pods Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod - ## @param ioEngine.podSecurityContext.enabled Enabled Mayastor I/O Engine pods' Security Context - ## @param ioEngine.podSecurityContext.fsGroup Set Mayastor I/O Engine pod's Security Context fsGroup + ## @param csiNode.podSecurityContext.enabled Enabled Mayastor I/O Engine pods' Security Context + ## @param csiNode.podSecurityContext.fsGroup Set Mayastor I/O Engine pod's Security Context fsGroup ## podSecurityContext: enabled: false fsGroup: 1001 ## Configure Container Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container - ## @param ioEngine.containerSecurityContext.enabled Enabled Mayastor I/O Engine containers' Security Context - ## @param ioEngine.containerSecurityContext.runAsUser Set Mayastor I/O Engine containers' Security Context runAsUser - ## @param ioEngine.containerSecurityContext.runAsNonRoot Set Mayastor I/O Engine containers' Security Context runAsNonRoot - ## @param ioEngine.containerSecurityContext.readOnlyRootFilesystem Set Mayastor I/O Engine containers' Security Context runAsNonRoot + ## @param csiNode.containerSecurityContext.enabled Enabled Mayastore CSI Plugin containers' Security Context + ## @param csiNode.containerSecurityContext.runAsUser Set Mayastore CSI Plugin containers' Security Context runAsUser + ## @param csiNode.containerSecurityContext.runAsNonRoot Set Mayastore CSI Plugin containers' Security Context runAsNonRoot + ## @param csiNode.containerSecurityContext.readOnlyRootFilesystem Set Mayastore CSI Plugin containers' Security Context runAsNonRoot ## containerSecurityContext: - enabled: false + enabled: true privileged: true + ## Mayastor CSI Plugin container's resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## @param primary.resources.limits The resources limits for MariaDB primary containers + ## @param primary.resources.requests The requested resources for MariaDB primary containers + ## + resources: + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi ## @param affinity Affinity for Mayastor I/O Engine pods assignment ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set @@ -1098,17 +1110,17 @@ operator: ## pullSecrets: [] -mayastorCpuCount: "1" -mayastorHugePagesGiB: "1" -mayastorPools: [] -# - node: "NODE_NAME" -# device: "DEVICE" -# This option is intended for development yamls and motivated by the problem of -# moac that does not update status of msp resource in some cases. Feel free to -# remove when no longer needed. -moacDebug: false -moac: false - +## @param diskPools.create +## +diskPools: + create: false + ## @param diskPools.pools Custom RBAC rules + ## Example: + ## pools: + ## - node: node-1 + ## device: /dev/sdb + pools: [] + ## @section RBAC parameter # From abb77a0a2d5344c4d7c4e690cf1d4779476df660 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sat, 18 Jun 2022 21:42:56 +0700 Subject: [PATCH 114/188] Add fstype StorageClass --- charts/mayastor/templates/StorageClass.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/StorageClass.yaml b/charts/mayastor/templates/StorageClass.yaml index 378e7353..4a77036c 100644 --- a/charts/mayastor/templates/StorageClass.yaml +++ b/charts/mayastor/templates/StorageClass.yaml @@ -8,7 +8,7 @@ parameters: ioTimeout: '60' local: 'false' # It is recommended to use xfs for Mayastor - fsType: 'xfs' + csi.storage.k8s.io/fstype: 'xfs' provisioner: io.openebs.csi-mayastor volumeBindingMode: WaitForFirstConsumer --- @@ -22,7 +22,7 @@ parameters: ioTimeout: '60' local: 'false' # It is recommended to use xfs for Mayastor - fsType: 'xfs' + csi.storage.k8s.io/fstype: 'xfs' provisioner: io.openebs.csi-mayastor volumeBindingMode: WaitForFirstConsumer --- \ No newline at end of file From bdeb011731383b1b904b88fafc61761774c2c272 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 11:57:30 +0700 Subject: [PATCH 115/188] Add HostAlias --- charts/mayastor/templates/agent-core/Deployment.yaml | 3 +++ charts/mayastor/templates/api-rest/Deployment.yaml | 3 +++ charts/mayastor/templates/csi-controller/Deployment.yaml | 3 +++ charts/mayastor/templates/csi-node/DaemonSet.yaml | 3 +++ charts/mayastor/templates/io-engine/DaemonSet.yaml | 3 +++ charts/mayastor/templates/operator/Deployment.yaml | 3 +++ charts/mayastor/values-test.yaml | 2 -- 7 files changed, 18 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index 60da0844..aadd5535 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -32,6 +32,9 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: + {{- if .Values.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.hostAliases "context" $) | nindent 8 }} + {{- end }} {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- if .Values.tolerations }} diff --git a/charts/mayastor/templates/api-rest/Deployment.yaml b/charts/mayastor/templates/api-rest/Deployment.yaml index 1492ac16..2ad3167a 100644 --- a/charts/mayastor/templates/api-rest/Deployment.yaml +++ b/charts/mayastor/templates/api-rest/Deployment.yaml @@ -32,6 +32,9 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: + {{- if .Values.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.hostAliases "context" $) | nindent 8 }} + {{- end }} {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- if .Values.tolerations }} diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index ec0098f1..b2ce24e2 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -33,6 +33,9 @@ spec: {{- end }} spec: dnsPolicy: ClusterFirstWithHostNet + {{- if .Values.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.hostAliases "context" $) | nindent 8 }} + {{- end }} hostNetwork: true {{- include "mayastor.imagePullSecrets" . | nindent 6 }} {{- if .Values.ioEngine.nodeSelector }} diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index a80ab352..8f568b97 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -33,6 +33,9 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: + {{- if .Values.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.hostAliases "context" $) | nindent 8 }} + {{- end }} hostNetwork: true {{- if .Values.csiNode.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.csiNode.nodeSelector "context" $) | nindent 8 }} diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 1def8397..4176f23e 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -42,6 +42,9 @@ spec: nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} {{- end }} dnsPolicy: ClusterFirstWithHostNet + {{- if .Values.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.hostAliases "context" $) | nindent 8 }} + {{- end }} hostNetwork: true {{- include "mayastor.imagePullSecrets" . | nindent 6 }} {{- if .Values.ioEngine.nodeSelector }} diff --git a/charts/mayastor/templates/operator/Deployment.yaml b/charts/mayastor/templates/operator/Deployment.yaml index 23d318ef..b14d4b79 100644 --- a/charts/mayastor/templates/operator/Deployment.yaml +++ b/charts/mayastor/templates/operator/Deployment.yaml @@ -32,6 +32,9 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: + {{- if .Values.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.hostAliases "context" $) | nindent 8 }} + {{- end }} {{- include "mayastor.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- if .Values.tolerations }} diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 201d6ba9..5ccbec5e 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -26,8 +26,6 @@ etcd: revisionHistoryLimit: 2 auth: rbac: - create: false - enabled: false allowNoneAuthentication: true rootPassword: password persistence: From 835967b06c9713e0c73a62e5699950882b344fb5 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 12:12:53 +0700 Subject: [PATCH 116/188] Add PriorityClass manifest --- charts/mayastor/templates/_helper.tpl | 12 ++++++++++++ .../templates/csi-controller/Deployment.yaml | 2 +- .../templates/csi-controller/PriorityClass.yaml | 15 +++++++++++++++ charts/mayastor/templates/csi-node/DaemonSet.yaml | 3 ++- .../templates/csi-node/PriorityClass.yaml | 15 +++++++++++++++ .../mayastor/templates/io-engine/DaemonSet.yaml | 2 +- .../templates/io-engine/PriorityClass.yaml | 15 +++++++++++++++ 7 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 charts/mayastor/templates/csi-controller/PriorityClass.yaml create mode 100644 charts/mayastor/templates/csi-node/PriorityClass.yaml create mode 100644 charts/mayastor/templates/io-engine/PriorityClass.yaml diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index f8f21672..d2384542 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -7,10 +7,22 @@ {{- printf "%s-api-rest" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} {{- end -}} +{{- define "mayastor.csiNode.fullname" -}} + {{- printf "%s-csi-node" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end -}} + +{{- define "mayastor.csiController.fullname" -}} + {{- printf "%s-csi-controller" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end -}} + {{- define "mayastor.etcd.fullname" -}} {{- printf "%s-etcd" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} {{- end -}} +{{- define "mayastor.ioEngine.fullname" -}} + {{- printf "%s-io-engine" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end -}} + {{- define "mayastor.metrics.fullname" -}} {{- printf "%s-metrics" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} {{- end -}} diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index b2ce24e2..4b24af26 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -4,7 +4,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ printf "%s-csi-controller" (include "common.names.fullname" .) }} + name: {{ include "mayastor.csiController.fullname" . }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} app.kubernetes.io/component: csi-controller diff --git a/charts/mayastor/templates/csi-controller/PriorityClass.yaml b/charts/mayastor/templates/csi-controller/PriorityClass.yaml new file mode 100644 index 00000000..65facbd4 --- /dev/null +++ b/charts/mayastor/templates/csi-controller/PriorityClass.yaml @@ -0,0 +1,15 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +description: This priority class should be used for the Mayastor CSI Controller deployment only. +globalDefault: false +metadata: + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: csi-controller + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} + name: {{ printf "%s-critical" (include "mayastor.csiController.fullname" .) }} +value: 910001000 \ No newline at end of file diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 8f568b97..b0caa4d7 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -1,7 +1,8 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ printf "%s-csi-node" (include "common.names.fullname" .) }} + + name: {{ include "mayastor.csiNode.fullname" . }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} app.kubernetes.io/component: csi-node diff --git a/charts/mayastor/templates/csi-node/PriorityClass.yaml b/charts/mayastor/templates/csi-node/PriorityClass.yaml new file mode 100644 index 00000000..905ceb56 --- /dev/null +++ b/charts/mayastor/templates/csi-node/PriorityClass.yaml @@ -0,0 +1,15 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +description: This priority class should be used for the Mayastor CSI driver node deployment only. +globalDefault: false +metadata: + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: csi-node + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} + name: {{ printf "%s-critical" (include "mayastor.csiNode.fullname" .) }} +value: 910000000 \ No newline at end of file diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 4176f23e..492a4811 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ printf "%s-io-engine" (include "common.names.fullname" .) }} + name: {{ include "mayastor.ioEngine.fullname" . }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} app.kubernetes.io/component: io-engine diff --git a/charts/mayastor/templates/io-engine/PriorityClass.yaml b/charts/mayastor/templates/io-engine/PriorityClass.yaml new file mode 100644 index 00000000..92987332 --- /dev/null +++ b/charts/mayastor/templates/io-engine/PriorityClass.yaml @@ -0,0 +1,15 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +description: This priority class should be used for the Mayastor CSI Dataplane deployment only. +globalDefault: false +metadata: + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: csi-node + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} + name: {{ printf "%s-critical" (include "mayastor.ioEngine.fullname" .) }} +value: 910002000 \ No newline at end of file From 0ff8a53da8c9e3ecf304ff1e8b2b05e4809e5675 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 12:22:40 +0700 Subject: [PATCH 117/188] Fix etcd rbac create --- charts/mayastor/templates/ClusterRole.yaml | 4 ++++ charts/mayastor/values.yaml | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/ClusterRole.yaml b/charts/mayastor/templates/ClusterRole.yaml index 230ea579..db39caa6 100644 --- a/charts/mayastor/templates/ClusterRole.yaml +++ b/charts/mayastor/templates/ClusterRole.yaml @@ -65,4 +65,8 @@ rules: - apiGroups: [""] resources: ["namespaces"] verbs: ["get"] + +- apiGroups: ["monitoring.coreos.com"] + resources: ["servicemonitors"] + verbs: ["get", "create"] {{- end }} \ No newline at end of file diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index f2a25c82..9c274f73 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -599,8 +599,7 @@ etcd: rbac: ## @param etcd.auth.rbac.create Switch to enable RBAC authentication ## - #create: true - enabled: false + create: false ## @param etcd.auth.rbac.allowNoneAuthentication Allow to use etcd without configuring RBAC authentication ## allowNoneAuthentication: true From ac61a9fbcfe2ff533b80ef486c28a9d43521d782 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 12:32:46 +0700 Subject: [PATCH 118/188] Apply priorityClassName --- charts/mayastor/templates/csi-controller/Deployment.yaml | 1 + charts/mayastor/templates/csi-node/DaemonSet.yaml | 3 ++- charts/mayastor/templates/io-engine/DaemonSet.yaml | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 4b24af26..d6fd84a8 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -41,6 +41,7 @@ spec: {{- if .Values.ioEngine.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.nodeSelector "context" $) | nindent 8 }} {{- end }} + priorityClassName: {{ printf "%s-critical" (include "mayastor.csiController.fullname" .) }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- if .Values.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index b0caa4d7..fccc94df 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -38,10 +38,11 @@ spec: hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.hostAliases "context" $) | nindent 8 }} {{- end }} hostNetwork: true + {{- include "mayastor.imagePullSecrets" . | nindent 6 }} {{- if .Values.csiNode.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.csiNode.nodeSelector "context" $) | nindent 8 }} {{- end }} - {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + priorityClassName: {{ printf "%s-critical" (include "mayastor.csiNode.fullname" .) }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- if .Values.csiNode.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.csiNode.tolerations "context" .) | nindent 8 }} diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 492a4811..17693c82 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -50,6 +50,7 @@ spec: {{- if .Values.ioEngine.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.nodeSelector "context" $) | nindent 8 }} {{- end }} + priorityClassName: {{ printf "%s-critical" (include "mayastor.ioEngine.fullname" .) }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- if .Values.ioEngine.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.tolerations "context" .) | nindent 8 }} From 2b5567f867893ace57f2eb6c23ae985d98fba866 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 16:31:13 +0700 Subject: [PATCH 119/188] Add csiNode rbac --- charts/mayastor/templates/_helper.tpl | 18 +++-- .../templates/csi-node/ClusterRole.yaml | 72 +++++++++++++++++++ .../csi-node/ClusterRoleBinding.yaml | 23 ++++++ .../templates/csi-node/DaemonSet.yaml | 18 +++++ .../templates/csi-node/ServiceAccount.yaml | 21 ++++++ charts/mayastor/values.yaml | 60 ++++++++++++++++ 6 files changed, 207 insertions(+), 5 deletions(-) create mode 100644 charts/mayastor/templates/csi-node/ClusterRole.yaml create mode 100644 charts/mayastor/templates/csi-node/ClusterRoleBinding.yaml create mode 100644 charts/mayastor/templates/csi-node/ServiceAccount.yaml diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index d2384542..98d0e9dd 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -40,6 +40,14 @@ {{- end -}} {{- end -}} +{{- define "mayastor.csiNode.serviceAccountName" -}} +{{- if .Values.csiNode.serviceAccount.create -}} + {{ default (printf "%s" (include "mayastor.csiNode.fullname" .)) .Values.csiNode.serviceAccount.name | trunc 63 | trimSuffix "-" }} +{{- else -}} + {{ default "default" .Values.csiNode.serviceAccount.name }} +{{- end -}} +{{- end -}} + {{/* Return the proper Mayastor image name */}} {{- define "mayastor.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }} @@ -65,11 +73,6 @@ {{ include "common.images.image" (dict "imageRoot" .Values.csiController.attacher.image "global" .Values.global) }} {{- end -}} -{{/* Return the proper Mayastor CSI Controller Livenessprobe image name */}} -{{- define "mayastor.csiController.livenessprobe.image" -}} - {{ include "common.images.image" (dict "imageRoot" .Values.csiController.livenessprobe.image "global" .Values.global) }} -{{- end -}} - {{/* Return the proper Mayastor CSI Controller Provisioner image name */}} {{- define "mayastor.csiController.provisioner.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.csiController.provisioner.image "global" .Values.global) }} @@ -85,6 +88,11 @@ {{ include "common.images.image" (dict "imageRoot" .Values.csiNode.driverRegistrar.image "global" .Values.global) }} {{- end -}} +{{/* Return the proper Mayastor CSI Controller Livenessprobe image name */}} +{{- define "mayastor.csiNode.livenessprobe.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.csiNode.livenessprobe.image "global" .Values.global) }} +{{- end -}} + {{/* Return the proper Mayastor I/O Engine image name */}} {{- define "mayastor.ioEngine.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.ioEngine.image "global" .Values.global) }} diff --git a/charts/mayastor/templates/csi-node/ClusterRole.yaml b/charts/mayastor/templates/csi-node/ClusterRole.yaml new file mode 100644 index 00000000..9ffedaa9 --- /dev/null +++ b/charts/mayastor/templates/csi-node/ClusterRole.yaml @@ -0,0 +1,72 @@ +# Source: mayastor-control-plane/templates/operator-rbac.yaml + +{{- if and .Values.csiNode.serviceAccount.create .Values.csiNode.rbac.create }} +apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} +kind: ClusterRole +metadata: + name: {{ include "mayastor.csiNode.fullname" . }} +rules: + # must create mayastor crd if it doesn't exist +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list"] + + # must read diskpool info +- apiGroups: ["openebs.io"] + resources: ["diskpools"] + verbs: ["get", "list", "watch", "update", "replace", "patch"] + # must update diskpool status +- apiGroups: ["openebs.io"] + resources: ["diskpools/status"] + verbs: ["update", "patch"] + + # external provisioner & attacher +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "create", "delete", "patch"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + + # external provisioner +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + + # external attacher +- apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] +- apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + + # CSI nodes must be listed +- apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + + # get kube-system namespace to retrieve Uid +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get"] + +- apiGroups: ["monitoring.coreos.com"] + resources: ["servicemonitors"] + verbs: ["get", "create"] +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/csi-node/ClusterRoleBinding.yaml b/charts/mayastor/templates/csi-node/ClusterRoleBinding.yaml new file mode 100644 index 00000000..5fd76865 --- /dev/null +++ b/charts/mayastor/templates/csi-node/ClusterRoleBinding.yaml @@ -0,0 +1,23 @@ +# Source: mayastor-control-plane/includes/operator-rbac.yaml + +{{- if and .Values.serviceAccount.create .Values.rbac.create }} +apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} +kind: ClusterRoleBinding +metadata: + name: {{ include "mayastor.csiNode.fullname" . }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "mayastor.csiNode.serviceAccountName" . }} + namespace: {{ include "common.names.namespace" . | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "mayastor.csiNode.fullname" . }} +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index fccc94df..71340f34 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -163,6 +163,24 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration + - name: liveness-probe + image: {{ include "mayastor.csiNode.livenessprobe.image" . }} + imagePullPolicy: {{ .Values.csiNode.livenessprobe.image.pullPolicy | quote }} + args: + - "--csi-address=$(CSI_SOCKET_ADDRESS)" + env: + - name: CSI_SOCKET_ADDRESS + value: {{ .Values.csiNode.socketAddress | quote }} + resources: + limits: + cpu: "100m" + memory: "50Mi" + requests: + cpu: "100m" + memory: "50Mi" + volumeMounts: + - name: plugin-dir + mountPath: /csi volumes: - name: device hostPath: diff --git a/charts/mayastor/templates/csi-node/ServiceAccount.yaml b/charts/mayastor/templates/csi-node/ServiceAccount.yaml new file mode 100644 index 00000000..ed72b8ac --- /dev/null +++ b/charts/mayastor/templates/csi-node/ServiceAccount.yaml @@ -0,0 +1,21 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "mayastor.csiNode.serviceAccountName" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if or .Values.csiNode.serviceAccount.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.csiNode.serviceAccount.annotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.csiNode.serviceAccount.annotations "context" $) | nindent 4 }} + {{- end }} + {{- end }} +automountServiceAccountToken: {{ .Values.csiNode.serviceAccount.automountServiceAccountToken }} +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 9c274f73..5368eb5c 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -574,6 +574,30 @@ csiNode: ## pullSecrets: [] + ## CSI livenessprobe + ## ref: https://github.com/kubernetes-csi/livenessprobe + ## + livenessprobe: + image: + # registry: quay.io + # repository: k8scsi/csi-node-driver-registrar + # tag: v2.1.0 + registry: k8s.gcr.io + repository: sig-storage/livenessprobe + tag: v2.6.0 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + nvme: # nvme_core module io timeout in seconds io_timeout: "30" @@ -581,6 +605,42 @@ csiNode: socketAddress: /csi/csi.sock + ## Specifies whether a ServiceAccount should be created + ## + serviceAccount: + ## @param csiNode.serviceAccount.create Enable the creation of a ServiceAccount for Mayastor Control Plane Rest pods + ## + create: true + ## @param csiNode.serviceAccount.name Name of the created ServiceAccount + ## If not set and create is true, a name is generated using the fullname template + ## + name: "" + ## @param csiNode.serviceAccount.automountServiceAccountToken Auto-mount the service account token in the pod + ## + automountServiceAccountToken: true + ## @param csiNode.serviceAccount.annotations Additional custom annotations for the ServiceAccount + ## + annotations: {} + ## Role Based Access + ## Ref: https://kubernetes.io/docs/admin/authorization/rbac/ + ## + rbac: + ## @param csiNode.rbac.create Specify whether RBAC resources should be created and used + ## + create: true + ## @param csiNode.rbac.rules Custom RBAC rules + ## Example: + ## rules: + ## - apiGroups: + ## - "" + ## resources: + ## - pods + ## verbs: + ## - get + ## - list + ## + rules: [] + ## @section etcd parameters ## etcd: From 39cd561dea491f4dbea469ea3c7cc49a1ef41c2d Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 16:32:19 +0700 Subject: [PATCH 120/188] Add csiNode ServiceAccount --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 71340f34..73dc627b 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -43,7 +43,7 @@ spec: nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.csiNode.nodeSelector "context" $) | nindent 8 }} {{- end }} priorityClassName: {{ printf "%s-critical" (include "mayastor.csiNode.fullname" .) }} - serviceAccountName: {{ include "mayastor.serviceAccountName" . }} + serviceAccountName: {{ include "mayastor.csiNode.serviceAccountName" . }} {{- if .Values.csiNode.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.csiNode.tolerations "context" .) | nindent 8 }} {{- end }} From fc4adf1f39b50d21a74e2e942853927c6686c587 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 16:42:13 +0700 Subject: [PATCH 121/188] Move resources to values.yaml --- .../templates/csi-node/DaemonSet.yaml | 30 ++++++------------- charts/mayastor/values.yaml | 28 +++++++++++++++-- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 73dc627b..be6c1414 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -108,13 +108,9 @@ spec: {{- else }} value: {{ .Values.csiNode.logLevel }} {{- end }} - resources: - limits: - cpu: "100m" - memory: "50Mi" - requests: - cpu: "100m" - memory: "50Mi" + {{- if .Values.csiNode.resources }} + resources: {{ toYaml .Values.csiNode.resources | nindent 12 }} + {{- end }} volumeMounts: - name: device mountPath: /dev @@ -151,13 +147,9 @@ spec: - containerPort: 10199 name: csi-driver protocol: TCP - resources: - limits: - cpu: "100m" - memory: "50Mi" - requests: - cpu: "100m" - memory: "50Mi" + {{- if .Values.csiNode.driverRegistrar.resources }} + resources: {{ toYaml .Values.csiNode.driverRegistrar.resources | nindent 12 }} + {{- end }} volumeMounts: - name: plugin-dir mountPath: /csi @@ -171,13 +163,9 @@ spec: env: - name: CSI_SOCKET_ADDRESS value: {{ .Values.csiNode.socketAddress | quote }} - resources: - limits: - cpu: "100m" - memory: "50Mi" - requests: - cpu: "100m" - memory: "50Mi" + {{- if .Values.csiNode.livenessprobe.resources }} + resources: {{ toYaml .Values.csiNode.livenessprobe.resources | nindent 12 }} + {{- end }} volumeMounts: - name: plugin-dir mountPath: /csi diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 5368eb5c..229ac465 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -523,10 +523,10 @@ csiNode: resources: limits: cpu: 100m - memory: 50Mi + memory: 64Mi requests: cpu: 100m - memory: 50Mi + memory: 64Mi ## @param affinity Affinity for Mayastor I/O Engine pods assignment ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set @@ -573,6 +573,18 @@ csiNode: ## - myRegistryKeySecretName ## pullSecrets: [] + ## CSI Driver Registrar resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## @param csiNode.driverRegistrar.resources.limits The resources limits for the Mayastor I/O Engine containers + ## @param csiNode.driverRegistrar.resources.requests The requested resources for the Mayastor I/O Engine containers + ## + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 100m + memory: 64Mi ## CSI livenessprobe ## ref: https://github.com/kubernetes-csi/livenessprobe @@ -597,6 +609,18 @@ csiNode: ## - myRegistryKeySecretName ## pullSecrets: [] + ## CSI liveness Probe resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## @param csiNode.livenessprobe.resources.limits The resources limits for the Mayastor I/O Engine containers + ## @param csiNode.livenessprobe.resources.requests The requested resources for the Mayastor I/O Engine containers + ## + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 100m + memory: 64Mi nvme: # nvme_core module io timeout in seconds From ea55f102b80723be9feaed1e06843f7134c2c399 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 16:54:55 +0700 Subject: [PATCH 122/188] Add healthz port --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 10 +++++++--- charts/mayastor/values.yaml | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index be6c1414..8dc4b1d1 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -129,10 +129,15 @@ spec: args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" - "--kubelet-registration-path=/var/lib/kubelet/plugins/io.openebs.mayastor/csi.sock" + - "--http-endpoint=:{{ .Values.csiNode.driverRegistrar.containerPorts.healthz }}" - "--v=5" env: - name: CSI_SOCKET_ADDRESS value: {{ .Values.csiNode.socketAddress | quote }} + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName lifecycle: preStop: exec: @@ -142,10 +147,9 @@ spec: - |- echo "Deleting stale file" rm -rf /registration/io.openebs.mayastor /registration/io.openebs.mayastor-reg.sock - # Mayastor node plugin gRPC server ports: - - containerPort: 10199 - name: csi-driver + - containerPort: {{ .Values.csiNode.driverRegistrar.containerPorts.healthz }} + name: healthz protocol: TCP {{- if .Values.csiNode.driverRegistrar.resources }} resources: {{ toYaml .Values.csiNode.driverRegistrar.resources | nindent 12 }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 229ac465..9a18c120 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -573,6 +573,10 @@ csiNode: ## - myRegistryKeySecretName ## pullSecrets: [] + ## @param agentCore.containerPorts.http Mayastor Agent Core GRPC container port + ## + containerPorts: + healthz: 10199 ## CSI Driver Registrar resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## @param csiNode.driverRegistrar.resources.limits The resources limits for the Mayastor I/O Engine containers From 6c38322f5e4b7983ff09e9be850c670404498b0c Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 17:47:39 +0700 Subject: [PATCH 123/188] Fix driverRegistrar http port --- charts/mayastor/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 9a18c120..ab2bed14 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -576,7 +576,7 @@ csiNode: ## @param agentCore.containerPorts.http Mayastor Agent Core GRPC container port ## containerPorts: - healthz: 10199 + healthz: 9809 ## CSI Driver Registrar resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## @param csiNode.driverRegistrar.resources.limits The resources limits for the Mayastor I/O Engine containers From 408296f50c949c9c21780e0574e8d1e7bc00dd25 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 17:56:18 +0700 Subject: [PATCH 124/188] Fix csi-plugin containerPort --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 4 ++++ charts/mayastor/values.yaml | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 8dc4b1d1..adf1ad7a 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -108,6 +108,10 @@ spec: {{- else }} value: {{ .Values.csiNode.logLevel }} {{- end }} + ports: + - name: grpc + containerPort: {{ .Values.csiNode.containerPorts.grpc }} + protocol: TCP {{- if .Values.csiNode.resources }} resources: {{ toYaml .Values.csiNode.resources | nindent 12 }} {{- end }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index ab2bed14..9545e648 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -497,6 +497,10 @@ csiNode: ## - myRegistryKeySecretName ## pullSecrets: [] + ## @param csiNode.containerPorts.http Mayastor Agent Core GRPC container port + ## + containerPorts: + grpc: 10199 ## Configure Pods Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod ## @param csiNode.podSecurityContext.enabled Enabled Mayastor I/O Engine pods' Security Context @@ -573,7 +577,7 @@ csiNode: ## - myRegistryKeySecretName ## pullSecrets: [] - ## @param agentCore.containerPorts.http Mayastor Agent Core GRPC container port + ## @param csiNode.driverRegistrar.containerPorts.http Mayastor Agent Core GRPC container port ## containerPorts: healthz: 9809 From d8f875dc87c4deacd1faf3b45426e88ad73bce00 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 17:59:40 +0700 Subject: [PATCH 125/188] Fix driverRegistrar healthz port --- charts/mayastor/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 9545e648..ba00a2c1 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -580,7 +580,7 @@ csiNode: ## @param csiNode.driverRegistrar.containerPorts.http Mayastor Agent Core GRPC container port ## containerPorts: - healthz: 9809 + healthz: 9810 ## CSI Driver Registrar resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## @param csiNode.driverRegistrar.resources.limits The resources limits for the Mayastor I/O Engine containers From cf1d9e92df506baca8eb0057d57b157d15aa091d Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 18:05:42 +0700 Subject: [PATCH 126/188] Fix driverRegistrar port --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index adf1ad7a..68ccc18d 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -133,7 +133,7 @@ spec: args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" - "--kubelet-registration-path=/var/lib/kubelet/plugins/io.openebs.mayastor/csi.sock" - - "--http-endpoint=:{{ .Values.csiNode.driverRegistrar.containerPorts.healthz }}" + - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.driverRegistrar.containerPorts.healthz }}" - "--v=5" env: - name: CSI_SOCKET_ADDRESS @@ -142,6 +142,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP lifecycle: preStop: exec: From 2ba302da8d7a86e11c38fcead688e69a7b4f4d84 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 18:13:34 +0700 Subject: [PATCH 127/188] Fix livenessprobe port --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 68ccc18d..77fa2b6c 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -172,6 +172,8 @@ spec: imagePullPolicy: {{ .Values.csiNode.livenessprobe.image.pullPolicy | quote }} args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" + - "--health-port=9809" + - "--metrics-address=20201" env: - name: CSI_SOCKET_ADDRESS value: {{ .Values.csiNode.socketAddress | quote }} From 0e206709fb67cb800da2271e72d2788a38527839 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 18:37:51 +0700 Subject: [PATCH 128/188] Fix healthz ports --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 11 ++++++++--- charts/mayastor/values.yaml | 8 +++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 77fa2b6c..35907895 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -133,7 +133,7 @@ spec: args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" - "--kubelet-registration-path=/var/lib/kubelet/plugins/io.openebs.mayastor/csi.sock" - - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.driverRegistrar.containerPorts.healthz }}" + - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.driverRegistrar.containerPorts.diagnostic }}" - "--v=5" env: - name: CSI_SOCKET_ADDRESS @@ -172,11 +172,16 @@ spec: imagePullPolicy: {{ .Values.csiNode.livenessprobe.image.pullPolicy | quote }} args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" - - "--health-port=9809" - - "--metrics-address=20201" + - "--health-port={{ .Values.csiNode.livenessprobe.containerPorts.health }}" + - "--metrics-address=$(MY_POD_IP):{{ .Values.csiNode.livenessprobe.containerPorts.metrics }}" + - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.livenessprobe.containerPorts.diagnostic }}" env: - name: CSI_SOCKET_ADDRESS value: {{ .Values.csiNode.socketAddress | quote }} + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP {{- if .Values.csiNode.livenessprobe.resources }} resources: {{ toYaml .Values.csiNode.livenessprobe.resources | nindent 12 }} {{- end }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index ba00a2c1..b1339145 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -580,7 +580,7 @@ csiNode: ## @param csiNode.driverRegistrar.containerPorts.http Mayastor Agent Core GRPC container port ## containerPorts: - healthz: 9810 + diagnostic: 9809 ## CSI Driver Registrar resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## @param csiNode.driverRegistrar.resources.limits The resources limits for the Mayastor I/O Engine containers @@ -617,6 +617,12 @@ csiNode: ## - myRegistryKeySecretName ## pullSecrets: [] + ## @param csiNode.livenessprobe.containerPorts.http Mayastor Agent Core GRPC container port + ## + containerPorts: + health: 9810 + diagnostic: 50212 + metrics: 50211 ## CSI liveness Probe resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## @param csiNode.livenessprobe.resources.limits The resources limits for the Mayastor I/O Engine containers From e602aa086c7b6cd3c34731573b104e7e69b3d5c5 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 18:39:57 +0700 Subject: [PATCH 129/188] Fix diagnostic port --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 35907895..56d56066 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -156,8 +156,8 @@ spec: echo "Deleting stale file" rm -rf /registration/io.openebs.mayastor /registration/io.openebs.mayastor-reg.sock ports: - - containerPort: {{ .Values.csiNode.driverRegistrar.containerPorts.healthz }} - name: healthz + - containerPort: {{ .Values.csiNode.driverRegistrar.containerPorts.diagnostic }} + name: diagnostic protocol: TCP {{- if .Values.csiNode.driverRegistrar.resources }} resources: {{ toYaml .Values.csiNode.driverRegistrar.resources | nindent 12 }} From 044ca6f1cac4356496d62189739270e2179020c4 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 18:44:39 +0700 Subject: [PATCH 130/188] Fix healthz ports --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 9 ++++----- charts/mayastor/values.yaml | 6 +++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 56d56066..61cb6181 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -133,7 +133,7 @@ spec: args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" - "--kubelet-registration-path=/var/lib/kubelet/plugins/io.openebs.mayastor/csi.sock" - - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.driverRegistrar.containerPorts.diagnostic }}" + - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.driverRegistrar.containerPorts.healthz }}" - "--v=5" env: - name: CSI_SOCKET_ADDRESS @@ -156,8 +156,8 @@ spec: echo "Deleting stale file" rm -rf /registration/io.openebs.mayastor /registration/io.openebs.mayastor-reg.sock ports: - - containerPort: {{ .Values.csiNode.driverRegistrar.containerPorts.diagnostic }} - name: diagnostic + - containerPort: {{ .Values.csiNode.driverRegistrar.containerPorts.healthz }} + name: healthz protocol: TCP {{- if .Values.csiNode.driverRegistrar.resources }} resources: {{ toYaml .Values.csiNode.driverRegistrar.resources | nindent 12 }} @@ -172,9 +172,8 @@ spec: imagePullPolicy: {{ .Values.csiNode.livenessprobe.image.pullPolicy | quote }} args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" - - "--health-port={{ .Values.csiNode.livenessprobe.containerPorts.health }}" - "--metrics-address=$(MY_POD_IP):{{ .Values.csiNode.livenessprobe.containerPorts.metrics }}" - - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.livenessprobe.containerPorts.diagnostic }}" + - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.livenessprobe.containerPorts.healthz }}" env: - name: CSI_SOCKET_ADDRESS value: {{ .Values.csiNode.socketAddress | quote }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index b1339145..10abe1c7 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -580,7 +580,7 @@ csiNode: ## @param csiNode.driverRegistrar.containerPorts.http Mayastor Agent Core GRPC container port ## containerPorts: - diagnostic: 9809 + healthz: 9809 ## CSI Driver Registrar resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## @param csiNode.driverRegistrar.resources.limits The resources limits for the Mayastor I/O Engine containers @@ -620,8 +620,8 @@ csiNode: ## @param csiNode.livenessprobe.containerPorts.http Mayastor Agent Core GRPC container port ## containerPorts: - health: 9810 - diagnostic: 50212 + # health: 9810 + healthz: 9810 metrics: 50211 ## CSI liveness Probe resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ From 138b7591f3098f311e7befec889a590aa3dc9322 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 18:47:23 +0700 Subject: [PATCH 131/188] Fix metrics port --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 61cb6181..771b3861 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -172,7 +172,6 @@ spec: imagePullPolicy: {{ .Values.csiNode.livenessprobe.image.pullPolicy | quote }} args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" - - "--metrics-address=$(MY_POD_IP):{{ .Values.csiNode.livenessprobe.containerPorts.metrics }}" - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.livenessprobe.containerPorts.healthz }}" env: - name: CSI_SOCKET_ADDRESS From 09ce2031a1d5235d53484e70a2866387a488986c Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 18:50:48 +0700 Subject: [PATCH 132/188] Add livenessprobe --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 771b3861..5ff8c3b8 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -155,6 +155,12 @@ spec: - |- echo "Deleting stale file" rm -rf /registration/io.openebs.mayastor /registration/io.openebs.mayastor-reg.sock + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.csiNode.driverRegistrar.containerPorts.healthz }} + initialDelaySeconds: 5 + timeoutSeconds: 5 ports: - containerPort: {{ .Values.csiNode.driverRegistrar.containerPorts.healthz }} name: healthz From d07eda1711b73b00bc76d3fe2d5c192d6e549b9f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 18:56:09 +0700 Subject: [PATCH 133/188] Remove lifecycle prestop --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 9 --------- charts/mayastor/values.yaml | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 5ff8c3b8..128eb1bb 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -146,15 +146,6 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - |- - echo "Deleting stale file" - rm -rf /registration/io.openebs.mayastor /registration/io.openebs.mayastor-reg.sock livenessProbe: httpGet: path: /healthz diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 10abe1c7..2b7f7c85 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -622,7 +622,7 @@ csiNode: containerPorts: # health: 9810 healthz: 9810 - metrics: 50211 + # metrics: 50211 ## CSI liveness Probe resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## @param csiNode.livenessprobe.resources.limits The resources limits for the Mayastor I/O Engine containers From 9060f38b7e8411e676e7263acbf472c89c831aca Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 18:58:57 +0700 Subject: [PATCH 134/188] Fix livenessProbe port --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 128eb1bb..d408915e 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -149,7 +149,7 @@ spec: livenessProbe: httpGet: path: /healthz - port: {{ .Values.csiNode.driverRegistrar.containerPorts.healthz }} + port: {{ .Values.csiNode.livenessprobe.containerPorts.healthz }} initialDelaySeconds: 5 timeoutSeconds: 5 ports: From 4c46d6b34dea3b66979f99e509147c2d90acc9f4 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 21:11:08 +0700 Subject: [PATCH 135/188] Update livenessprobe to v2.7.0 --- charts/mayastor/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 2b7f7c85..0ecb987d 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -604,7 +604,7 @@ csiNode: # tag: v2.1.0 registry: k8s.gcr.io repository: sig-storage/livenessprobe - tag: v2.6.0 + tag: v2.7.0 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images From a944290a8044ea67e3533d71d7b2430b94dcde47 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 21:33:13 +0700 Subject: [PATCH 136/188] Update sig-storage/csi-node-driver-registrar to v2.5.1 --- charts/mayastor/README.md | 97 +++++++++++++++++++ .../templates/csi-node/DaemonSet.yaml | 10 +- charts/mayastor/values.yaml | 8 +- 3 files changed, 105 insertions(+), 10 deletions(-) create mode 100644 charts/mayastor/README.md diff --git a/charts/mayastor/README.md b/charts/mayastor/README.md new file mode 100644 index 00000000..e08ff572 --- /dev/null +++ b/charts/mayastor/README.md @@ -0,0 +1,97 @@ + + +# Helm chart for Mayastor + +Mayastor is a cloud-native declarative data plane written in Rust. Our goal is to abstract storage resources and their differences through the data plane such that users only need to supply the what and do not have to worry about the how so that individual teams stay in control. + +Mayastor also try to be as unopinionated as possible. What this means is that we try to work with the existing storage systems you might already have and unify them as abstract resources instead of swapping them out whenever the resources are local or remote. + +[Overview of Mayastor](https://mayastor.gitbook.io) + +**This chart is not maintained by the upstream project and any issues with the chart should be raised [here](https://github.com/startechnica/apps/issues/new/choose)** + +## TL;DR + +```bash +$ helm repo add startechnica https://startechnica.github.io/apps +$ helm install my-release startechnica/mayastor +``` + +## Prerequisites + +- Kubernetes 1.20+ +- Helm 3.2.0+ + +## Installing the Chart + +To install the chart with the release name `my-release` on `my-release` namespace: + +```bash +$ helm repo add startechnica https://startechnica.github.io/apps +$ helm install my-release startechnica/mayastor --namespace my-release --create-namespace +``` + +These commands deploy Mayastor on the Kubernetes cluster in the default configuration. + +> **Tip**: List all releases using `helm list -A` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```bash +$ helm delete my-release --namespace my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Parameters + +### Global parameters + +| Name | Description | Value | +| ------------------------- | ----------------------------------------------- | ----- | +| `global.imageRegistry` | Global Docker image registry | `""` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` | + +### Common parameters + +| Name | Description | Value | +| ------------------- | ------------------------------------------------------------------------------------------ | --------------- | +| `nameOverride` | String to partially override mayastor.fullname template (will maintain the release name) | `""` | +| `fullnameOverride` | String to fully override mayastor.fullname template | `""` | +| `kubeVersion` | Force target Kubernetes version (using Helm capabilities if not set) | `""` | +| `clusterDomain` | Kubernetes Cluster Domain | `cluster.local` | +| `extraDeploy` | Extra objects to deploy (value evaluated as a template) | `[]` | +| `commonLabels` | Add labels to all the deployed resources | `{}` | +| `commonAnnotations` | Add annotations to all the deployed resources | `{}` | + +### Setting Pod's affinity + +This chart allows you to set your custom affinity using the `affinity` parameter. Find more information about Pod's affinity in the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). + +### Deploying extra resources + +There are cases where you may want to deploy extra objects, such a ConfigMap containing your app's configuration or some extra deployment with a micro service used by your app. For covering this case, the chart allows adding the full specification of other objects using the `extraDeploy` parameter. + +## Troubleshooting + +Find more information about how to deal with common errors related to Startechnica's Helm charts in [this troubleshooting guide](https://startechnica.github.io/doc/troubleshoot-helm-chart-issues). + +## Upgrading + +## License + +Copyright © 2022 Startechnica + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index d408915e..44e09dd7 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -129,11 +129,11 @@ spec: mountPropagation: "Bidirectional" - name: csi-driver-registrar image: {{ include "mayastor.csiNode.driverRegistrar.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + imagePullPolicy: {{ .Values.csiNode.driverRegistrar.image.pullPolicy | quote }} args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" - "--kubelet-registration-path=/var/lib/kubelet/plugins/io.openebs.mayastor/csi.sock" - - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.driverRegistrar.containerPorts.healthz }}" + - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.driverRegistrar.containerPorts.http }}" - "--v=5" env: - name: CSI_SOCKET_ADDRESS @@ -149,11 +149,11 @@ spec: livenessProbe: httpGet: path: /healthz - port: {{ .Values.csiNode.livenessprobe.containerPorts.healthz }} + port: {{ .Values.csiNode.livenessprobe.containerPorts.http }} initialDelaySeconds: 5 timeoutSeconds: 5 ports: - - containerPort: {{ .Values.csiNode.driverRegistrar.containerPorts.healthz }} + - containerPort: {{ .Values.csiNode.driverRegistrar.containerPorts.http }} name: healthz protocol: TCP {{- if .Values.csiNode.driverRegistrar.resources }} @@ -169,7 +169,7 @@ spec: imagePullPolicy: {{ .Values.csiNode.livenessprobe.image.pullPolicy | quote }} args: - "--csi-address=$(CSI_SOCKET_ADDRESS)" - - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.livenessprobe.containerPorts.healthz }}" + - "--http-endpoint=$(MY_POD_IP):{{ .Values.csiNode.livenessprobe.containerPorts.http }}" env: - name: CSI_SOCKET_ADDRESS value: {{ .Values.csiNode.socketAddress | quote }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 0ecb987d..147931b4 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -564,7 +564,7 @@ csiNode: # tag: v2.1.0 registry: k8s.gcr.io repository: sig-storage/csi-node-driver-registrar - tag: v2.5.0 + tag: v2.5.1 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -580,7 +580,7 @@ csiNode: ## @param csiNode.driverRegistrar.containerPorts.http Mayastor Agent Core GRPC container port ## containerPorts: - healthz: 9809 + http: 9809 ## CSI Driver Registrar resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## @param csiNode.driverRegistrar.resources.limits The resources limits for the Mayastor I/O Engine containers @@ -620,9 +620,7 @@ csiNode: ## @param csiNode.livenessprobe.containerPorts.http Mayastor Agent Core GRPC container port ## containerPorts: - # health: 9810 - healthz: 9810 - # metrics: 50211 + http: 9810 ## CSI liveness Probe resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## @param csiNode.livenessprobe.resources.limits The resources limits for the Mayastor I/O Engine containers From 8946cd768135dfe9edecf67facfda0b8d9b30555 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 21:41:36 +0700 Subject: [PATCH 137/188] Update sig-storage/csi-attacher to v3.5.0 --- charts/mayastor/values.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 147931b4..255566d8 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -389,7 +389,7 @@ csiController: image: registry: k8s.gcr.io repository: sig-storage/csi-attacher - tag: v3.4.0 + tag: v3.5.0 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -427,14 +427,14 @@ csiController: metrics: enabled: false - ## CSI livenessprobe - ## ref: https://github.com/kubernetes-csi/livenessprobe + ## CSI external-health-monitor-controller + ## ref: https://kubernetes-csi.github.io/docs/external-health-monitor-controller.html ## - livenessprobe: + healthMonitor: image: registry: k8s.gcr.io - repository: sig-storage/livenessprobe - tag: v2.6.0 + repository: csi-external-health-monitor-controller + tag: v0.5.0 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images From 700e28c9668798673a2cd3241acc7bed5d20c58f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 21:43:15 +0700 Subject: [PATCH 138/188] Update sig-storage/csi-provisioner to v3.1.1 --- charts/mayastor/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 255566d8..a9f8ecae 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -402,14 +402,14 @@ csiController: ## - myRegistryKeySecretName ## pullSecrets: [] - ## @param agentCore.containerPorts.http Mayastor Agent Core GRPC container port + ## @param csiController.attacher.containerPorts.http Mayastor Agent Core GRPC container port ## containerPorts: metrics: 9601 ## CSI attacher resource requests and limits ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ - ## @param attacher.resources.limits The resources limits for CSI attacher containers - ## @param attacher.resources.requests The requested resources for CSI attacher containers + ## @param csiController.attacher.resources.limits The resources limits for CSI attacher containers + ## @param csiController.attacher.resources.requests The requested resources for CSI attacher containers ## resources: ## Example: @@ -455,7 +455,7 @@ csiController: image: registry: k8s.gcr.io repository: sig-storage/csi-provisioner - tag: v3.1.0 + tag: v3.1.1 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images From a562ca9e434db2155e14a02bd6d115378895690d Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 22:05:07 +0700 Subject: [PATCH 139/188] Add csi-external-health-monitor-controller --- charts/mayastor/templates/_helper.tpl | 5 ++++ .../templates/csi-controller/Deployment.yaml | 26 ++++++++++++++++--- charts/mayastor/values.yaml | 2 ++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index 98d0e9dd..3fdaf6f8 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -73,6 +73,11 @@ {{ include "common.images.image" (dict "imageRoot" .Values.csiController.attacher.image "global" .Values.global) }} {{- end -}} +{{/* Return the proper Mayastor CSI Controller Health Monitor image name */}} +{{- define "mayastor.csiController.healthMonitor.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.csiController.healthMonitor.image "global" .Values.global) }} +{{- end -}} + {{/* Return the proper Mayastor CSI Controller Provisioner image name */}} {{- define "mayastor.csiController.provisioner.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.csiController.provisioner.image "global" .Values.global) }} diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index d6fd84a8..45e7645d 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -78,7 +78,7 @@ spec: {{- end }} env: - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock + value: {{ .Values.csiController.csiSocketAddress | quote }} - name: NAMESPACE valueFrom: fieldRef: @@ -102,11 +102,11 @@ spec: image: {{ include "mayastor.csiController.attacher.image" . }} imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} args: - - "--csi-address=$(ADDRESS)" + - "--csi-address=$(CSI_SOCKET_ADDRESS)" - "--v=2" env: - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock + value: {{ .Values.csiController.csiSocketAddress | quote }} - name: RUST_BACKTRACE {{- if .Values.diagnosticMode.enabled }} value: full @@ -121,6 +121,26 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-health-monitor + image: {{ include "mayastor.csiController.healthMonitor.image" . }} + imagePullPolicy: {{ .Values.csiController.healthMonitor.image.pullPolicy | quote }} + args: + - "--csiAddress=$(ADDRESS)" + - "--v=5" + env: + - name: ADDRESS + value: {{ .Values.csiController.csiSocketAddress | quote }} + - name: RUST_BACKTRACE + {{- if .Values.diagnosticMode.enabled }} + value: full + {{- else }} + value: full + {{- end }} + - name: RUST_LOG + value: {{ .Values.csiController.logLevel }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-controller image: {{ include "mayastor.csiController.image" . }} imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index a9f8ecae..15b498d3 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -469,6 +469,8 @@ csiController: ## pullSecrets: [] + csiSocketAddress: /var/lib/csi/sockets/pluginproxy/csi.sock + csiNode: enabled: true logLevel: info From 384c9db1e9d797f82d645622dce217ec0e76ec94 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 22:11:00 +0700 Subject: [PATCH 140/188] Revert back sig-storage/csi-attacher to v3.4.0 --- charts/mayastor/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 15b498d3..837de7a2 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -389,7 +389,7 @@ csiController: image: registry: k8s.gcr.io repository: sig-storage/csi-attacher - tag: v3.5.0 + tag: v3.4.0 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -434,7 +434,7 @@ csiController: image: registry: k8s.gcr.io repository: csi-external-health-monitor-controller - tag: v0.5.0 + tag: v0.4.0 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images From 850560e4fa6f2c6ee290f6c4ae49302a79750705 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 22:14:03 +0700 Subject: [PATCH 141/188] Fix sig-storage/csi-external-health-monitor-controller repository --- charts/mayastor/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 837de7a2..5cf387b2 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -433,8 +433,8 @@ csiController: healthMonitor: image: registry: k8s.gcr.io - repository: csi-external-health-monitor-controller - tag: v0.4.0 + repository: sig-storage/csi-external-health-monitor-controller + tag: v0.5.0 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images From 33236f6ba73118b712850be7619a5ffac5d4f75c Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 22:16:42 +0700 Subject: [PATCH 142/188] Update sig-storage/csi-attacher to v3.5.0 --- charts/mayastor/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 5cf387b2..803685d1 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -389,7 +389,7 @@ csiController: image: registry: k8s.gcr.io repository: sig-storage/csi-attacher - tag: v3.4.0 + tag: v3.5.0 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images From 8f6f15759f4f6573f47703b05ef9dc4694385da2 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 22:18:00 +0700 Subject: [PATCH 143/188] Fix csi-attacher path --- charts/mayastor/templates/csi-controller/Deployment.yaml | 2 +- charts/mayastor/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 45e7645d..6b83c945 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -102,7 +102,7 @@ spec: image: {{ include "mayastor.csiController.attacher.image" . }} imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} args: - - "--csi-address=$(CSI_SOCKET_ADDRESS)" + - "--csi-address=$(ADDRESS)" - "--v=2" env: - name: ADDRESS diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 803685d1..3b24c150 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -469,7 +469,7 @@ csiController: ## pullSecrets: [] - csiSocketAddress: /var/lib/csi/sockets/pluginproxy/csi.sock + csiSocketAddress: "/var/lib/csi/sockets/pluginproxy/csi.sock" csiNode: enabled: true From 0d937c78a8142d39b349c196249d1412fd43dd05 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 22:21:39 +0700 Subject: [PATCH 144/188] Fix csi-health-monitor csi-address --- charts/mayastor/templates/csi-controller/Deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 6b83c945..bc9ac57e 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -103,7 +103,7 @@ spec: imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} args: - "--csi-address=$(ADDRESS)" - - "--v=2" + - "--v=5" env: - name: ADDRESS value: {{ .Values.csiController.csiSocketAddress | quote }} @@ -125,7 +125,7 @@ spec: image: {{ include "mayastor.csiController.healthMonitor.image" . }} imagePullPolicy: {{ .Values.csiController.healthMonitor.image.pullPolicy | quote }} args: - - "--csiAddress=$(ADDRESS)" + - "--csi-address=$(ADDRESS)" - "--v=5" env: - name: ADDRESS From 1bbd35553ce6f0bb916be6754c0e7eb3546cedcf Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 22:25:31 +0700 Subject: [PATCH 145/188] Add csi-external-health-monitor-controller condition --- charts/mayastor/templates/csi-controller/Deployment.yaml | 4 +++- charts/mayastor/values.yaml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index bc9ac57e..b3649a1f 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -111,7 +111,7 @@ spec: {{- if .Values.diagnosticMode.enabled }} value: full {{- else }} - value: full + value: "1" {{- end }} - name: RUST_LOG value: {{ .Values.csiController.logLevel }} @@ -121,6 +121,7 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- if .Values.csiNode.healthMonitor.enabled }} - name: csi-health-monitor image: {{ include "mayastor.csiController.healthMonitor.image" . }} imagePullPolicy: {{ .Values.csiController.healthMonitor.image.pullPolicy | quote }} @@ -141,6 +142,7 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- end }} - name: csi-controller image: {{ include "mayastor.csiController.image" . }} imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 3b24c150..74011a4a 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -431,6 +431,7 @@ csiController: ## ref: https://kubernetes-csi.github.io/docs/external-health-monitor-controller.html ## healthMonitor: + enabled: false image: registry: k8s.gcr.io repository: sig-storage/csi-external-health-monitor-controller From 75c58038fdf30d789895dfb9a75a378ef7ed7183 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Sun, 19 Jun 2022 22:27:39 +0700 Subject: [PATCH 146/188] Fix csi-health-monitor condition --- charts/mayastor/templates/csi-controller/Deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index b3649a1f..31caa9d7 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -121,7 +121,7 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- if .Values.csiNode.healthMonitor.enabled }} + {{- if .Values.csiController.healthMonitor.enabled }} - name: csi-health-monitor image: {{ include "mayastor.csiController.healthMonitor.image" . }} imagePullPolicy: {{ .Values.csiController.healthMonitor.image.pullPolicy | quote }} From 20156f3a8821529481c064d9bca93f32b72e6bb3 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 10:14:57 +0700 Subject: [PATCH 147/188] test: ioEngine.ioEngine: debug --- charts/mayastor/values-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 5ccbec5e..2bbd44bb 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -47,7 +47,7 @@ diskPools: # device: /dev/sdb ioEngine: - logLevel: info + logLevel: debug tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane From 52c2f6422f44d63481b958bc1204e786b4e8ab60 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 10:15:38 +0700 Subject: [PATCH 148/188] test: ioEngine.ioEngine: debug --- charts/mayastor/values-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 2bbd44bb..aca93bff 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -47,7 +47,7 @@ diskPools: # device: /dev/sdb ioEngine: - logLevel: debug + ioEngine: debug tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane From b56b41d307443ad9dfe22138bba2bf364028c9b9 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 10:20:13 +0700 Subject: [PATCH 149/188] test: ioEngine.logLevel: debug --- charts/mayastor/values-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index aca93bff..2bbd44bb 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -47,7 +47,7 @@ diskPools: # device: /dev/sdb ioEngine: - ioEngine: debug + logLevel: debug tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane From 25b1af88cd0ed1def19006034ed167c0d2489c9a Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 10:29:58 +0700 Subject: [PATCH 150/188] Add io-engine affinity --- .../templates/io-engine/DaemonSet.yaml | 14 +++++------ charts/mayastor/values.yaml | 25 +++++++++++++++++-- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 17693c82..3b0323f5 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -22,24 +22,24 @@ spec: minReadySeconds: 10 template: metadata: - {{- if .Values.podAnnotations }} + {{- if .Values.ioEngine.podAnnotations }} annotations: - {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.podAnnotations "context" $) | nindent 8 }} {{- end }} labels: {{- include "common.labels.standard" . | nindent 8 }} app.kubernetes.io/component: io-engine openebs.io/logging: "true" - {{- if .Values.podLabels }} - {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- if .Values.ioEngine.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.podLabels "context" $) | nindent 8 }} {{- end }} spec: {{- if .Values.ioEngine.affinity }} affinity: {{- include "common.tplvalues.render" ( dict "value" .Values.ioEngine.affinity "context" $) | nindent 8 }} {{- else }} affinity: - podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "context" $) | nindent 10 }} - podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "context" $) | nindent 10 }} - nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.ioEngine.podAffinityPreset "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.ioEngine.podAntiAffinityPreset "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.ioEngine.nodeAffinityPreset.type "key" .Values.ioEngine.nodeAffinityPreset.key "values" .Values.ioEngine.nodeAffinityPreset.values) | nindent 10 }} {{- end }} dnsPolicy: ClusterFirstWithHostNet {{- if .Values.hostAliases }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 74011a4a..d9a87820 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -1062,9 +1062,30 @@ ioEngine: ## - myRegistryKeySecretName ## pullSecrets: [] - ## @param ioEngine.replicaCount Number of Mayastor I/O Engine replicas to deploy + ## @param ioEngine.updateStrategy.type updateStrategy for Mayastor Daemonset + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies + ## + updateStrategy: + type: RollingUpdate + ## @param ioEngine.podLabels Extra labels for Mayastor pods + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + ## + podLabels: {} + ## @param ioEngine.podAnnotations Annotations for Mayastor pods + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + ## + podAnnotations: {} + ## @param ioEngine.podAffinityPreset Pod affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard` + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAffinityPreset: "" + ## @param ioEngine.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard` + ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAntiAffinityPreset: "" + ## Node affinity preset + ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity ## - replicaCount: 1 ## @param ioEngine.containerPorts.http Mayastor I/O Engine HTTP container port ## @param ioEngine.containerPorts.https Mayastor I/O Engine HTTPS container port ## From 4c814fd6c78a569c01893b2d1b46ffb98edea3a9 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 10:42:28 +0700 Subject: [PATCH 151/188] Add -N to io-Engine --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 3b0323f5..a1748646 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -108,6 +108,7 @@ spec: # For example -l 1,2,10-20 means use core 1, 2, 10 to 20. # Note: Ensure that the CPU resources are updated accordingly. # If you use 2 CPUs, the CPU: field should also read 2. + - "-N$(MY_NODE_NAME)" - "-g$(MY_POD_IP)" {{- if .Values.ioEngine.cpuCount }} - "-l{{ include "mayastor.cpuFlag" . | quote }}" From 1648f092590124fc519184e379d21e15212c84ad Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 10:44:50 +0700 Subject: [PATCH 152/188] Change csiNode port to values --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index 44e09dd7..e2dbee80 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -74,7 +74,7 @@ spec: args: - "--csi-socket=$(CSI_SOCKET_ADDRESS)" - "--node-name=$(MY_NODE_NAME)" - - "--grpc-endpoint=$(MY_POD_IP):10199" + - "--grpc-endpoint=$(MY_POD_IP):{{ .Values.csiNode.containerPorts.grpc }}" {{- if .Values.csiNode.nvme.io_timeout_enabled }} - "--nvme-core-io-timeout={{ .Values.csiNode.nvme.io_timeout }}" {{- end }} From d20e21bf2f197377cb9a0c6f3f0f79889cdac8cc Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 10:46:52 +0700 Subject: [PATCH 153/188] Add ioEngine.nodeAffinityPreset --- charts/mayastor/values.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index d9a87820..bb410dc2 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -1086,6 +1086,25 @@ ioEngine: ## Node affinity preset ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity ## + nodeAffinityPreset: + ## @param nodeAffinityPreset.type Node affinity preset type. Ignored if `affinity` is set. Allowed values: `soft` or `hard` + ## + type: "" + ## @param nodeAffinityPreset.key Node label key to match. Ignored if `affinity` is set. + ## E.g. + ## key: "kubernetes.io/e2e-az-name" + ## + key: "" + ## @param nodeAffinityPreset.values Node label values to match. Ignored if `affinity` is set. + ## E.g. + ## values: + ## - e2e-az1 + ## - e2e-az2 + ## + values: [] + ## Node affinity preset + ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity + ## ## @param ioEngine.containerPorts.http Mayastor I/O Engine HTTP container port ## @param ioEngine.containerPorts.https Mayastor I/O Engine HTTPS container port ## From 14c348d7d4954dc92b9f45f841eda4512ffaafaf Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 11:22:12 +0700 Subject: [PATCH 154/188] Fix ioEngine.priorityClassName --- .../templates/io-engine/DaemonSet.yaml | 6 ++ charts/mayastor/values.yaml | 74 ++++++++++--------- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index a1748646..0f80edd3 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -50,7 +50,13 @@ spec: {{- if .Values.ioEngine.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.nodeSelector "context" $) | nindent 8 }} {{- end }} + {{- if .Values.ioEngine.priorityClassName }} + priorityClassName: {{ .Values.ioEngine.priorityClassName | quote }} + {{- else if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName | quote }} + {{- else }} priorityClassName: {{ printf "%s-critical" (include "mayastor.ioEngine.fullname" .) }} + {{- end }} serviceAccountName: {{ include "mayastor.serviceAccountName" . }} {{- if .Values.ioEngine.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.tolerations "context" .) | nindent 8 }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index bb410dc2..ed578edd 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -1087,24 +1087,49 @@ ioEngine: ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity ## nodeAffinityPreset: - ## @param nodeAffinityPreset.type Node affinity preset type. Ignored if `affinity` is set. Allowed values: `soft` or `hard` + ## @param ioEngine.nodeAffinityPreset.type Node affinity preset type. Ignored if `affinity` is set. Allowed values: `soft` or `hard` ## type: "" - ## @param nodeAffinityPreset.key Node label key to match. Ignored if `affinity` is set. + ## @param ioEngine.nodeAffinityPreset.key Node label key to match. Ignored if `affinity` is set. ## E.g. ## key: "kubernetes.io/e2e-az-name" ## key: "" - ## @param nodeAffinityPreset.values Node label values to match. Ignored if `affinity` is set. + ## @param ioEngine.nodeAffinityPreset.values Node label values to match. Ignored if `affinity` is set. ## E.g. ## values: ## - e2e-az1 ## - e2e-az2 ## values: [] - ## Node affinity preset - ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity + ## @param ioEngine.affinity Affinity for Mayastor I/O Engine pods assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set + ## + affinity: {} + ## @param ioEngine.nodeSelector Node labels for Mayastor I/O Engine pods assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ ## + nodeSelector: + openebs.io/engine: mayastor + kubernetes.io/arch: amd64 + ## @param ioEngine.tolerations Tolerations for Mayastor I/O Engine pods assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @param ioEngine.topologySpreadConstraints Topology Spread Constraints for Mayastor I/O Engine pods assignment + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## E.g. + ## topologySpreadConstraints: + ## - maxSkew: 1 + ## topologyKey: topology.kubernetes.io/zone + ## whenUnsatisfiable: DoNotSchedule + ## + topologySpreadConstraints: {} + ## @param ioEngine.priorityClassName Priority class for Mayastor I/O Engine primary pods assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ + ## + priorityClassName: "" ## @param ioEngine.containerPorts.http Mayastor I/O Engine HTTP container port ## @param ioEngine.containerPorts.https Mayastor I/O Engine HTTPS container port ## @@ -1195,41 +1220,18 @@ ioEngine: containerSecurityContext: enabled: true privileged: true - ## @param affinity Affinity for Mayastor I/O Engine pods assignment - ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity - ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set - ## - affinity: {} - ## @param nodeSelector Node labels for Mayastor I/O Engine pods assignment - ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ - ## - nodeSelector: - openebs.io/engine: mayastor - kubernetes.io/arch: amd64 - ## @param tolerations Tolerations for Mayastor I/O Engine pods assignment - ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - ## - tolerations: [] - ## @param topologySpreadConstraints Topology Spread Constraints for Mayastor I/O Engine pods assignment - ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ - ## E.g. - ## topologySpreadConstraints: - ## - maxSkew: 1 - ## topologyKey: topology.kubernetes.io/zone - ## whenUnsatisfiable: DoNotSchedule - ## - topologySpreadConstraints: {} + operator: enabled: true - ## Mayastor Control Plane MSP Operator image + ## Mayastor CSI Operator image ## ref: https://hub.docker.com/r/mayadata/mcp-msp-operator/tags - ## @param image.registry Mayastor Control Plane MSP Operator image registry - ## @param image.repository Mayastor Control Plane MSP Operator image repository - ## @param image.tag Mayastor Control Plane MSP Operator image tag (immutable tags are recommended) - ## @param image.pullPolicy Mayastor Control Plane MSP Operator image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - ## @param image.debug Specify if debug logs should be enabled + ## @param operator.image.registry Mayastor CSI MSP Operator image registry + ## @param operator.image.repository Mayastor CSI MSP Operator image repository + ## @param operator.image.tag Mayastor CSI MSP Operator image tag (immutable tags are recommended) + ## @param operator.image.pullPolicy Mayastor CSI MSP Operator image pull policy + ## @param operator.image.pullSecrets Specify docker-registry secret names as an array + ## @param operator.image.debug Specify if debug logs should be enabled ## image: registry: docker.io From d17ff0101c9b41173ee203c3ddff4d8b2d78e661 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 11:59:04 +0700 Subject: [PATCH 155/188] Move updateStrategy to values --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 7 +++---- charts/mayastor/values.yaml | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 0f80edd3..12de0e66 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -15,10 +15,9 @@ spec: selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: io-engine - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 + {{- if .Values.ioEngine.updateStrategy }} + updateStrategy: {{- toYaml .Values.ioEngine.updateStrategy | nindent 4 }} + {{- end }} minReadySeconds: 10 template: metadata: diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index ed578edd..27f95a2b 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -1067,6 +1067,8 @@ ioEngine: ## updateStrategy: type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 ## @param ioEngine.podLabels Extra labels for Mayastor pods ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ ## From 3e3a7904d1be67357ced7a58dd8d51a22ca0db08 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:10:58 +0700 Subject: [PATCH 156/188] Change RUST_LOG logic --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 12de0e66..e9010c33 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -81,9 +81,8 @@ spec: image: busybox:latest command: - sh - args: - -c - - |- + - | trap "exit 1" TERM until nc -vzw 5 {{ include "mayastor.agentCore.fullname" . }} {{ .Values.agentCore.service.ports.grpc }} do @@ -143,10 +142,7 @@ spec: {{- end }} - name: RUST_LOG {{- if .Values.diagnosticMode.enabled }} - value: debug - {{- else }} - value: {{ .Values.ioEngine.logLevel }} - {{- end }} + value: {{ ternary "debug" (.Values.ioEngine.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} ports: - name: io-engine containerPort: {{ .Values.ioEngine.containerPorts.ioEngine }} From 3ebc05690b9132fbd393e117263b69f7b46cbe21 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:12:32 +0700 Subject: [PATCH 157/188] Fix RUST_LOG logic --- charts/mayastor/README.md | 2 ++ charts/mayastor/templates/io-engine/DaemonSet.yaml | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/README.md b/charts/mayastor/README.md index e08ff572..eaeff567 100644 --- a/charts/mayastor/README.md +++ b/charts/mayastor/README.md @@ -66,6 +66,8 @@ The command removes all the Kubernetes components associated with the chart and | `commonLabels` | Add labels to all the deployed resources | `{}` | | `commonAnnotations` | Add annotations to all the deployed resources | `{}` | +### I/O Engine Dataplane Deployment + ### Setting Pod's affinity This chart allows you to set your custom affinity using the `affinity` parameter. Find more information about Pod's affinity in the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index e9010c33..278819af 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -141,7 +141,6 @@ spec: value: full {{- end }} - name: RUST_LOG - {{- if .Values.diagnosticMode.enabled }} value: {{ ternary "debug" (.Values.ioEngine.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} ports: - name: io-engine From a3a671f58e504088ce5085dbc536f38e5740dcb0 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:15:14 +0700 Subject: [PATCH 158/188] Fix RUST_BACKTRACE logic --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 278819af..a01c9743 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -135,11 +135,7 @@ spec: - name: NVMF_TCP_MAX_QUEUE_DEPTH value: "32" - name: RUST_BACKTRACE - {{- if .Values.diagnosticMode.enabled }} - value: full - {{- else }} - value: full - {{- end }} + value: {{ ternary "full" "info" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} - name: RUST_LOG value: {{ ternary "debug" (.Values.ioEngine.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} ports: From 71a54cc01ccf58c8625b7dfd307b48e901a90d87 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:16:57 +0700 Subject: [PATCH 159/188] Fix RUST_BACKTRACE and RUST_LOG logic --- charts/mayastor/templates/csi-node/DaemonSet.yaml | 12 ++---------- charts/mayastor/templates/io-engine/DaemonSet.yaml | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/charts/mayastor/templates/csi-node/DaemonSet.yaml b/charts/mayastor/templates/csi-node/DaemonSet.yaml index e2dbee80..674cb939 100644 --- a/charts/mayastor/templates/csi-node/DaemonSet.yaml +++ b/charts/mayastor/templates/csi-node/DaemonSet.yaml @@ -97,17 +97,9 @@ spec: fieldRef: fieldPath: status.podIP - name: RUST_BACKTRACE - {{- if .Values.diagnosticMode.enabled }} - value: full - {{- else }} - value: "1" - {{- end }} + value: {{ ternary "full" "1" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} - name: RUST_LOG - {{- if .Values.diagnosticMode.enabled }} - value: debug - {{- else }} - value: {{ .Values.csiNode.logLevel }} - {{- end }} + value: {{ ternary "debug" (.Values.csiNode.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} ports: - name: grpc containerPort: {{ .Values.csiNode.containerPorts.grpc }} diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index a01c9743..7bef634b 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -135,7 +135,7 @@ spec: - name: NVMF_TCP_MAX_QUEUE_DEPTH value: "32" - name: RUST_BACKTRACE - value: {{ ternary "full" "info" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + value: {{ ternary "full" "1" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} - name: RUST_LOG value: {{ ternary "debug" (.Values.ioEngine.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} ports: From 5e9e63490256515e2204b9233e9a797a353ca523 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:43:56 +0700 Subject: [PATCH 160/188] Add externalEtcd --- charts/mayastor/README.md | 15 +++++++-- charts/mayastor/templates/_helper.tpl | 12 ++++++- .../templates/agent-core/Deployment.yaml | 32 ++++++++----------- .../templates/api-rest/Deployment.yaml | 29 ++++++++++------- .../templates/io-engine/DaemonSet.yaml | 4 +-- charts/mayastor/values.yaml | 5 ++- 6 files changed, 61 insertions(+), 36 deletions(-) diff --git a/charts/mayastor/README.md b/charts/mayastor/README.md index eaeff567..ddd52f3b 100644 --- a/charts/mayastor/README.md +++ b/charts/mayastor/README.md @@ -2,7 +2,7 @@ # Helm chart for Mayastor -Mayastor is a cloud-native declarative data plane written in Rust. Our goal is to abstract storage resources and their differences through the data plane such that users only need to supply the what and do not have to worry about the how so that individual teams stay in control. +Mayastor is a cloud-native declarative data plane written in Rust. The goal is to abstract storage resources and their differences through the data plane such that users only need to supply the what and do not have to worry about the how so that individual teams stay in control. Mayastor also try to be as unopinionated as possible. What this means is that we try to work with the existing storage systems you might already have and unify them as abstract resources instead of swapping them out whenever the resources are local or remote. @@ -59,6 +59,7 @@ The command removes all the Kubernetes components associated with the chart and | Name | Description | Value | | ------------------- | ------------------------------------------------------------------------------------------ | --------------- | | `nameOverride` | String to partially override mayastor.fullname template (will maintain the release name) | `""` | +| `namespaceOverride` | | `""` | | `fullnameOverride` | String to fully override mayastor.fullname template | `""` | | `kubeVersion` | Force target Kubernetes version (using Helm capabilities if not set) | `""` | | `clusterDomain` | Kubernetes Cluster Domain | `cluster.local` | @@ -68,6 +69,16 @@ The command removes all the Kubernetes components associated with the chart and ### I/O Engine Dataplane Deployment +### CSI Node Deployment + +### CSI Controller Deployment + +### Agent Core Deployment + +### API Rest Deployment + +### ETCD Deployment + ### Setting Pod's affinity This chart allows you to set your custom affinity using the `affinity` parameter. Find more information about Pod's affinity in the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). @@ -80,8 +91,6 @@ There are cases where you may want to deploy extra objects, such a ConfigMap con Find more information about how to deal with common errors related to Startechnica's Helm charts in [this troubleshooting guide](https://startechnica.github.io/doc/troubleshoot-helm-chart-issues). -## Upgrading - ## License Copyright © 2022 Startechnica diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index 3fdaf6f8..0e69e972 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -131,4 +131,14 @@ {{- end }} {{- printf "%d" (add $i 1) }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} + +{{/* Return the etcd hostname */}} +{{- define "mayastor.etcd.host" -}} + {{- ternary (include "mayastor.etcd.fullname" .) .Values.externalEtcd.host .Values.etcd.enabled -}} +{{- end -}} + +{{/* Return the etcd client port */}} +{{- define "mayastor.etcd.clientPort" -}} + {{- ternary .Values.etcd.service.ports.client .Values.externalEtcd.port .Values.etcd.enabled | quote -}} +{{- end -}} \ No newline at end of file diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index aadd5535..1c39caac 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -44,19 +44,19 @@ spec: topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} {{- end }} initContainers: - - command: + - name: etcd-probe + image: busybox:latest + command: - sh - -c - - >- - trap "exit 1" TERM; - until nc -vzw 5 {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.port }}; + - | + trap "exit 1" TERM + until nc -vzw 5 {{ include "mayastor.etcd.host" . }} {{ include "mayastor.etcd.port" . }} do - date; - echo "Waiting for etcd..."; - sleep 1; - done; - image: busybox:latest - name: etcd-probe + date + echo "Waiting for etcd..." + sleep 1 + done containers: - name: agent-core image: {{ include "mayastor.agentCore.image" . }} @@ -71,7 +71,7 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} {{- else }} args: - - "--store={{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.port }}" + - "--store={{ include "mayastor.etcd.host" . }}:{{ include "mayastor.etcd.port" . }}" - "--request-timeout=5s" - "--cache-period=30s" - "--grpc-server-addr=https://0.0.0.0:{{ .Values.agentCore.containerPorts.grpc }}" @@ -85,14 +85,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: RUST_LOG - value: {{ .Values.agentCore.logLevel }} - name: RUST_BACKTRACE - {{- if .Values.diagnosticMode.enabled }} - value: full - {{- else }} - value: full - {{- end }} + value: {{ ternary "full" "1" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + - name: RUST_LOG + value: {{ ternary "debug" (.Values.agentCore.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} imagePullPolicy: {{ .Values.agentCore.image.pullPolicy | quote }} ports: - name: grpc diff --git a/charts/mayastor/templates/api-rest/Deployment.yaml b/charts/mayastor/templates/api-rest/Deployment.yaml index 2ad3167a..0e46c656 100644 --- a/charts/mayastor/templates/api-rest/Deployment.yaml +++ b/charts/mayastor/templates/api-rest/Deployment.yaml @@ -44,14 +44,23 @@ spec: topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} {{- end }} initContainers: - - command: + - name: etcd-probe + image: busybox:latest + command: - sh - -c - - trap "exit 1" TERM; until nc -vz {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.port }}; do echo "Waiting for etcd..."; sleep 1; done; - image: busybox:latest - name: etcd-probe + - | + trap "exit 1" TERM + until nc -vzw 5 {{ include "mayastor.etcd.host" . }} {{ include "mayastor.etcd.clientPort" . }} + do + date + echo "Waiting for etcd..." + sleep 1; + done containers: - name: api-rest + image: {{ include "mayastor.apiRest.image" . }} + imagePullPolicy: {{ .Values.apiRest.image.pullPolicy | quote }} resources: limits: cpu: 100m @@ -59,14 +68,17 @@ spec: requests: cpu: 50m memory: 32Mi - image: {{ include "mayastor.apiRest.image" . }} - imagePullPolicy: {{ .Values.apiRest.image.pullPolicy | quote }} args: - "--dummy-certificates" - "--no-auth" - "--http=0.0.0.0:{{ .Values.apiRest.containerPorts.https }}" - "--request-timeout=5s" - "--core-grpc=https://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.ports.grpc }}" + env: + - name: RUST_LOG + value: info + - name: RUST_BACKTRACE + value: full ports: - name: http containerPort: {{ .Values.apiRest.containerPorts.http }} @@ -74,9 +86,4 @@ spec: - name: https containerPort: {{ .Values.apiRest.containerPorts.https }} protocol: TCP - env: - - name: RUST_LOG - value: info - - name: RUST_BACKTRACE - value: full {{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 7bef634b..137097ec 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -71,7 +71,7 @@ spec: - -c - | trap "exit 1" TERM - until nc -vzw 5 {{ include "mayastor.etcd.fullname" . }} {{ .Values.etcd.service.ports.client }} + until nc -vzw 5 {{ include "mayastor.etcd.host" . }} {{ include "mayastor.etcd.port" . }} do date echo "Waiting for etcd..." @@ -117,7 +117,7 @@ spec: {{- if .Values.ioEngine.cpuCount }} - "-l{{ include "mayastor.cpuFlag" . | quote }}" {{- end }} - - "-p{{ include "mayastor.etcd.fullname" . }}:{{ .Values.etcd.service.ports.client }}" + - "-p{{ include "mayastor.etcd.host" . }}:{{ include "mayastor.etcd.clientPort" . }}" - "-Rhttps://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.ports.grpc }}" - "-y/var/local/io-engine/config.yaml" {{- end }} diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 27f95a2b..1fbf4428 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -799,6 +799,10 @@ etcd: client: 2379 peer: 2380 +externalEtcd: + host: "" + port: "" + nats: enabled: false ## Number of replicas @@ -1223,7 +1227,6 @@ ioEngine: enabled: true privileged: true - operator: enabled: true ## Mayastor CSI Operator image From 3609ba69ca4da62754f5c126511c31aaedc4cde1 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:47:32 +0700 Subject: [PATCH 161/188] Fix etcd port --- charts/mayastor/templates/agent-core/Deployment.yaml | 4 ++-- charts/mayastor/templates/io-engine/DaemonSet.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index 1c39caac..3da4b43a 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -51,7 +51,7 @@ spec: - -c - | trap "exit 1" TERM - until nc -vzw 5 {{ include "mayastor.etcd.host" . }} {{ include "mayastor.etcd.port" . }} + until nc -vzw 5 {{ include "mayastor.etcd.host" . }} {{ include "mayastor.etcd.clientPort" . }} do date echo "Waiting for etcd..." @@ -71,7 +71,7 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} {{- else }} args: - - "--store={{ include "mayastor.etcd.host" . }}:{{ include "mayastor.etcd.port" . }}" + - "--store={{ include "mayastor.etcd.host" . }}:{{ include "mayastor.etcd.clientPort" . }}" - "--request-timeout=5s" - "--cache-period=30s" - "--grpc-server-addr=https://0.0.0.0:{{ .Values.agentCore.containerPorts.grpc }}" diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 137097ec..5c41f329 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -71,7 +71,7 @@ spec: - -c - | trap "exit 1" TERM - until nc -vzw 5 {{ include "mayastor.etcd.host" . }} {{ include "mayastor.etcd.port" . }} + until nc -vzw 5 {{ include "mayastor.etcd.host" . }} {{ include "mayastor.etcd.clientPort" . }} do date echo "Waiting for etcd..." From 2af78022a0876ebf6c6ccabdd610f485c88ed220 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:53:55 +0700 Subject: [PATCH 162/188] Fix initContainers command --- charts/mayastor/templates/agent-core/Deployment.yaml | 2 +- charts/mayastor/templates/csi-controller/Deployment.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index 3da4b43a..058a6e2f 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -49,7 +49,7 @@ spec: command: - sh - -c - - | + - |- trap "exit 1" TERM until nc -vzw 5 {{ include "mayastor.etcd.host" . }} {{ include "mayastor.etcd.clientPort" . }} do diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 31caa9d7..92fdfd1e 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -60,6 +60,7 @@ spec: trap "exit 1" TERM until nc -vzw 5 {{ include "mayastor.apiRest.fullname" . }} {{ .Values.apiRest.service.ports.https }} do + date echo "$(date) Waiting for REST API endpoint to become available" sleep 1 done From ced82e840f9fcbe532f54be0418b0a82d7196eb7 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:56:33 +0700 Subject: [PATCH 163/188] Fix agentCore --- charts/mayastor/templates/agent-core/Deployment.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index 058a6e2f..cd1a37df 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -49,14 +49,7 @@ spec: command: - sh - -c - - |- - trap "exit 1" TERM - until nc -vzw 5 {{ include "mayastor.etcd.host" . }} {{ include "mayastor.etcd.clientPort" . }} - do - date - echo "Waiting for etcd..." - sleep 1 - done + - sleep containers: - name: agent-core image: {{ include "mayastor.agentCore.image" . }} From ae7a7fc74b7d46c166a5c2c5d23873b4cfb892d7 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:58:26 +0700 Subject: [PATCH 164/188] Fix: agentCore --- charts/mayastor/templates/agent-core/Deployment.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index cd1a37df..f66b48e2 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -46,10 +46,6 @@ spec: initContainers: - name: etcd-probe image: busybox:latest - command: - - sh - - -c - - sleep containers: - name: agent-core image: {{ include "mayastor.agentCore.image" . }} From 3d028cba9c5b8bae25f6621ecbd806d4a0746c66 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 13:01:30 +0700 Subject: [PATCH 165/188] Commit --- charts/mayastor/templates/agent-core/Deployment.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index f66b48e2..0b830306 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -40,9 +40,6 @@ spec: {{- if .Values.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} {{- end }} - {{- if .Values.topologySpreadConstraints }} - topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} - {{- end }} initContainers: - name: etcd-probe image: busybox:latest From 36349e811710ffc651c42879040e25e7b3a331af Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 13:03:32 +0700 Subject: [PATCH 166/188] Commit --- charts/mayastor/templates/agent-core/Deployment.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index 0b830306..daf63950 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -40,9 +40,6 @@ spec: {{- if .Values.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} {{- end }} - initContainers: - - name: etcd-probe - image: busybox:latest containers: - name: agent-core image: {{ include "mayastor.agentCore.image" . }} From c8523b50ed4ae715e79c2a17379dade2cb02adeb Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 13:06:10 +0700 Subject: [PATCH 167/188] Commit --- charts/mayastor/templates/agent-core/Deployment.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index daf63950..ae61c1f8 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -32,14 +32,6 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: - {{- if .Values.hostAliases }} - hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.hostAliases "context" $) | nindent 8 }} - {{- end }} - {{- include "mayastor.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "mayastor.serviceAccountName" . }} - {{- if .Values.tolerations }} - tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} - {{- end }} containers: - name: agent-core image: {{ include "mayastor.agentCore.image" . }} From 3ad676d7de124bcd474a20093ec9e160132e093f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 13:29:40 +0700 Subject: [PATCH 168/188] Disable agentCore --- charts/mayastor/values-test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 2bbd44bb..65a23009 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -11,6 +11,9 @@ tolerations: key: node-role.kubernetes.io/master operator: Exists +agentCore: + enabled: false + csiNode: tolerations: - effect: NoSchedule From 79dcf2bf90f16cb69ee88df12b0bbe598ab80b5b Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 13:32:57 +0700 Subject: [PATCH 169/188] Fix helper --- charts/mayastor/templates/_helper.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index 0e69e972..bdc7b8f1 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -140,5 +140,5 @@ {{/* Return the etcd client port */}} {{- define "mayastor.etcd.clientPort" -}} - {{- ternary .Values.etcd.service.ports.client .Values.externalEtcd.port .Values.etcd.enabled | quote -}} + {{- ternary .Values.etcd.service.ports.client .Values.externalEtcd.port .Values.etcd.enabled -}} {{- end -}} \ No newline at end of file From 605243611c03de171b3d437e9d557bd3e94fb428 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 13:36:39 +0700 Subject: [PATCH 170/188] Fix agentCore --- .../templates/agent-core/Deployment.yaml | 25 +++++++++++++++++++ charts/mayastor/values-test.yaml | 3 --- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index ae61c1f8..4c021157 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -32,6 +32,31 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} spec: + {{- if .Values.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.hostAliases "context" $) | nindent 8 }} + {{- end }} + {{- include "mayastor.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "mayastor.serviceAccountName" . }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} + {{- end }} + initContainers: + - name: etcd-probe + image: busybox:latest + command: + - sh + - -c + - | + trap "exit 1" TERM + until nc -vzw 5 {{ include "mayastor.etcd.host" . }} {{ include "mayastor.etcd.clientPort" . }} + do + date + echo "Waiting for etcd..." + sleep 1; + done containers: - name: agent-core image: {{ include "mayastor.agentCore.image" . }} diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 65a23009..2bbd44bb 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -11,9 +11,6 @@ tolerations: key: node-role.kubernetes.io/master operator: Exists -agentCore: - enabled: false - csiNode: tolerations: - effect: NoSchedule From ef2bf68e6e5c99fea93bf61995ad2bba2286301d Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 13:46:41 +0700 Subject: [PATCH 171/188] Move log value to values.yaml --- charts/mayastor/templates/_helper.tpl | 5 ++++ .../templates/agent-core/Deployment.yaml | 3 +- .../templates/api-rest/Deployment.yaml | 11 +++---- .../templates/csi-controller/Deployment.yaml | 4 ++- .../templates/io-engine/DaemonSet.yaml | 6 ++-- charts/mayastor/values.yaml | 30 +++++++++++++++++++ 6 files changed, 50 insertions(+), 9 deletions(-) diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index bdc7b8f1..fc57b2c9 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -113,6 +113,11 @@ {{ include "common.images.image" (dict "imageRoot" .Values.operator.image "global" .Values.global) }} {{- end -}} +{{/* Return the proper Mayastor Probe image name */}} +{{- define "mayastor.probe.image" -}} + {{ include "common.images.image" (dict "imageRoot" .Values.probe.image "global" .Values.global) }} +{{- end -}} + {{/* Return the proper image name (for the init container volume-permissions image) */}} {{- define "mayastor.volumePermissions.image" -}} {{ include "common.images.image" (dict "imageRoot" .Values.volumePermissions.image "global" .Values.global) }} diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index 4c021157..2c9d279e 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -45,7 +45,8 @@ spec: {{- end }} initContainers: - name: etcd-probe - image: busybox:latest + image: {{ include "mayastor.probe.image" . }} + imagePullPolicy: {{ .Values.probe.image.pullPolicy }} command: - sh - -c diff --git a/charts/mayastor/templates/api-rest/Deployment.yaml b/charts/mayastor/templates/api-rest/Deployment.yaml index 0e46c656..1259e1a0 100644 --- a/charts/mayastor/templates/api-rest/Deployment.yaml +++ b/charts/mayastor/templates/api-rest/Deployment.yaml @@ -45,7 +45,8 @@ spec: {{- end }} initContainers: - name: etcd-probe - image: busybox:latest + image: {{ include "mayastor.probe.image" . }} + imagePullPolicy: {{ .Values.probe.image.pullPolicy }} command: - sh - -c @@ -55,7 +56,7 @@ spec: do date echo "Waiting for etcd..." - sleep 1; + sleep 1 done containers: - name: api-rest @@ -75,10 +76,10 @@ spec: - "--request-timeout=5s" - "--core-grpc=https://{{ include "mayastor.agentCore.fullname" . }}:{{ .Values.agentCore.service.ports.grpc }}" env: - - name: RUST_LOG - value: info - name: RUST_BACKTRACE - value: full + value: {{ ternary "full" "1" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + - name: RUST_LOG + value: {{ ternary "debug" (.Values.apiRest.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} ports: - name: http containerPort: {{ .Values.apiRest.containerPorts.http }} diff --git a/charts/mayastor/templates/csi-controller/Deployment.yaml b/charts/mayastor/templates/csi-controller/Deployment.yaml index 92fdfd1e..2ff346b4 100644 --- a/charts/mayastor/templates/csi-controller/Deployment.yaml +++ b/charts/mayastor/templates/csi-controller/Deployment.yaml @@ -51,7 +51,7 @@ spec: {{- end }} initContainers: - name: api-rest-probe - image: busybox:latest + image: {{ include "mayastor.probe.image" . }} command: - sh args: @@ -99,6 +99,7 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- if .Values.csiController.attacher.enabled }} - name: csi-attacher image: {{ include "mayastor.csiController.attacher.image" . }} imagePullPolicy: {{ .Values.csiController.image.pullPolicy | quote }} @@ -122,6 +123,7 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- end }} {{- if .Values.csiController.healthMonitor.enabled }} - name: csi-health-monitor image: {{ include "mayastor.csiController.healthMonitor.image" . }} diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 5c41f329..a1b3e352 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -65,7 +65,8 @@ spec: {{- end }} initContainers: - name: etcd-probe - image: busybox:latest + image: {{ include "mayastor.probe.image" . }} + imagePullPolicy: {{ .Values.probe.image.pullPolicy }} command: - sh - -c @@ -78,7 +79,8 @@ spec: sleep 1 done - name: agent-core-probe - image: busybox:latest + image: {{ include "mayastor.probe.image" . }} + imagePullPolicy: {{ .Values.probe.image.pullPolicy }} command: - sh - -c diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 1fbf4428..ade8ffb4 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -386,6 +386,7 @@ csiController: ## ref: https://github.com/kubernetes-csi/external-attacher ## attacher: + enabled: true image: registry: k8s.gcr.io repository: sig-storage/csi-attacher @@ -943,6 +944,7 @@ agentCore: apiRest: enabled: true + logLevel: info ## Mayastor Control Plane Rest image ## ref: https://hub.docker.com/r/mayadata/mcp-rest/tags ## @param image.registry Mayastor Control Plane Rest image registry @@ -1255,6 +1257,34 @@ operator: ## pullSecrets: [] +probe: + enabled: true + ## Mayastor CSI Operator image + ## ref: https://hub.docker.com/r/mayadata/mcp-msp-operator/tags + ## @param operator.image.registry Mayastor CSI MSP Operator image registry + ## @param operator.image.repository Mayastor CSI MSP Operator image repository + ## @param operator.image.tag Mayastor CSI MSP Operator image tag (immutable tags are recommended) + ## @param operator.image.pullPolicy Mayastor CSI MSP Operator image pull policy + ## @param operator.image.pullSecrets Specify docker-registry secret names as an array + ## @param operator.image.debug Specify if debug logs should be enabled + ## + image: + registry: docker.io + repository: busybox + tag: latest + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param diskPools.create ## diskPools: From 3cafb6fecb1b4b3aa09f7761d8b8418656b4eefe Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 14:09:45 +0700 Subject: [PATCH 172/188] Move operator and probe resources to values.yaml --- .../templates/io-engine/DaemonSet.yaml | 3 +- .../templates/operator/Deployment.yaml | 39 ++++++++++++------- charts/mayastor/values.yaml | 30 ++++++++++++++ 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index a1b3e352..51254c8b 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -114,7 +114,6 @@ spec: # For example -l 1,2,10-20 means use core 1, 2, 10 to 20. # Note: Ensure that the CPU resources are updated accordingly. # If you use 2 CPUs, the CPU: field should also read 2. - - "-N$(MY_NODE_NAME)" - "-g$(MY_POD_IP)" {{- if .Values.ioEngine.cpuCount }} - "-l{{ include "mayastor.cpuFlag" . | quote }}" @@ -139,7 +138,7 @@ spec: - name: RUST_BACKTRACE value: {{ ternary "full" "1" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} - name: RUST_LOG - value: {{ ternary "debug" (.Values.ioEngine.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + value: info,io_engine={{ ternary "debug" (.Values.ioEngine.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} ports: - name: io-engine containerPort: {{ .Values.ioEngine.containerPorts.ioEngine }} diff --git a/charts/mayastor/templates/operator/Deployment.yaml b/charts/mayastor/templates/operator/Deployment.yaml index b14d4b79..d4acb78f 100644 --- a/charts/mayastor/templates/operator/Deployment.yaml +++ b/charts/mayastor/templates/operator/Deployment.yaml @@ -44,33 +44,44 @@ spec: topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} {{- end }} initContainers: - - command: + - name: etcd-probe + image: busybox:latest + command: - sh - -c - - trap "exit 1" TERM; until nc -vzw 5 mayastor-etcd 2379; do date; echo "Waiting for etcd..."; sleep 1; done; - image: busybox:latest - name: etcd-probe + - | + trap "exit 1" TERM + until nc -vzw 5 mayastor-etcd 2379 + do + date + echo "Waiting for etcd..." + sleep 1 + done + {{- if .Values.probe.resources }} + resources: {{ toYaml .Values.probe.resources | nindent 12 }} + {{- end }} containers: - name: operator image: {{ include "mayastor.operator.image" . }} imagePullPolicy: {{ .Values.operator.image.pullPolicy | quote }} args: - "-e http://{{ include "mayastor.apiRest.fullname" . }}:{{ .Values.apiRest.containerPorts.https }}" + - "-n={{ include "common.names.namespace" . }}" + - "--request-timeout={{ .Values.operator.requestTimeout }}" - "--interval=30s" + {{ if .Values.jaeger.enabled }} + - "--jaeger={{ .Values.jaeger.agent.name }}:{{ .Values.jaeger.agent.port }}" + {{ end }} env: - name: MY_POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - - name: RUST_LOG - value: info,msp_operator=info - name: RUST_BACKTRACE - value: full - resources: - limits: - cpu: 100m - memory: 32Mi - requests: - cpu: 50m - memory: 16Mi + value: {{ ternary "full" "1" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + - name: RUST_LOG + value: info,operator_diskpool={{ ternary "debug" (.Values.operator.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + {{- if .Values.operator.resources }} + resources: {{ toYaml .Values.operator.resources | nindent 12 }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index ade8ffb4..2cd4393e 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -804,6 +804,9 @@ externalEtcd: host: "" port: "" +jaeger: + enabled: false + nats: enabled: false ## Number of replicas @@ -1231,6 +1234,7 @@ ioEngine: operator: enabled: true + logLevel: info ## Mayastor CSI Operator image ## ref: https://hub.docker.com/r/mayadata/mcp-msp-operator/tags ## @param operator.image.registry Mayastor CSI MSP Operator image registry @@ -1256,6 +1260,20 @@ operator: ## - myRegistryKeySecretName ## pullSecrets: [] + ## Mayastor Operator resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## @param operator.resources.limits The resources limits for Mayastor Operator containers + ## @param operator.resources.requests The requested resources for Mayastor Operator containers + ## + resources: + limits: + cpu: 100m + memory: 32Mi + requests: + cpu: 50m + memory: 16Mi + + requestTimeout: 5s probe: enabled: true @@ -1284,6 +1302,18 @@ probe: ## - myRegistryKeySecretName ## pullSecrets: [] + ## Mayastor Probe resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## @param probe.resources.limits The resources limits for Mayastor Probe containers + ## @param probe.resources.requests The requested resources for Mayastor Probe containers + ## + resources: + limits: + cpu: 100m + memory: 32Mi + requests: + cpu: 50m + memory: 16Mi ## @param diskPools.create ## From aa827c7c2c753442a16b9b3ef01803b3ef9363b3 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 14:58:58 +0700 Subject: [PATCH 173/188] Check hugePages --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 51254c8b..5c661a9d 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -131,6 +131,11 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: REQUESTS_HUGEPAGES_1GI + valueFrom: + resourceFieldRef: + containerName: io-engine + resource: requests.hugepages-2Mi - name: NVME_QPAIR_CONNECT_ASYNC value: "true" - name: NVMF_TCP_MAX_QUEUE_DEPTH From 8756a68850676b11a48eaf066b61ed522d8a0d8e Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 14:59:11 +0700 Subject: [PATCH 174/188] Check hugepages-2Mi --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 5c661a9d..9d56ebce 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -131,7 +131,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - - name: REQUESTS_HUGEPAGES_1GI + - name: REQUESTS_HUGEPAGES_2Mi valueFrom: resourceFieldRef: containerName: io-engine From 296ab952fbe3fe8d3536321687a29b1e0ec2d6a2 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 14:59:23 +0700 Subject: [PATCH 175/188] Check hugepages-2Mi --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 9d56ebce..0320d16a 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -131,7 +131,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - - name: REQUESTS_HUGEPAGES_2Mi + - name: REQUESTS_HUGEPAGES_2MI valueFrom: resourceFieldRef: containerName: io-engine From 1de444a619f712f1f08aec1974951036c9acb8c5 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 15:04:31 +0700 Subject: [PATCH 176/188] Trace podInfo hugePages --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 0320d16a..8525f3e6 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -163,6 +163,8 @@ spec: mountPath: /dev/shm - name: configlocation mountPath: /var/local/io-engine/ + - mountPath: /opt/podinfo + name: podinfo {{- if .Values.metrics.enabled }} - name: metrics-exporter-pool image: {{ .Values.metrics.image }} @@ -217,4 +219,12 @@ spec: - name: configlocation hostPath: path: /var/local/io-engine/ - type: DirectoryOrCreate \ No newline at end of file + type: DirectoryOrCreate + - name: podinfo + downwardAPI: + items: + - path: "hugepages_2Mi_request" + resourceFieldRef: + containerName: io-engine + resource: requests.hugepages-2Mi + divisor: 2Mi \ No newline at end of file From a818dacc221c15358a03554106276be33d51a732 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 15:06:08 +0700 Subject: [PATCH 177/188] Commit --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 8525f3e6..e2d2db84 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -227,4 +227,4 @@ spec: resourceFieldRef: containerName: io-engine resource: requests.hugepages-2Mi - divisor: 2Mi \ No newline at end of file + divisor: 1Mi \ No newline at end of file From 01a308cd2fe021e793c4b57cb02de6be57d10c40 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 15:06:25 +0700 Subject: [PATCH 178/188] Commit --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index e2d2db84..1bd1d13f 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -163,7 +163,7 @@ spec: mountPath: /dev/shm - name: configlocation mountPath: /var/local/io-engine/ - - mountPath: /opt/podinfo + - mountPath: /etc/podinfo name: podinfo {{- if .Values.metrics.enabled }} - name: metrics-exporter-pool From f963a15eaacd5266a14065f5b66c34c607d15121 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 16:06:48 +0700 Subject: [PATCH 179/188] Add debug env --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 1bd1d13f..51bb467c 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -91,7 +91,14 @@ spec: date echo "Waiting for agent-core-grpc services..." sleep 1 + echo "Huepages: $(REQUESTS_HUGEPAGES_2MI)"" done + env: + - name: REQUESTS_HUGEPAGES_2MI + valueFrom: + resourceFieldRef: + containerName: io-engine + resource: requests.hugepages-2Mi containers: - name: io-engine image: {{ include "mayastor.ioEngine.image" . }} @@ -131,11 +138,6 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - - name: REQUESTS_HUGEPAGES_2MI - valueFrom: - resourceFieldRef: - containerName: io-engine - resource: requests.hugepages-2Mi - name: NVME_QPAIR_CONNECT_ASYNC value: "true" - name: NVMF_TCP_MAX_QUEUE_DEPTH From 72a0f4471c27caf36c72864fd502a5fbf250f472 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 16:06:59 +0700 Subject: [PATCH 180/188] Add debug parameter --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 51bb467c..d89c46a6 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -145,7 +145,7 @@ spec: - name: RUST_BACKTRACE value: {{ ternary "full" "1" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} - name: RUST_LOG - value: info,io_engine={{ ternary "debug" (.Values.ioEngine.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + value: debug,io_engine={{ ternary "debug" (.Values.ioEngine.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} ports: - name: io-engine containerPort: {{ .Values.ioEngine.containerPorts.ioEngine }} From 23cc33442b37411c902330a3c43290cdc9c616a0 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 16:31:59 +0700 Subject: [PATCH 181/188] Add Jaeger --- charts/mayastor/Chart.yaml | 9 ++++++++- charts/mayastor/templates/_helper.tpl | 4 ++++ .../templates/agent-core/Deployment.yaml | 5 ++++- .../templates/io-engine/DaemonSet.yaml | 2 +- .../templates/jaeger-operator/Jaeger.yaml | 20 +++++++++++++++++++ .../templates/operator/Deployment.yaml | 2 +- charts/mayastor/values.yaml | 5 +++++ 7 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 charts/mayastor/templates/jaeger-operator/Jaeger.yaml diff --git a/charts/mayastor/Chart.yaml b/charts/mayastor/Chart.yaml index f8f49e16..0daccc3d 100644 --- a/charts/mayastor/Chart.yaml +++ b/charts/mayastor/Chart.yaml @@ -13,8 +13,15 @@ dependencies: repository: https://charts.bitnami.com/bitnami tags: - bitnami-etcd - # version: 6.x.x version: 8.x.x + - name: jaeger-operator + condition: jaeger.enabled + repository: https://jaegertracing.github.io/helm-charts + version: 2.32.2 + - name: loki-stack + condition: lokiStack.enabled + version: 2.6.5 + repository: https://grafana.github.io/helm-charts description: Mayastor is a cloud-native declarative data plane written in Rust. Our goal is to abstract storage resources and their differences through the data plane such that users only need to supply the what and do not have to worry about diff --git a/charts/mayastor/templates/_helper.tpl b/charts/mayastor/templates/_helper.tpl index fc57b2c9..81345916 100644 --- a/charts/mayastor/templates/_helper.tpl +++ b/charts/mayastor/templates/_helper.tpl @@ -19,6 +19,10 @@ {{- printf "%s-etcd" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} {{- end -}} +{{- define "mayastor.jaeger.fullname" -}} + {{- printf "%s-jaeger" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end -}} + {{- define "mayastor.ioEngine.fullname" -}} {{- printf "%s-io-engine" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} {{- end -}} diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index 2c9d279e..9e2886b6 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -73,8 +73,11 @@ spec: {{- else }} args: - "--store={{ include "mayastor.etcd.host" . }}:{{ include "mayastor.etcd.clientPort" . }}" - - "--request-timeout=5s" + - "--request-timeout={{ .Values.agentCore.requestTimeout | default "5s" }}" - "--cache-period=30s" + {{ if .Values.jaeger.enabled }} + - "--jaeger={{ .Values.jaeger.agent.name }}:{{ .Values.jaeger.agent.port }}" + {{ end }} - "--grpc-server-addr=https://0.0.0.0:{{ .Values.agentCore.containerPorts.grpc }}" {{- end }} env: diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index d89c46a6..0539d15f 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -91,7 +91,7 @@ spec: date echo "Waiting for agent-core-grpc services..." sleep 1 - echo "Huepages: $(REQUESTS_HUGEPAGES_2MI)"" + echo "Huepages: $(REQUESTS_HUGEPAGES_2MI)" done env: - name: REQUESTS_HUGEPAGES_2MI diff --git a/charts/mayastor/templates/jaeger-operator/Jaeger.yaml b/charts/mayastor/templates/jaeger-operator/Jaeger.yaml new file mode 100644 index 00000000..d530bb39 --- /dev/null +++ b/charts/mayastor/templates/jaeger-operator/Jaeger.yaml @@ -0,0 +1,20 @@ +{{- if .Values.jaeger.enabled }} +apiVersion: jaegertracing.io/v1 +kind: Jaeger +metadata: + name: {{ include "mayastor.jaeger.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} +spec: + strategy: allInOne + ingress: + enabled: false + {{- include "jaeger_scheduling" . }} + query: + serviceType: NodePort + nodePort: 30012 + storage: + type: memory + options: + memory: + max-traces: 100000 +{{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/operator/Deployment.yaml b/charts/mayastor/templates/operator/Deployment.yaml index d4acb78f..f9bc1b45 100644 --- a/charts/mayastor/templates/operator/Deployment.yaml +++ b/charts/mayastor/templates/operator/Deployment.yaml @@ -67,7 +67,7 @@ spec: args: - "-e http://{{ include "mayastor.apiRest.fullname" . }}:{{ .Values.apiRest.containerPorts.https }}" - "-n={{ include "common.names.namespace" . }}" - - "--request-timeout={{ .Values.operator.requestTimeout }}" + - "--request-timeout={{ .Values.operator.requestTimeout | default "5s" }}" - "--interval=30s" {{ if .Values.jaeger.enabled }} - "--jaeger={{ .Values.jaeger.agent.name }}:{{ .Values.jaeger.agent.port }}" diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 2cd4393e..ed45f108 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -807,6 +807,9 @@ externalEtcd: jaeger: enabled: false +lokiStack: + enabled: false + nats: enabled: false ## Number of replicas @@ -1043,6 +1046,8 @@ apiRest: ## timeoutSeconds: 300 sessionAffinityConfig: {} + requestTimeout: 5s + ioEngine: logLevel: info cpuCount: "" From 28460c8023fafd202a98e244ba85dd245dfc5bd5 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 16:43:03 +0700 Subject: [PATCH 182/188] Add generateName to DiskPool --- charts/mayastor/templates/DiskPool.yaml | 4 ++++ charts/mayastor/values-test.yaml | 6 ------ charts/mayastor/values.yaml | 3 +++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/charts/mayastor/templates/DiskPool.yaml b/charts/mayastor/templates/DiskPool.yaml index b4222c5f..5d1e9eb9 100644 --- a/charts/mayastor/templates/DiskPool.yaml +++ b/charts/mayastor/templates/DiskPool.yaml @@ -5,7 +5,11 @@ apiVersion: "openebs.io/v1alpha1" kind: DiskPool metadata: + {{- if not (empty $.Values.diskPools.generateName) }} + generateName: {{ $.Values.diskPools.generateName }} + {{- else }} name: pool-on-{{ .node }} + {{- end }} namespace: {{ include "common.names.namespace" $ | quote }} spec: node: {{ .node }} diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 2bbd44bb..4eff4f65 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -22,12 +22,6 @@ csiNode: etcd: clusterDomain: nebula.mci.net.id - replicaCount: 1 - revisionHistoryLimit: 2 - auth: - rbac: - allowNoneAuthentication: true - rootPassword: password persistence: enabled: true storageClass: openebs-hostpath diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index ed45f108..72abaff8 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -1324,6 +1324,9 @@ probe: ## diskPools: create: false + ## @param diskPools.generateName Let Kubernetes generate pool name + ## + generateName: "" ## @param diskPools.pools Custom RBAC rules ## Example: ## pools: From 58a2e3b471b208d7de743a15860342e2af75556f Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 16:45:12 +0700 Subject: [PATCH 183/188] Move agent-core resources to values.yaml --- charts/mayastor/templates/agent-core/Deployment.yaml | 12 ++++-------- charts/mayastor/values.yaml | 12 ++++++++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/charts/mayastor/templates/agent-core/Deployment.yaml b/charts/mayastor/templates/agent-core/Deployment.yaml index 9e2886b6..a53ec66a 100644 --- a/charts/mayastor/templates/agent-core/Deployment.yaml +++ b/charts/mayastor/templates/agent-core/Deployment.yaml @@ -61,6 +61,7 @@ spec: containers: - name: agent-core image: {{ include "mayastor.agentCore.image" . }} + imagePullPolicy: {{ .Values.agentCore.image.pullPolicy | quote }} {{- if .Values.diagnosticMode.enabled }} command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} {{- else if .Values.command }} @@ -93,16 +94,11 @@ spec: value: {{ ternary "full" "1" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} - name: RUST_LOG value: {{ ternary "debug" (.Values.agentCore.logLevel) (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} - imagePullPolicy: {{ .Values.agentCore.image.pullPolicy | quote }} ports: - name: grpc containerPort: {{ .Values.agentCore.containerPorts.grpc }} protocol: TCP - resources: - limits: - cpu: 1000m - memory: 32Mi - requests: - cpu: 500m - memory: 16Mi + {{- if .Values.agentCore.resources }} + resources: {{ toYaml .Values.agentCore.resources | nindent 12 }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 72abaff8..4ab1e0b4 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -890,6 +890,18 @@ agentCore: ## containerPorts: grpc: 50051 + ## Mayastor Agent Core resource requests and limits + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## @param agentCore.resources.limits The resources limits for Mayastor Agent Core containers + ## @param agentCore.resources.requests The requested resources for Mayastor Agent Core containers + ## + resources: + limits: + cpu: 1000m + memory: 32Mi + requests: + cpu: 500m + memory: 16Mi service: ## @param service.type Mayastor Control Plane Rest Kubernetes service type ## From 2704b90eac91a159c939abab11e448ec01be684c Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 16:47:24 +0700 Subject: [PATCH 184/188] Move api-rest resources to values.yaml --- charts/mayastor/templates/api-rest/Deployment.yaml | 10 +++------- charts/mayastor/values.yaml | 12 ++++++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/charts/mayastor/templates/api-rest/Deployment.yaml b/charts/mayastor/templates/api-rest/Deployment.yaml index 1259e1a0..9d8f16a8 100644 --- a/charts/mayastor/templates/api-rest/Deployment.yaml +++ b/charts/mayastor/templates/api-rest/Deployment.yaml @@ -62,13 +62,9 @@ spec: - name: api-rest image: {{ include "mayastor.apiRest.image" . }} imagePullPolicy: {{ .Values.apiRest.image.pullPolicy | quote }} - resources: - limits: - cpu: 100m - memory: 64Mi - requests: - cpu: 50m - memory: 32Mi + {{- if .Values.apiRest.resources }} + resources: {{ toYaml .Values.apiRest.resources | nindent 12 }} + {{- end }} args: - "--dummy-certificates" - "--no-auth" diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index 4ab1e0b4..c8098d5e 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -994,6 +994,18 @@ apiRest: containerPorts: http: 8080 https: 8081 + ## Mayastor API Rest resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## @param apiRest.resources.limits The resources limits for the Mayastor API Rest containers + ## @param apiRest.resources.requests The requested resources for the Mayastor API Rest containers + ## + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 50m + memory: 32Mi tls: enabled: true ## Mayastor Control Plane Rest service parameters From cf761660b1233871ab3581fab1ea9eb245c2efd1 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 16:54:25 +0700 Subject: [PATCH 185/188] Add ioEngine debug --- charts/mayastor/templates/io-engine/DaemonSet.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 0539d15f..42e466ea 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -85,13 +85,14 @@ spec: - sh - -c - | + echo "Huepages: $(REQUESTS_HUGEPAGES_2MI)" + cat /etc/podinfo/hugepages_2Mi_request trap "exit 1" TERM until nc -vzw 5 {{ include "mayastor.agentCore.fullname" . }} {{ .Values.agentCore.service.ports.grpc }} do date echo "Waiting for agent-core-grpc services..." sleep 1 - echo "Huepages: $(REQUESTS_HUGEPAGES_2MI)" done env: - name: REQUESTS_HUGEPAGES_2MI @@ -99,6 +100,9 @@ spec: resourceFieldRef: containerName: io-engine resource: requests.hugepages-2Mi + volumeMounts: + - mountPath: /etc/podinfo + name: podinfo containers: - name: io-engine image: {{ include "mayastor.ioEngine.image" . }} From 6e5fbbe5b107dde208d0d05f3a836f7623136471 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 17:13:56 +0700 Subject: [PATCH 186/188] Add ioEngine sidecars values --- .../templates/io-engine/DaemonSet.yaml | 73 +++++++++---------- charts/mayastor/values-test.yaml | 24 ++++++ charts/mayastor/values.yaml | 14 ++++ 3 files changed, 73 insertions(+), 38 deletions(-) diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 42e466ea..448e7b36 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -85,8 +85,6 @@ spec: - sh - -c - | - echo "Huepages: $(REQUESTS_HUGEPAGES_2MI)" - cat /etc/podinfo/hugepages_2Mi_request trap "exit 1" TERM until nc -vzw 5 {{ include "mayastor.agentCore.fullname" . }} {{ .Values.agentCore.service.ports.grpc }} do @@ -94,15 +92,12 @@ spec: echo "Waiting for agent-core-grpc services..." sleep 1 done - env: - - name: REQUESTS_HUGEPAGES_2MI - valueFrom: - resourceFieldRef: - containerName: io-engine - resource: requests.hugepages-2Mi - volumeMounts: - - mountPath: /etc/podinfo - name: podinfo + {{- if .Values.initContainers }} + {{- include "common.tplvalues.render" (dict "value" .Values.initContainers "context" $) | trim | nindent 8 }} + {{- end }} + {{- if .Values.ioEngine.initContainers }} + {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.initContainers "context" $) | trim | nindent 8 }} + {{- end }} containers: - name: io-engine image: {{ include "mayastor.ioEngine.image" . }} @@ -171,6 +166,9 @@ spec: mountPath: /var/local/io-engine/ - mountPath: /etc/podinfo name: podinfo + {{- if .Values.ioEngine.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} {{- if .Values.metrics.enabled }} - name: metrics-exporter-pool image: {{ .Values.metrics.image }} @@ -206,31 +204,30 @@ spec: resources: {{ toYaml .Values.metrics.resources | nindent 12 }} {{- end }} {{- end }} + {{- if .Values.ioEngine.sidecars }} + {{- include "common.tplvalues.render" ( dict "value" .Values.ioEngine.sidecars "context" $) | nindent 8 }} + {{- end }} volumes: - - name: device - hostPath: - path: /dev - type: Directory - - name: udev - hostPath: - path: /run/udev - type: Directory - - name: dshm - emptyDir: - medium: Memory - sizeLimit: "1Gi" - - name: hugepage - emptyDir: - medium: HugePages - - name: configlocation - hostPath: - path: /var/local/io-engine/ - type: DirectoryOrCreate - - name: podinfo - downwardAPI: - items: - - path: "hugepages_2Mi_request" - resourceFieldRef: - containerName: io-engine - resource: requests.hugepages-2Mi - divisor: 1Mi \ No newline at end of file + - name: device + hostPath: + path: /dev + type: Directory + - name: udev + hostPath: + path: /run/udev + type: Directory + - name: dshm + emptyDir: + medium: Memory + sizeLimit: "1Gi" + - name: hugepage + emptyDir: + medium: HugePages + - name: configlocation + hostPath: + path: /var/local/io-engine/ + type: DirectoryOrCreate + + {{- if .Values.ioEngine.extraVolumes }} + {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.extraVolumes "context" $) | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index 4eff4f65..de308346 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -49,3 +49,27 @@ ioEngine: - effect: NoSchedule key: node-role.kubernetes.io/master operator: Exists + extraVolumes: + - name: podinfo + downwardAPI: + items: + - path: "hugepages_2Mi_request" + resourceFieldRef: + containerName: io-engine + resource: requests.hugepages-2Mi + divisor: 1Mi + sidecars: + - name: debug-probe + image: busybox:latest + command: + - sleep + - inf + env: + - name: REQUESTS_HUGEPAGES_2MI + valueFrom: + resourceFieldRef: + containerName: io-engine + resource: requests.hugepages-2Mi + volumeMounts: + - mountPath: /etc/podinfo + name: podinfo \ No newline at end of file diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index c8098d5e..edb0ea64 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -1072,6 +1072,8 @@ apiRest: requestTimeout: 5s +## @section Mayastor I/O Engine Dataplane parameters +## ioEngine: logLevel: info cpuCount: "" @@ -1260,6 +1262,18 @@ ioEngine: containerSecurityContext: enabled: true privileged: true + ## @param ioEngine.extraVolumes Optionally specify extra list of additional volumes to the FreeRADIUS pod(s) + ## + extraVolumes: [] + ## @param ioEngine.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the FreeRADIUS container(s) + ## + extraVolumeMounts: [] + ## @param ioEngine.initContainers Add additional init containers for the FreeRADIUS pod(s) + ## + initContainers: [] + ## @param ioEngine.sidecars Add additional sidecar containers for the FreeRADIUS pod(s) + ## + sidecars: [] operator: enabled: true From 51fa66a569ab23c7423aa94faa8dc0e9436bf0ee Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 17:25:53 +0700 Subject: [PATCH 187/188] Fix csiNode RBAC --- charts/mayastor/templates/csi-node/ClusterRole.yaml | 13 +++++++++++-- charts/mayastor/templates/io-engine/DaemonSet.yaml | 4 +++- charts/mayastor/values-test.yaml | 9 +++++++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/charts/mayastor/templates/csi-node/ClusterRole.yaml b/charts/mayastor/templates/csi-node/ClusterRole.yaml index 9ffedaa9..e8113583 100644 --- a/charts/mayastor/templates/csi-node/ClusterRole.yaml +++ b/charts/mayastor/templates/csi-node/ClusterRole.yaml @@ -1,10 +1,15 @@ -# Source: mayastor-control-plane/templates/operator-rbac.yaml - {{- if and .Values.csiNode.serviceAccount.create .Values.csiNode.rbac.create }} apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} kind: ClusterRole metadata: name: {{ include "mayastor.csiNode.fullname" . }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} rules: # must create mayastor crd if it doesn't exist - apiGroups: ["apiextensions.k8s.io"] @@ -69,4 +74,8 @@ rules: - apiGroups: ["monitoring.coreos.com"] resources: ["servicemonitors"] verbs: ["get", "create"] + + {{- if .Values.csiNode.rbac.rules }} + {{- include "common.tplvalues.render" ( dict "value" .Values.csiNode.rbac.rules "context" $ ) | nindent 2 }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/mayastor/templates/io-engine/DaemonSet.yaml b/charts/mayastor/templates/io-engine/DaemonSet.yaml index 448e7b36..6d3c0664 100644 --- a/charts/mayastor/templates/io-engine/DaemonSet.yaml +++ b/charts/mayastor/templates/io-engine/DaemonSet.yaml @@ -166,6 +166,9 @@ spec: mountPath: /var/local/io-engine/ - mountPath: /etc/podinfo name: podinfo + {{- if .Values.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" .Values.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} {{- if .Values.ioEngine.extraVolumeMounts }} {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.extraVolumeMounts "context" $) | nindent 12 }} {{- end }} @@ -227,7 +230,6 @@ spec: hostPath: path: /var/local/io-engine/ type: DirectoryOrCreate - {{- if .Values.ioEngine.extraVolumes }} {{- include "common.tplvalues.render" (dict "value" .Values.ioEngine.extraVolumes "context" $) | nindent 8 }} {{- end }} \ No newline at end of file diff --git a/charts/mayastor/values-test.yaml b/charts/mayastor/values-test.yaml index de308346..1fb2e895 100644 --- a/charts/mayastor/values-test.yaml +++ b/charts/mayastor/values-test.yaml @@ -62,8 +62,13 @@ ioEngine: - name: debug-probe image: busybox:latest command: - - sleep - - inf + - sh + - -c + - | + echo "$(REQUESTS_HUGEPAGES_2MI)" + ls -liah /etc/podinfo + cat /etc/podinfo/hugepages_2Mi_request + sleep inf env: - name: REQUESTS_HUGEPAGES_2MI valueFrom: From b3ca3b032fcc338404cb3f121a25c0c0fe8b0ca6 Mon Sep 17 00:00:00 2001 From: firmansyahn <80936459+firmansyahn@users.noreply.github.com> Date: Mon, 20 Jun 2022 18:34:09 +0700 Subject: [PATCH 188/188] [mayastor] Release v0.1.0 --- charts/mayastor/values.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/mayastor/values.yaml b/charts/mayastor/values.yaml index edb0ea64..6074162a 100644 --- a/charts/mayastor/values.yaml +++ b/charts/mayastor/values.yaml @@ -525,8 +525,8 @@ csiNode: privileged: true ## Mayastor CSI Plugin container's resource requests and limits ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ - ## @param primary.resources.limits The resources limits for MariaDB primary containers - ## @param primary.resources.requests The requested resources for MariaDB primary containers + ## @param csiNode..resources.limits The resources limits for MariaDB primary containers + ## @param csiNode..resources.requests The requested resources for MariaDB primary containers ## resources: limits: @@ -535,20 +535,20 @@ csiNode: requests: cpu: 100m memory: 64Mi - ## @param affinity Affinity for Mayastor I/O Engine pods assignment + ## @param csiNode.affinity Affinity for CSI Plugin Engine pods assignment ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set ## affinity: {} - ## @param nodeSelector Node labels for Mayastor I/O Engine pods assignment + ## @param csiNode.nodeSelector Node labels for CSI Plugin Engine pods assignment ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ ## nodeSelector: {} - ## @param tolerations Tolerations for Mayastor I/O Engine pods assignment + ## @param csiNode.tolerations Tolerations for CSI Plugin Engine pods assignment ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## tolerations: [] - ## @param topologySpreadConstraints Topology Spread Constraints for Mayastor I/O Engine pods assignment + ## @param csiNode.topologySpreadConstraints Topology Spread Constraints for CSI Plugin Engine pods assignment ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ ## E.g. ## topologySpreadConstraints: