diff --git a/README.md b/README.md index 36a956c..c48c174 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,34 @@ The idea here is to use a single file as the block device, using Linux’s loop, - The size limit is enforced by the operating system, based on the backing file size. - Since volumes are backed by different files, each file could be formatted using different filesystems, and/or customized with different filesystem options. +### Why use Helm hooks to install/uninstall + +Storage classes are one of the foundation building blocks for setting up a solution, so they need to be in place before a solution is installed and only be removed after a solution is uninstalled. This means that the Helm chart needs to be installed and removed seperatly from other Helm charts. One way to allow us to use this Helm chart as part of an umbrella Helm chart is to use Helm Hooks. + +In order to ensure that we have a fully functioning storage class before creating other resources we can leverage Helm hooks weight. This also allows us to specify the order of resource creation otherwise Helm views custom resources as a single bucket and simply relies on file name order. + +Its also important to ensure that we don't remove resources that are necessary to run storage class before we have removed the solution, as Helm will be left in a deadlock situation waiting for PVCs to be deleted which will never be deleted. + +### Recommended locations for data directory + +Docker +--- +Recommended location: +``` +dataDir: /var/lib/csi/rawfile +``` + +To view locations on Docker +``` +docker run -it --rm --net=host --ipc=host --uts=host --pid=host --security-opt=seccomp=unconfined --privileged -v /:/host alpine /bin/ash -c "df -Th" +``` + +CRC +--- +Recommended location: +``` +dataDir: /var/lib/csi/rawfile +``` ## License [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fopenebs%2Frawfile-localpv.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fopenebs%2Frawfile-localpv?ref=badge_large) diff --git a/consts.py b/consts.py index ac9b857..49c5981 100644 --- a/consts.py +++ b/consts.py @@ -1,7 +1,7 @@ import os PROVISIONER_NAME = os.getenv("PROVISIONER_NAME", "rawfile.csi.openebs.io") -PROVISIONER_VERSION = "0.7.0" +PROVISIONER_VERSION = "0.7.1" DATA_DIR = "/data" CONFIG = {} RESOURCE_EXHAUSTED_EXIT_CODE = 101 diff --git a/deploy/charts/rawfile-csi/Chart.yaml b/deploy/charts/rawfile-csi/Chart.yaml index 1eca783..b441d31 100644 --- a/deploy/charts/rawfile-csi/Chart.yaml +++ b/deploy/charts/rawfile-csi/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: rawfile-csi description: RawFile Driver Container Storage Interface type: application -version: 0.7.0 -appVersion: 0.7.0 +version: 0.7.1 +appVersion: 0.7.1 diff --git a/deploy/charts/rawfile-csi/templates/00-driver.yaml b/deploy/charts/rawfile-csi/templates/00-driver.yaml deleted file mode 100644 index 0e1d18c..0000000 --- a/deploy/charts/rawfile-csi/templates/00-driver.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: {{ .Values.provisionerName }} -spec: - attachRequired: false - podInfoOnMount: true - fsGroupPolicy: File - storageCapacity: true - volumeLifecycleModes: - - Persistent diff --git a/deploy/charts/rawfile-csi/templates/00-security-context-constraints.yaml b/deploy/charts/rawfile-csi/templates/00-security-context-constraints.yaml new file mode 100644 index 0000000..4cdd92a --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/00-security-context-constraints.yaml @@ -0,0 +1,52 @@ +{{- if (and (eq (.Values.securityContextConstraints.enabled | default true) true ) ($.Capabilities.APIVersions.Has "security.openshift.io/v1")) }} +kind: SecurityContextConstraints +apiVersion: security.openshift.io/v1 +metadata: + name: {{ .Values.securityContextConstraints.name }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 0 }}" + "helm.sh/hook-delete-policy": before-hook-creation + +seLinuxContext: + type: RunAsAny +runAsUser: + type: RunAsAny +fsGroup: + type: RunAsAny +supplementalGroups: + type: RunAsAny + +allowedCapabilities: + - '*' +seccompProfiles: + - '*' +allowedUnsafeSysctls: + - '*' +volumes: + - '*' + +defaultAddCapabilities: null +requiredDropCapabilities: null +readOnlyRootFilesystem: false +priority: null + +allowHostPorts: true +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: true +allowHostPID: true +allowHostNetwork: true +allowPrivilegeEscalation: true + +groups: [] +users: +- 'system:serviceaccount:{{ .Release.Namespace }}:{{ include "rawfile-csi.fullname" . }}-driver' +--- +{{- end -}} diff --git a/deploy/charts/rawfile-csi/templates/01-driver.yaml b/deploy/charts/rawfile-csi/templates/01-driver.yaml new file mode 100644 index 0000000..fdf98ef --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/01-driver.yaml @@ -0,0 +1,21 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: {{ .Values.provisionerName }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 1 }}" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + attachRequired: false + podInfoOnMount: true + fsGroupPolicy: File + storageCapacity: true + volumeLifecycleModes: + - Persistent diff --git a/deploy/charts/rawfile-csi/templates/01-flow-schema-control.yaml b/deploy/charts/rawfile-csi/templates/01-flow-schema-control.yaml new file mode 100644 index 0000000..3abfdb2 --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/01-flow-schema-control.yaml @@ -0,0 +1,43 @@ +{{- if ($.Capabilities.APIVersions.Has "flowcontrol.apiserver.k8s.io/v1alpha1") }} +apiVersion: flowcontrol.apiserver.k8s.io/v1alpha1 +kind: FlowSchema +metadata: + name: {{ include "rawfile-csi.fullname" . }}-driver + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 1 }}" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + priorityLevelConfiguration: + name: exempt + distinguisherMethod: + type: ByUser + rules: + - nonResourceRules: + - nonResourceURLs: + - '*' + verbs: + - '*' + resourceRules: + - apiGroups: + - '*' + clusterScope: true + namespaces: + - '*' + resources: + - '*' + verbs: + - '*' + subjects: + - kind: ServiceAccount + serviceAccount: + name: {{ include "rawfile-csi.fullname" . }}-driver + namespace: {{ .Release.Namespace }} +--- +{{- end -}} \ No newline at end of file diff --git a/deploy/charts/rawfile-csi/templates/00-rbac.yaml b/deploy/charts/rawfile-csi/templates/01-rbac.yaml similarity index 51% rename from deploy/charts/rawfile-csi/templates/00-rbac.yaml rename to deploy/charts/rawfile-csi/templates/01-rbac.yaml index 3830937..1e92a51 100644 --- a/deploy/charts/rawfile-csi/templates/00-rbac.yaml +++ b/deploy/charts/rawfile-csi/templates/01-rbac.yaml @@ -2,6 +2,16 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "rawfile-csi.fullname" . }}-driver + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 1 }}" + "helm.sh/hook-delete-policy": before-hook-creation imagePullSecrets: {{- toYaml .Values.imagePullSecrets | nindent 2 }} --- @@ -9,6 +19,16 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "rawfile-csi.fullname" . }}-provisioner + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 1 }}" + "helm.sh/hook-delete-policy": before-hook-creation rules: - apiGroups: [""] resources: ["secrets"] @@ -54,6 +74,16 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "rawfile-csi.fullname" . }}-provisioner + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 1 }}" + "helm.sh/hook-delete-policy": before-hook-creation subjects: - kind: ServiceAccount name: {{ include "rawfile-csi.fullname" . }}-driver @@ -67,6 +97,16 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "rawfile-csi.fullname" . }}-broker + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 1 }}" + "helm.sh/hook-delete-policy": before-hook-creation rules: - apiGroups: [""] resources: ["persistentvolumes"] @@ -79,6 +119,16 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "rawfile-csi.fullname" . }}-broker + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 1 }}" + "helm.sh/hook-delete-policy": before-hook-creation subjects: - kind: ServiceAccount name: {{ include "rawfile-csi.fullname" . }}-driver @@ -92,6 +142,16 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "rawfile-csi.fullname" . }}-resizer + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 1 }}" + "helm.sh/hook-delete-policy": before-hook-creation rules: - apiGroups: [""] resources: ["secrets"] @@ -116,6 +176,16 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "rawfile-csi.fullname" . }}-resizer + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 1 }}" + "helm.sh/hook-delete-policy": before-hook-creation subjects: - kind: ServiceAccount name: {{ include "rawfile-csi.fullname" . }}-driver diff --git a/deploy/charts/rawfile-csi/templates/01-servicemonitor.yaml b/deploy/charts/rawfile-csi/templates/01-servicemonitor.yaml deleted file mode 100644 index ae4c58d..0000000 --- a/deploy/charts/rawfile-csi/templates/01-servicemonitor.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if .Values.serviceMonitor.enabled }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ include "rawfile-csi.fullname" . }}-node - labels: - {{- include "rawfile-csi.labels" . | nindent 4 }} -spec: - endpoints: - - port: metrics - path: /metrics - interval: {{ .Values.serviceMonitor.interval }} - jobLabel: "helm.sh/chart" - namespaceSelector: - matchNames: - - {{ .Release.Namespace }} - selector: - matchLabels: - {{- include "rawfile-csi.selectorLabels" . | nindent 6 }} - component: node -{{- end }} diff --git a/deploy/charts/rawfile-csi/templates/01-controller-plugin.yaml b/deploy/charts/rawfile-csi/templates/02-controller-plugin.yaml similarity index 70% rename from deploy/charts/rawfile-csi/templates/01-controller-plugin.yaml rename to deploy/charts/rawfile-csi/templates/02-controller-plugin.yaml index bed2a0c..dea5ae6 100644 --- a/deploy/charts/rawfile-csi/templates/01-controller-plugin.yaml +++ b/deploy/charts/rawfile-csi/templates/02-controller-plugin.yaml @@ -5,6 +5,15 @@ metadata: labels: {{- include "rawfile-csi.labels" . | nindent 4 }} component: controller + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 2 }}" + "helm.sh/hook-delete-policy": before-hook-creation spec: type: ClusterIP selector: @@ -16,6 +25,16 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: {{ include "rawfile-csi.fullname" . }}-controller + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 2 }}" + "helm.sh/hook-delete-policy": before-hook-creation spec: replicas: 1 serviceName: {{ include "rawfile-csi.fullname" . }} diff --git a/deploy/charts/rawfile-csi/templates/01-node-plugin.yaml b/deploy/charts/rawfile-csi/templates/02-node-plugin.yaml similarity index 80% rename from deploy/charts/rawfile-csi/templates/01-node-plugin.yaml rename to deploy/charts/rawfile-csi/templates/02-node-plugin.yaml index ae20e48..2e381a3 100644 --- a/deploy/charts/rawfile-csi/templates/01-node-plugin.yaml +++ b/deploy/charts/rawfile-csi/templates/02-node-plugin.yaml @@ -5,6 +5,15 @@ metadata: labels: {{- include "rawfile-csi.labels" . | nindent 4 }} component: node + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 2 }}" + "helm.sh/hook-delete-policy": before-hook-creation spec: type: ClusterIP ports: @@ -20,6 +29,16 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: {{ include "rawfile-csi.fullname" . }}-node + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 2 }}" + "helm.sh/hook-delete-policy": before-hook-creation spec: updateStrategy: rollingUpdate: @@ -51,7 +70,7 @@ spec: type: DirectoryOrCreate - name: data-dir hostPath: - path: /var/csi/rawfile + path: {{ .Values.node.dataDir }} type: DirectoryOrCreate containers: - name: csi-driver @@ -66,6 +85,8 @@ spec: value: unix:///csi/csi.sock - name: IMAGE_REPOSITORY value: "{{ .Values.node.image.repository }}" + - name: DATA_DIR + value: "{{ .Values.node.dataDir }}" {{- if regexMatch "^.*-ci$" .Values.node.image.tag }} - name: IMAGE_TAG value: "{{ .Values.node.image.tag }}" diff --git a/deploy/charts/rawfile-csi/templates/02-servicemonitor.yaml b/deploy/charts/rawfile-csi/templates/02-servicemonitor.yaml new file mode 100644 index 0000000..de38e8c --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/02-servicemonitor.yaml @@ -0,0 +1,30 @@ +{{- if (and (eq (.Values.serviceMonitor.enabled | default true) true) ($.Capabilities.APIVersions.Has "monitoring.coreos.com/v1")) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "rawfile-csi.fullname" . }}-node + labels: + {{- include "rawfile-csi.labels" . | nindent 4 }} + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 2 }}" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + endpoints: + - port: metrics + path: /metrics + interval: {{ .Values.serviceMonitor.interval }} + jobLabel: "helm.sh/chart" + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "rawfile-csi.selectorLabels" . | nindent 6 }} + component: node +{{- end }} \ No newline at end of file diff --git a/deploy/charts/rawfile-csi/templates/03-storage-class.yaml b/deploy/charts/rawfile-csi/templates/03-storage-class.yaml new file mode 100644 index 0000000..08c772e --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/03-storage-class.yaml @@ -0,0 +1,21 @@ +{{- if (eq (.Values.storageClass.enabled | default true) true) }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.storageClass.name }} + labels: + {{- include "rawfile-csi.labels" . | nindent 4 }} + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 3 }}" + "helm.sh/hook-delete-policy": before-hook-creation +provisioner: rawfile.csi.openebs.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +{{- end }} \ No newline at end of file diff --git a/deploy/charts/rawfile-csi/templates/04-uninstall-security-context-constraints.yaml b/deploy/charts/rawfile-csi/templates/04-uninstall-security-context-constraints.yaml new file mode 100644 index 0000000..08330ac --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/04-uninstall-security-context-constraints.yaml @@ -0,0 +1,50 @@ +{{- if (and (eq (.Values.securityContextConstraints.enabled | default true) true ) ($.Capabilities.APIVersions.Has "security.openshift.io/v1")) }} +kind: SecurityContextConstraints +apiVersion: security.openshift.io/v1 +metadata: + name: {{ .Values.securityContextConstraints.name }} + labels: + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 4 }}" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed + +seLinuxContext: + type: RunAsAny +runAsUser: + type: RunAsAny +fsGroup: + type: RunAsAny +supplementalGroups: + type: RunAsAny + +allowedCapabilities: + - '*' +seccompProfiles: + - '*' +allowedUnsafeSysctls: + - '*' +volumes: + - '*' + +defaultAddCapabilities: null +requiredDropCapabilities: null +readOnlyRootFilesystem: false +priority: null + +allowHostPorts: true +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: true +allowHostPID: true +allowHostNetwork: true +allowPrivilegeEscalation: true + +groups: [] +users: +- 'system:serviceaccount:{{ .Release.Namespace }}:{{ .Release.Name }}-post-delete' +--- +{{- end -}} diff --git a/deploy/charts/rawfile-csi/templates/05-uninstall-clusterrole.yaml b/deploy/charts/rawfile-csi/templates/05-uninstall-clusterrole.yaml new file mode 100644 index 0000000..ae6a79e --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/05-uninstall-clusterrole.yaml @@ -0,0 +1,40 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-post-delete + labels: + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 5 }}" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +rules: + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings", "clusterroles", "clusterrolebindings"] + verbs: ["get", "list", "watch", "delete"] + + - apiGroups: [""] + resources: [ "persistentvolumes"] + verbs: ["get", "list", "watch", "delete"] + + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "delete"] + + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["get", "list", "watch", "delete"] + + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csidrivers"] + verbs: ["get", "list", "watch", "delete"] + + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + verbs: ["get", "list", "watch", "delete"] + + - apiGroups: ["flowcontrol.apiserver.k8s.io"] + resources: ["flowschema"] + verbs: ["get", "list", "watch", "delete"] \ No newline at end of file diff --git a/deploy/charts/rawfile-csi/templates/05-uninstall-clusterrolebinding.yaml b/deploy/charts/rawfile-csi/templates/05-uninstall-clusterrolebinding.yaml new file mode 100644 index 0000000..f84b40c --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/05-uninstall-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-post-delete + labels: + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 5 }}" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-post-delete + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}-post-delete + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/charts/rawfile-csi/templates/05-uninstall-configmap.yaml b/deploy/charts/rawfile-csi/templates/05-uninstall-configmap.yaml new file mode 100644 index 0000000..f8a4313 --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/05-uninstall-configmap.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-post-delete + namespace: {{ .Release.Namespace }} + labels: + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 5 }}" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +data: + post-delete.sh: |- + #!/bin/bash + + set -e + + KUBEHOME=/tmp/.kube + export KUBECONFIG=${KUBEHOME}/config + export KUBECACHEDIR=${KUBEHOME}/cache + + mkdir -p ${KUBECACHEDIR} + + sleep 10 + + for i in {1..10} # you can also use {0..9} + do + terminatingPods=$(kubectl get pv --no-headers --cache-dir ${KUBECACHEDIR} | awk '{ if ($5 == "Terminating") { print $1} }') + if [ "$terminatingPods" = "" ] + then + break + else + echo "Waiting for PV to finish" + sleep 3 + fi + done + + {{- if (eq (.Values.storageClass.enabled | default true) true) }} + kubectl delete StorageClass {{ .Values.storageClass.name }} -v=9 --cache-dir ${KUBECACHEDIR} + + {{- end }} + + {{- if (and (eq (.Values.serviceMonitor.enabled | default true) true) ($.Capabilities.APIVersions.Has "monitoring.coreos.com/v1")) }} + kubectl delete ServiceMonitor {{ include "rawfile-csi.fullname" . }}-node -n {{ .Release.Namespace }} -v=9 --cache-dir ${KUBECACHEDIR} + + {{- end }} + kubectl delete Service {{ include "rawfile-csi.fullname" . }}-node -n {{ .Release.Namespace }} -v=9 --cache-dir ${KUBECACHEDIR} + kubectl delete DaemonSet {{ include "rawfile-csi.fullname" . }}-node -n {{ .Release.Namespace }} -v=9 --cache-dir ${KUBECACHEDIR} + + kubectl delete Service {{ include "rawfile-csi.fullname" . }}-controller -n {{ .Release.Namespace }} -v=9 --cache-dir ${KUBECACHEDIR} + kubectl delete StatefulSet {{ include "rawfile-csi.fullname" . }}-controller -n {{ .Release.Namespace }} -v=9 --cache-dir ${KUBECACHEDIR} + + kubectl delete ClusterRoleBinding {{ include "rawfile-csi.fullname" . }}-resizer -v=9 --cache-dir ${KUBECACHEDIR} + kubectl delete ClusterRole {{ include "rawfile-csi.fullname" . }}-resizer -v=9 --cache-dir ${KUBECACHEDIR} + kubectl delete ClusterRoleBinding {{ include "rawfile-csi.fullname" . }}-broker -v=9 --cache-dir ${KUBECACHEDIR} + kubectl delete ClusterRole {{ include "rawfile-csi.fullname" . }}-broker -v=9 --cache-dir ${KUBECACHEDIR} + kubectl delete ClusterRoleBinding {{ include "rawfile-csi.fullname" . }}-provisioner -v=9 --cache-dir ${KUBECACHEDIR} + kubectl delete ClusterRole {{ include "rawfile-csi.fullname" . }}-provisioner -v=9 --cache-dir ${KUBECACHEDIR} + {{- if ($.Capabilities.APIVersions.Has "flowcontrol.apiserver.k8s.io/v1alpha1") }} + kubectl delete FlowSchema {{ include "rawfile-csi.fullname" . }}-driver -v=9 --cache-dir ${KUBECACHEDIR} + {{- end }} + kubectl delete ServiceAccount {{ include "rawfile-csi.fullname" . }}-driver -n {{ .Release.Namespace }} -v=9 --cache-dir ${KUBECACHEDIR} + + kubectl delete CSIDriver {{ .Values.provisionerName }} -v=9 --cache-dir ${KUBECACHEDIR} + {{- if (and (eq (.Values.securityContextConstraints.enabled | default true) true ) ($.Capabilities.APIVersions.Has "security.openshift.io/v1")) }} + kubectl delete SecurityContextConstraints {{ .Values.securityContextConstraints.name }} -v=9 --cache-dir ${KUBECACHEDIR} + + {{- end -}} diff --git a/deploy/charts/rawfile-csi/templates/05-uninstall-flow-schema-control.yaml b/deploy/charts/rawfile-csi/templates/05-uninstall-flow-schema-control.yaml new file mode 100644 index 0000000..f4bb27b --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/05-uninstall-flow-schema-control.yaml @@ -0,0 +1,41 @@ +{{- if ($.Capabilities.APIVersions.Has "flowcontrol.apiserver.k8s.io/v1alpha1") }} +apiVersion: flowcontrol.apiserver.k8s.io/v1alpha1 +kind: FlowSchema +metadata: + name: {{ .Release.Name }}-post-delete + labels: + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 5 }}" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +spec: + priorityLevelConfiguration: + name: {{ .Release.Name }}-post-delete + distinguisherMethod: + type: ByUser + rules: + - nonResourceRules: + - nonResourceURLs: + - '*' + verbs: + - '*' + resourceRules: + - apiGroups: + - '*' + clusterScope: true + namespaces: + - '*' + resources: + - '*' + verbs: + - '*' + subjects: + - kind: ServiceAccount + serviceAccount: + name: {{ .Release.Name }}-post-delete + namespace: {{ .Release.Namespace }} +--- +{{- end -}} \ No newline at end of file diff --git a/deploy/charts/rawfile-csi/templates/05-uninstall-priority-level-configuration.yaml b/deploy/charts/rawfile-csi/templates/05-uninstall-priority-level-configuration.yaml new file mode 100644 index 0000000..84f4a1f --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/05-uninstall-priority-level-configuration.yaml @@ -0,0 +1,9 @@ +{{- if ($.Capabilities.APIVersions.Has "flowcontrol.apiserver.k8s.io/v1alpha1") }} +apiVersion: flowcontrol.apiserver.k8s.io/v1alpha1 +kind: PriorityLevelConfiguration +metadata: + name: {{ .Release.Name }}-post-delete +spec: + type: Exempt +--- +{{- end -}} \ No newline at end of file diff --git a/deploy/charts/rawfile-csi/templates/05-uninstall-role.yaml b/deploy/charts/rawfile-csi/templates/05-uninstall-role.yaml new file mode 100644 index 0000000..3490f1a --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/05-uninstall-role.yaml @@ -0,0 +1,18 @@ + +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-post-delete + namespace: {{ .Release.Namespace }} + labels: + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 5 }}" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["get", "list", "watch", "delete"] \ No newline at end of file diff --git a/deploy/charts/rawfile-csi/templates/05-uninstall-rolebinding.yaml b/deploy/charts/rawfile-csi/templates/05-uninstall-rolebinding.yaml new file mode 100644 index 0000000..506b6b8 --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/05-uninstall-rolebinding.yaml @@ -0,0 +1,21 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-post-delete + namespace: {{ .Release.Namespace }} + labels: + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 5 }}" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-post-delete + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ .Release.Name }}-post-delete + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/charts/rawfile-csi/templates/05-uninstall-serviceaccount.yaml b/deploy/charts/rawfile-csi/templates/05-uninstall-serviceaccount.yaml new file mode 100644 index 0000000..02c0eae --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/05-uninstall-serviceaccount.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-post-delete + namespace: {{ .Release.Namespace }} + labels: + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 5 }}" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 2 }} \ No newline at end of file diff --git a/deploy/charts/rawfile-csi/templates/06-uninstall-job.yaml b/deploy/charts/rawfile-csi/templates/06-uninstall-job.yaml new file mode 100644 index 0000000..4f679c2 --- /dev/null +++ b/deploy/charts/rawfile-csi/templates/06-uninstall-job.yaml @@ -0,0 +1,42 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-post-delete + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "{{ add (.Values.hookWeight | default 0) 6 }}" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +spec: + template: + metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + spec: + serviceAccountName: {{ .Release.Name }}-post-delete + restartPolicy: Never + containers: + - name: post-delete-job + image: "{{ .Values.uninstaller.image.repository }}:{{ .Values.uninstaller.image.tag }}" + imagePullPolicy: {{ .Values.uninstaller.image.pullPolicy }} + command: + - 'sh' + - '-c' + - '/scripts/post-delete.sh' + volumeMounts: + - name: pre-install-script + mountPath: "/scripts" + volumes: + - name: pre-install-script + configMap: + defaultMode: 0777 + name: {{ .Release.Name }}-post-delete \ No newline at end of file diff --git a/deploy/charts/rawfile-csi/values.yaml b/deploy/charts/rawfile-csi/values.yaml index 592b604..6706159 100644 --- a/deploy/charts/rawfile-csi/values.yaml +++ b/deploy/charts/rawfile-csi/values.yaml @@ -1,9 +1,9 @@ provisionerName: "rawfile.csi.openebs.io" - +hookWeight: -99 defaults: &defaults image: repository: docker.io/openebs/rawfile-localpv - tag: 0.7.0 + tag: 0.7.1 pullPolicy: Always resources: limits: @@ -18,10 +18,26 @@ controller: node: <<: *defaults + dataDir: /var/lib/csi/rawfile metrics: enabled: false +uninstaller: + image: + repository: mcr.microsoft.com/oss/kubernetes/kubectl + tag: v1.21.0 + pullPolicy: Always + imagePullSecrets: [] + +securityContextConstraints: + enabled: true + name: rawfile + serviceMonitor: enabled: true interval: 1m + +storageClass: + enabled: true + name: rawfile diff --git a/orchestrator/k8s.py b/orchestrator/k8s.py index f45057e..f4e288f 100644 --- a/orchestrator/k8s.py +++ b/orchestrator/k8s.py @@ -49,6 +49,7 @@ def run_on_node(fn, node): "namespace": "kube-system", # FIXME "nodeSelector": json.dumps({"kubernetes.io/hostname": node}), "cmd": json.dumps(fn), + "data_dir": CONFIG["data_dir"], "image_repository": CONFIG["image_repository"], "image_tag": CONFIG["image_tag"], } diff --git a/templates/task.yaml b/templates/task.yaml index 4445af8..201decf 100644 --- a/templates/task.yaml +++ b/templates/task.yaml @@ -11,7 +11,7 @@ spec: volumes: - name: data-dir hostPath: - path: /var/csi/rawfile + path: {data_dir} type: DirectoryOrCreate nodeSelector: {nodeSelector} containers: