Add support for HA KOTS (#47)

* Add support for HA KOTS
replicatedhq · May 30, 2024 · c719441 · c719441
1 parent 15fbb23
commit c719441
Show file tree

Hide file tree

Showing 21 changed files with 143 additions and 164 deletions.
diff --git a/scripts/build-local.sh b/scripts/build-local.sh
@@ -2,7 +2,7 @@
 
 set -e
 
-export CURRENT_USER=`id -u -n`
+export CURRENT_USER=${GITHUB_USER:-$(id -u -n)}
 export KOTS_VERSION=$1
 export KOTS_TAG=v${KOTS_VERSION}
 export KOTSADM_REGISTRY=kotsadm # docker.io is implicit

diff --git a/scripts/build-ttl.sh b/scripts/build-ttl.sh
@@ -2,11 +2,12 @@
 
 set -e
 
-GIT_COMMIT=$1
+export GIT_COMMIT=${1:-main}
+export CURRENT_USER=${GITHUB_USER:-$(id -u -n)}
 export CHART_VERSION=0.0.0-${GIT_COMMIT}
 export KOTS_VERSION=24h
 export KOTS_TAG=24h
-export KOTSADM_REGISTRY=ttl.sh/$USER
+export KOTSADM_REGISTRY=ttl.sh/$CURRENT_USER
 
 curl -O -L https://raw.githubusercontent.com/replicatedhq/kots/${GIT_COMMIT}/.image.env
 export $(cat .image.env | sed 's/#.*//g' | xargs)
@@ -15,7 +16,7 @@ envsubst < Chart.yaml.tmpl > Chart.yaml
 envsubst < values.yaml.tmpl > values.yaml
 
 rm -f admin-console-*.tgz
-export CHART_NAME=`helm package . | rev | cut -d/ -f1 | rev`
-helm push $CHART_NAME oci://ttl.sh/$USER
+export CHART_NAME=$(helm package . | rev | cut -d/ -f1 | rev)
+helm push $CHART_NAME oci://ttl.sh/$CURRENT_USER
 
 rm -f Chart.yaml values.yaml .image.env
diff --git a/templates/NOTES.txt b/templates/NOTES.txt
diff --git a/templates/abandon-resources-hook.yaml b/templates/abandon-resources-hook.yaml
@@ -6,6 +6,7 @@ metadata:
   annotations:
     helm.sh/hook: pre-upgrade
     helm.sh/hook-weight: "2"
+    helm.sh/hook-delete-policy: hook-succeeded
   labels:
     {{- include "admin-console.labels" . | nindent 4 }}
 spec:

diff --git a/templates/kotsadm-clusterrole.yaml b/templates/kotsadm-clusterrole.yaml
@@ -1,4 +1,4 @@
-{{ if not .Values.minimalRBAC }}
+{{- if not .Values.minimalRBAC }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -12,4 +12,4 @@ rules:
   - '*'
   verbs:
   - '*'
-{{ end }}
+{{- end }}
diff --git a/templates/kotsadm-clusterrolebinding.yaml b/templates/kotsadm-clusterrolebinding.yaml
@@ -1,4 +1,4 @@
-{{ if not .Values.minimalRBAC }}
+{{- if not .Values.minimalRBAC }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -13,4 +13,4 @@ subjects:
 - kind: ServiceAccount
   name: kotsadm
   namespace: {{ .Release.Namespace }}
-{{ end }}
+{{- end }}
diff --git a/templates/kotsadm-statefulset.yaml → templates/kotsadm-deployment.yaml b/templates/kotsadm-statefulset.yaml → templates/kotsadm-deployment.yaml
@@ -1,14 +1,14 @@
 apiVersion: apps/v1
-kind: StatefulSet
+kind: Deployment
 metadata:
   labels:
     {{- include "admin-console.labels" . | nindent 4 }}
   name: kotsadm
 spec:
+  replicas: 1
   selector:
     matchLabels:
       app: kotsadm
-  serviceName: kotsadm
   template:
     metadata:
       annotations:
@@ -54,7 +54,6 @@ spec:
             secretKeyRef:
               key: key
               name: kotsadm-session
-{{- if not .Values.isHelmManaged }}
         - name: RQLITE_PASSWORD
           valueFrom:
             secretKeyRef:
@@ -65,11 +64,12 @@ spec:
             secretKeyRef:
               key: uri
               name: kotsadm-rqlite
-{{- end }}
         - name: POD_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: POD_OWNER_KIND
+          value: deployment
         - name: API_ENCRYPTION_KEY
           valueFrom:
             secretKeyRef:
@@ -87,8 +87,6 @@ spec:
         - name: EMBEDDED_CLUSTER_VERSION
           value: {{ .Values.embeddedClusterVersion | quote }}
 {{- end }}
-        - name: IS_HELM_MANAGED
-          value: {{ .Values.isHelmManaged | quote }}
         - name: DISABLE_OUTBOUND_CONNECTIONS
           value: {{ .Values.isAirgap | quote }}
         image: {{ .Values.images.kotsadm }}
@@ -113,13 +111,10 @@ spec:
             cpu: 100m
             memory: 100Mi
         volumeMounts:
-        - mountPath: /kotsadmdata
-          name: kotsadmdata
         - mountPath: /backup
           name: backup
         - mountPath: /tmp
           name: tmp
-{{ if not .Values.isHelmManaged }}
       initContainers:
       - args:
         - plan
@@ -174,7 +169,7 @@ spec:
         - mountPath: /migrations
           name: migrations
       - command:
-        - /restore.sh
+        - /restore-db.sh
         env:
         - name: RQLITE_PASSWORD
           valueFrom:
@@ -183,7 +178,7 @@ spec:
               name: kotsadm-rqlite
         image: {{ .Values.images.kotsadm }}
         imagePullPolicy: IfNotPresent
-        name: restore-data
+        name: restore-db
         resources:
           limits:
             cpu: "1"
@@ -192,36 +187,20 @@ spec:
             cpu: 100m
             memory: 100Mi
         volumeMounts:
-        - mountPath: /kotsadmdata
-          name: kotsadmdata
         - mountPath: /backup
           name: backup
         - mountPath: /tmp
           name: tmp
-{{ end }}
       restartPolicy: Always
       securityContext:
         fsGroup: 1001
         runAsUser: 1001
       serviceAccountName: kotsadm
       volumes:
-      - persistentVolumeClaim:
-          claimName: kotsadmdata
-        name: kotsadmdata
       - emptyDir:
           medium: Memory
         name: migrations
       - emptyDir: {}
         name: backup
       - emptyDir: {}
-        name: tmp
-  volumeClaimTemplates:
-  - metadata:
-      name: kotsadmdata
-    spec:
-      accessModes:
-      - ReadWriteOnce
-      resources:
-        requests:
-          storage: 4Gi
-
+        name: tmp
diff --git a/templates/kotsadm-role.yaml b/templates/kotsadm-role.yaml
@@ -1,4 +1,4 @@
-{{ if .Values.minimalRBAC }}
+{{- if .Values.minimalRBAC }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -12,4 +12,4 @@ rules:
   - '*'
   verbs:
   - '*'
-{{ end }}
+{{- end }}
diff --git a/templates/kotsadm-rolebinding.yaml b/templates/kotsadm-rolebinding.yaml
@@ -1,4 +1,4 @@
-{{ if .Values.minimalRBAC }}
+{{- if .Values.minimalRBAC }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
@@ -13,4 +13,4 @@ subjects:
 - kind: ServiceAccount
   name: kotsadm
   namespace: {{ .Release.Namespace }}
-{{ end }}
+{{- end }}
diff --git a/templates/kotsadm-service.yaml b/templates/kotsadm-service.yaml
@@ -1,4 +1,4 @@
-{{ if .Values.service.enabled }}
+{{- if .Values.service.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -18,4 +18,4 @@ spec:
   type: {{ .Values.service.type }}
 status:
   loadBalancer: {}
-{{ end }}
+{{- end }}
diff --git a/templates/kurl-proxy-deployment.yaml b/templates/kurl-proxy-deployment.yaml
@@ -1,4 +1,4 @@
-{{ if .Values.kurlProxy.enabled }}
+{{- if .Values.kurlProxy.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -51,4 +51,4 @@ spec:
         configMap:
           name: kotsadm-application-metadata
           optional: true
-{{ end }}
+{{- end }}
diff --git a/templates/kurl-proxy-rbac.yaml b/templates/kurl-proxy-rbac.yaml
@@ -1,4 +1,4 @@
-{{ if .Values.kurlProxy.enabled }}
+{{- if .Values.kurlProxy.enabled }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -34,4 +34,4 @@ subjects:
 - kind: ServiceAccount
   name: kurl-proxy
   namespace: {{ .Release.Namespace }}
-{{ end }}
+{{- end }}
diff --git a/templates/kurl-proxy-service.yaml b/templates/kurl-proxy-service.yaml
@@ -1,4 +1,4 @@
-{{ if .Values.kurlProxy.enabled }}
+{{- if .Values.kurlProxy.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -16,4 +16,4 @@ spec:
 {{- if .Values.kurlProxy.nodePort }}
     nodePort: {{ .Values.kurlProxy.nodePort }}
 {{- end}}
-{{ end }}
+{{- end }}
diff --git a/templates/migrate-pvc-hook.yaml b/templates/migrate-pvc-hook.yaml
@@ -0,0 +1,63 @@
+{{- $kotsadmSts := lookup "apps/v1" "StatefulSet" .Release.Namespace "kotsadm" }}
+{{- if $kotsadmSts }}
+# if kotsadm statefulset exists, we need to migrate the data to rqlite
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kotsadm-migrate-pvc-to-rqlite
+  annotations:
+    helm.sh/hook: pre-upgrade
+    helm.sh/hook-weight: "1"
+  labels:
+    {{- include "admin-console.labels" . | nindent 4 }}
+spec:
+  template:
+    metadata:
+      labels:
+        {{- include "admin-console.labels" . | nindent 8 }}
+    spec:
+      initContainers:
+      - command:
+        - /bin/sh
+        - -c
+        - |
+          kubectl scale sts/kotsadm -n {{ .Release.Namespace }} --replicas=0
+          kubectl wait --for=delete pod --selector=app=kotsadm -n {{ .Release.Namespace }} --timeout=300s
+        image: {{ .Values.images.kotsadm }}
+        imagePullPolicy: IfNotPresent
+        name: scale-down-kotsadm
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+      containers:
+      - command:
+        - /kotsadm
+        - migrate
+        - pvc-to-rqlite
+        env:
+        - name: RQLITE_URI
+          valueFrom:
+            secretKeyRef:
+              key: uri
+              name: kotsadm-rqlite
+        image: {{ .Values.images.kotsadm }}
+        imagePullPolicy: IfNotPresent
+        name: migrate-pvc-to-rqlite
+        resources:
+          limits:
+            cpu: "1"
+            memory: 2Gi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - mountPath: /kotsadmdata
+          name: kotsadmdata
+      restartPolicy: OnFailure
+      serviceAccountName: kotsadm
+      volumes:
+      - persistentVolumeClaim:
+          claimName: kotsadmdata-kotsadm-0
+        name: kotsadmdata
+{{- end }}