From 16dd8e34c8a09c10bcb1e9c39b6f1e4fd96ab626 Mon Sep 17 00:00:00 2001
From: Christopher Swenson <christopher.swenson@hashicorp.com>
Date: Thu, 7 Mar 2024 09:51:41 -0800
Subject: [PATCH 1/6] Add annotation on config change

When updating the Vault config (and corresponding)
configmap, we now generate a checksum of the config
and set it as an annotation on both the configmap
and the Vault StatefulSet pod template.

This allows the deployer to know what pods need to
be restarted to pick up the a changed config.

We still recommend using the standard upgrade
[method for Vault on Kubernetes](https://developer.hashicorp.com/vault/tutorials/kubernetes/kubernetes-raft-deployment-guide#upgrading-vault-on-kubernetes),
i.e., using the `OnDelete` strategy
for the Vault StatefulSet, so updating the config
and doing a `helm upgrade` should not trigger the
pods to restart, and then deleting pods one
at a time, starting with the standby pods.

With `kubectl` and `jq`, you can check check which
pods need to be updated by first getting the value
of the current configmap checksum:

```shell
kubectl get pods -o json | jq -r ".items[] | select(.metadata.annotations.\"config/checksum\" != $(kubectl get configmap vault-config -o json | jq '.metadata.annotations."config/checksum"') ) | .metadata.name"
```

Fixes #748.
---
 templates/_helpers.tpl                 | 30 ++++++++++++++++++++++++--
 templates/server-config-configmap.yaml | 22 +++----------------
 2 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index 8f77f9220..2f88ed6a6 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -457,9 +457,10 @@ Sets the injector deployment update strategy
 {{/*
 Sets extra pod annotations
 */}}
-{{- define "vault.annotations" -}}
-  {{- if .Values.server.annotations }}
+{{- define "vault.annotations" }}
       annotations:
+        config/checksum: {{ include "vault.config" . | sha256sum }}
+  {{- if .Values.server.annotations }}
         {{- $tp := typeOf .Values.server.annotations }}
         {{- if eq $tp "string" }}
           {{- tpl .Values.server.annotations . | nindent 8 }}
@@ -1075,3 +1076,28 @@ Supported inputs are Values.ui
 {{- end -}}
 {{- end }}
 {{- end -}}
+
+{{/*
+config file from values
+*/}}
+{{- define "vault.config" -}}
+  {{- if or (eq .mode "ha") (eq .mode "standalone") }}
+  {{- $type := typeOf (index .Values.server .mode).config }}
+  {{- if eq $type "string" }}
+    disable_mlock = true
+  {{- if eq .mode "standalone" }}
+    {{ tpl .Values.server.standalone.config . | nindent 4 | trim }}
+  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "false") }}
+    {{ tpl .Values.server.ha.config . | nindent 4 | trim }}
+  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
+    {{ tpl .Values.server.ha.raft.config . | nindent 4 | trim }}
+  {{ end }}
+  {{- else }}
+  {{- if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
+{{ merge (dict "disable_mlock" true) (index .Values.server .mode).raft.config | toPrettyJson | indent 4 }}
+  {{- else }}
+{{ merge (dict "disable_mlock" true) (index .Values.server .mode).config | toPrettyJson | indent 4 }}
+  {{- end }}
+  {{- end }}
+  {{- end }}
+{{- end -}}
\ No newline at end of file
diff --git a/templates/server-config-configmap.yaml b/templates/server-config-configmap.yaml
index 5c660579f..1086dc820 100644
--- a/templates/server-config-configmap.yaml
+++ b/templates/server-config-configmap.yaml
@@ -18,27 +18,11 @@ metadata:
     app.kubernetes.io/name: {{ include "vault.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
+  annotations:
+    config/checksum: {{ include "vault.config" . | sha256sum }}
 data:
   extraconfig-from-values.hcl: |-
-  {{- if or (eq .mode "ha") (eq .mode "standalone") }}
-  {{- $type := typeOf (index .Values.server .mode).config }}
-  {{- if eq $type "string" }}
-    disable_mlock = true
-  {{- if eq .mode "standalone" }}
-    {{ tpl .Values.server.standalone.config . | nindent 4 | trim }}
-  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "false") }}
-    {{ tpl .Values.server.ha.config . | nindent 4 | trim }}
-  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
-    {{ tpl .Values.server.ha.raft.config . | nindent 4 | trim }}
-  {{ end }}
-  {{- else }}
-  {{- if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
-{{ merge (dict "disable_mlock" true) (index .Values.server .mode).raft.config | toPrettyJson | indent 4 }}
-  {{- else }}
-{{ merge (dict "disable_mlock" true) (index .Values.server .mode).config | toPrettyJson | indent 4 }}
-  {{- end }}
-  {{- end }}
-  {{- end }}
+    {{ template "vault.config" . }}
 {{- end }}
 {{- end }}
 {{- end }}

From f77fc3cfe1974de3d9531c4dd56df290b7c14e14 Mon Sep 17 00:00:00 2001
From: Christopher Swenson <christopher.swenson@hashicorp.com>
Date: Mon, 11 Mar 2024 10:11:07 -0700
Subject: [PATCH 2/6] Address PR comments

And update Test.dockerfile to use virtualenv pip
install, as the global pip install no longer works.
---
 templates/_helpers.tpl                 |  4 +++-
 templates/server-config-configmap.yaml |  4 +++-
 test/docker/Test.dockerfile            |  6 +++++-
 test/unit/server-configmap.bats        | 19 +++++++++++++++++++
 test/unit/server-statefulset.bats      | 19 +++++++++++++++++++
 values.yaml                            |  9 ++++++++-
 6 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index 2f88ed6a6..99dee980b 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -459,7 +459,9 @@ Sets extra pod annotations
 */}}
 {{- define "vault.annotations" }}
       annotations:
-        config/checksum: {{ include "vault.config" . | sha256sum }}
+  {{- if .Values.server.configAnnotation }}
+        vaultproject.io/config-checksum: {{ include "vault.config" . | sha256sum }}
+  {{- end }}
   {{- if .Values.server.annotations }}
         {{- $tp := typeOf .Values.server.annotations }}
         {{- if eq $tp "string" }}
diff --git a/templates/server-config-configmap.yaml b/templates/server-config-configmap.yaml
index 1086dc820..2dbc973f1 100644
--- a/templates/server-config-configmap.yaml
+++ b/templates/server-config-configmap.yaml
@@ -18,8 +18,10 @@ metadata:
     app.kubernetes.io/name: {{ include "vault.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- if .Values.server.configAnnotation }}
   annotations:
-    config/checksum: {{ include "vault.config" . | sha256sum }}
+    vaultproject.io/config-checksum: {{ include "vault.config" . | sha256sum }}
+{{- end }}
 data:
   extraconfig-from-values.hcl: |-
     {{ template "vault.config" . }}
diff --git a/test/docker/Test.dockerfile b/test/docker/Test.dockerfile
index 472a97acd..69baa4763 100644
--- a/test/docker/Test.dockerfile
+++ b/test/docker/Test.dockerfile
@@ -28,7 +28,11 @@ RUN apk update && apk add --no-cache --virtual .build-deps \
     jq
 
 # yq
-RUN pip install yq
+RUN python3 -m venv venv && \
+    . venv/bin/activate && \
+    pip install yq && \
+    ln -s $PWD/venv/bin/yq /usr/local/bin/yq && \
+    deactivate
 
 # gcloud
 RUN curl -OL https://dl.google.com/dl/cloudsdk/channels/rapid/install_google_cloud_sdk.bash && \
diff --git a/test/unit/server-configmap.bats b/test/unit/server-configmap.bats
index eea7e7008..13833727e 100755
--- a/test/unit/server-configmap.bats
+++ b/test/unit/server-configmap.bats
@@ -139,3 +139,22 @@ load _helpers
       yq 'length > 0' | tee /dev/stderr)
   [ "${actual}" = "false" ]
 }
+
+@test "server/ConfigMap: config checksum annotation defaults to off" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-config-configmap.yaml \
+      . | tee /dev/stderr |
+      yq '.metadata.annotations["vaultproject.io/config-checksum"] == null' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "server/ConfigMap: config checksum annotation can be enabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-config-configmap.yaml \
+      --set 'server.configAnnotation=true' \
+      . | tee /dev/stderr |
+      yq '.metadata.annotations["vaultproject.io/config-checksum"] == null' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats
index 8acd9ee91..d67023050 100755
--- a/test/unit/server-statefulset.bats
+++ b/test/unit/server-statefulset.bats
@@ -1681,6 +1681,25 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
+@test "server/standalone-StatefulSet: config checksum annotation defaults to off" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-statefulset.yaml \
+      . | tee /dev/stderr |
+      yq '.spec.template.metadata.annotations["vaultproject.io/config-checksum"] == null' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "server/standalone-StatefulSet: config checksum annotation can be enabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-statefulset.yaml \
+      --set 'server.configAnnotation=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.metadata.annotations["vaultproject.io/config-checksum"] == null' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
 #--------------------------------------------------------------------
 # priorityClassName
 
diff --git a/values.yaml b/values.yaml
index 17f5ca572..a57b9de69 100644
--- a/values.yaml
+++ b/values.yaml
@@ -668,6 +668,13 @@ server:
   # of the annotations to apply to the server pods
   annotations: {}
 
+  # Add an annotation to the server configmap and the statefulset pods,
+  # vaultproject.io/config-checksum, that is a hash of the Vault configuration.
+  # This can be used together with an OnDelete deployment strategy to help
+  # identify which pods still need to be deleted during a deployment to pick up
+  # any configuration changes.
+  configAnnotation: false
+
   # Enables a headless service to be used by the Vault Statefulset
   service:
     enabled: true
@@ -714,7 +721,7 @@ server:
     #     PreferDualStack: Allocates IPv4 and IPv6 cluster IPs for the Service.
     #     RequireDualStack: Allocates Service .spec.ClusterIPs from both IPv4 and IPv6 address ranges.
     ipFamilyPolicy: ""
-    
+
     # Sets the families that should be supported and the order in which they should be applied to ClusterIP as well.
     # Can be IPv4 and/or IPv6.
     ipFamilies: []

From fc5721ddd858f026962ee98ee1a9e8781d871a14 Mon Sep 17 00:00:00 2001
From: Christopher Swenson <swenson@swenson.io>
Date: Mon, 11 Mar 2024 14:18:04 -0700
Subject: [PATCH 3/6] Update templates/_helpers.tpl

Co-authored-by: Tom Proctor <tomhjp@users.noreply.github.com>
---
 templates/_helpers.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index 99dee980b..de0c95f3a 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -460,7 +460,7 @@ Sets extra pod annotations
 {{- define "vault.annotations" }}
       annotations:
   {{- if .Values.server.configAnnotation }}
-        vaultproject.io/config-checksum: {{ include "vault.config" . | sha256sum }}
+        vault.hashicorp.com/config-checksum: {{ include "vault.config" . | sha256sum }}
   {{- end }}
   {{- if .Values.server.annotations }}
         {{- $tp := typeOf .Values.server.annotations }}

From de1ca097a248f48a1cb9b5caa58543d7e4df7e81 Mon Sep 17 00:00:00 2001
From: Christopher Swenson <christopher.swenson@hashicorp.com>
Date: Mon, 11 Mar 2024 14:19:35 -0700
Subject: [PATCH 4/6] Update annotations

---
 templates/server-config-configmap.yaml | 2 +-
 test/unit/server-configmap.bats        | 4 ++--
 test/unit/server-statefulset.bats      | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/templates/server-config-configmap.yaml b/templates/server-config-configmap.yaml
index 2dbc973f1..46daf3e10 100644
--- a/templates/server-config-configmap.yaml
+++ b/templates/server-config-configmap.yaml
@@ -20,7 +20,7 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- if .Values.server.configAnnotation }}
   annotations:
-    vaultproject.io/config-checksum: {{ include "vault.config" . | sha256sum }}
+    vault.hashicorp.com/config-checksum: {{ include "vault.config" . | sha256sum }}
 {{- end }}
 data:
   extraconfig-from-values.hcl: |-
diff --git a/test/unit/server-configmap.bats b/test/unit/server-configmap.bats
index 13833727e..bad3ab483 100755
--- a/test/unit/server-configmap.bats
+++ b/test/unit/server-configmap.bats
@@ -145,7 +145,7 @@ load _helpers
   local actual=$(helm template \
       --show-only templates/server-config-configmap.yaml \
       . | tee /dev/stderr |
-      yq '.metadata.annotations["vaultproject.io/config-checksum"] == null' | tee /dev/stderr)
+      yq '.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
   [ "${actual}" = "true" ]
 }
 
@@ -155,6 +155,6 @@ load _helpers
       --show-only templates/server-config-configmap.yaml \
       --set 'server.configAnnotation=true' \
       . | tee /dev/stderr |
-      yq '.metadata.annotations["vaultproject.io/config-checksum"] == null' | tee /dev/stderr)
+      yq '.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
   [ "${actual}" = "false" ]
 }
diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats
index d67023050..a84e1c431 100755
--- a/test/unit/server-statefulset.bats
+++ b/test/unit/server-statefulset.bats
@@ -1686,7 +1686,7 @@ load _helpers
   local actual=$(helm template \
       --show-only templates/server-statefulset.yaml \
       . | tee /dev/stderr |
-      yq '.spec.template.metadata.annotations["vaultproject.io/config-checksum"] == null' | tee /dev/stderr)
+      yq '.spec.template.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
   [ "${actual}" = "true" ]
 }
 
@@ -1696,7 +1696,7 @@ load _helpers
       --show-only templates/server-statefulset.yaml \
       --set 'server.configAnnotation=true' \
       . | tee /dev/stderr |
-      yq '.spec.template.metadata.annotations["vaultproject.io/config-checksum"] == null' | tee /dev/stderr)
+      yq '.spec.template.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
   [ "${actual}" = "false" ]
 }
 

From de9f647f4c639812c936377539726a4ecfe66ee9 Mon Sep 17 00:00:00 2001
From: Christopher Swenson <christopher.swenson@hashicorp.com>
Date: Tue, 12 Mar 2024 08:53:02 -0700
Subject: [PATCH 5/6] Use name includeConfigAnnotation

---
 templates/_helpers.tpl                 | 2 +-
 templates/server-config-configmap.yaml | 2 +-
 test/unit/server-configmap.bats        | 2 +-
 test/unit/server-statefulset.bats      | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index de0c95f3a..7a22d04cc 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -459,7 +459,7 @@ Sets extra pod annotations
 */}}
 {{- define "vault.annotations" }}
       annotations:
-  {{- if .Values.server.configAnnotation }}
+  {{- if .Values.server.includeConfigAnnotation }}
         vault.hashicorp.com/config-checksum: {{ include "vault.config" . | sha256sum }}
   {{- end }}
   {{- if .Values.server.annotations }}
diff --git a/templates/server-config-configmap.yaml b/templates/server-config-configmap.yaml
index 46daf3e10..1fed2e690 100644
--- a/templates/server-config-configmap.yaml
+++ b/templates/server-config-configmap.yaml
@@ -18,7 +18,7 @@ metadata:
     app.kubernetes.io/name: {{ include "vault.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- if .Values.server.configAnnotation }}
+{{- if .Values.server.includeConfigAnnotation }}
   annotations:
     vault.hashicorp.com/config-checksum: {{ include "vault.config" . | sha256sum }}
 {{- end }}
diff --git a/test/unit/server-configmap.bats b/test/unit/server-configmap.bats
index bad3ab483..dcb9076be 100755
--- a/test/unit/server-configmap.bats
+++ b/test/unit/server-configmap.bats
@@ -153,7 +153,7 @@ load _helpers
   cd `chart_dir`
   local actual=$(helm template \
       --show-only templates/server-config-configmap.yaml \
-      --set 'server.configAnnotation=true' \
+      --set 'server.includeConfigAnnotation=true' \
       . | tee /dev/stderr |
       yq '.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
   [ "${actual}" = "false" ]
diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats
index a84e1c431..aee98ce8b 100755
--- a/test/unit/server-statefulset.bats
+++ b/test/unit/server-statefulset.bats
@@ -1694,7 +1694,7 @@ load _helpers
   cd `chart_dir`
   local actual=$(helm template \
       --show-only templates/server-statefulset.yaml \
-      --set 'server.configAnnotation=true' \
+      --set 'server.includeConfigAnnotation=true' \
       . | tee /dev/stderr |
       yq '.spec.template.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
   [ "${actual}" = "false" ]

From fd5abeb6c549edef5d84bffd563a6f7c7bcc7c0b Mon Sep 17 00:00:00 2001
From: Christopher Swenson <christopher.swenson@hashicorp.com>
Date: Mon, 18 Mar 2024 10:27:17 -0700
Subject: [PATCH 6/6] Check that annotations are not set by default

---
 test/unit/server-statefulset.bats | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats
index aee98ce8b..b2b8efe4e 100755
--- a/test/unit/server-statefulset.bats
+++ b/test/unit/server-statefulset.bats
@@ -1690,6 +1690,15 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
+@test "server/standalone-StatefulSet: config checksum annotation off does not set annotations" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-statefulset.yaml \
+      . | tee /dev/stderr |
+      yq '.spec.template.metadata.annotations | length == 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
 @test "server/standalone-StatefulSet: config checksum annotation can be enabled" {
   cd `chart_dir`
   local actual=$(helm template \