From ccb06d59e1e435e99aaf298501b5fef33fa6efd1 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 1 May 2024 14:49:21 +0100 Subject: [PATCH] Update integration test, switch to authorization --- templates/prometheus-servicemonitor.yaml | 8 ++-- test/acceptance/_helpers.bash | 1 - test/acceptance/server-telemetry.bats | 41 ++++++------------- test/acceptance/server-test/telemetry.yaml | 31 -------------- test/acceptance/server-test/vault-server.yaml | 26 ++++++++++++ .../server-test/vault-telemetry.yaml | 16 ++++++++ test/unit/prometheus-servicemonitor.bats | 10 ++--- values.yaml | 8 ++-- 8 files changed, 69 insertions(+), 72 deletions(-) delete mode 100644 test/acceptance/server-test/telemetry.yaml create mode 100644 test/acceptance/server-test/vault-server.yaml create mode 100644 test/acceptance/server-test/vault-telemetry.yaml diff --git a/templates/prometheus-servicemonitor.yaml b/templates/prometheus-servicemonitor.yaml index 02630fcd5..32f6d7f6f 100644 --- a/templates/prometheus-servicemonitor.yaml +++ b/templates/prometheus-servicemonitor.yaml @@ -36,9 +36,6 @@ spec: - port: {{ include "vault.scheme" . }} interval: {{ .Values.serverTelemetry.serviceMonitor.interval }} scrapeTimeout: {{ .Values.serverTelemetry.serviceMonitor.scrapeTimeout }} - {{- if .Values.serverTelemetry.serviceMonitor.bearerTokenFile }} - bearerTokenFile: {{ .Values.serverTelemetry.serviceMonitor.bearerTokenFile }} - {{- end }} scheme: {{ include "vault.scheme" . | lower }} path: /v1/sys/metrics params: @@ -52,6 +49,11 @@ spec: tlsConfig: insecureSkipVerify: true {{- end }} + {{- $authz := .Values.serverTelemetry.serviceMonitor.authorization }} + {{- if $authz }} + authorization: + {{- toYaml $authz | nindent 6 }} + {{- end }} namespaceSelector: matchNames: - {{ include "vault.namespace" . }} diff --git a/test/acceptance/_helpers.bash b/test/acceptance/_helpers.bash index 490cdeb29..49638ddaf 100644 --- a/test/acceptance/_helpers.bash +++ b/test/acceptance/_helpers.bash @@ -92,7 +92,6 @@ wait_for_running() { for i in $(seq 60); do if [ -n "$(check ${POD_NAME})" ]; then echo "${POD_NAME} is ready." - sleep 5 return fi diff --git a/test/acceptance/server-telemetry.bats b/test/acceptance/server-telemetry.bats index a7c4e0dd7..fb207f0c3 100644 --- a/test/acceptance/server-telemetry.bats +++ b/test/acceptance/server-telemetry.bats @@ -10,46 +10,29 @@ load _helpers kubectl create namespace acceptance kubectl config set-context --current --namespace=acceptance + # Install prometheus-operator and friends. helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update - helm install \ + helm upgrade --install \ --wait \ - --version 39.6.0 \ + --version 58.3.1 \ prometheus prometheus-community/kube-prometheus-stack - helm install \ + # Upgrade Vault install with telemetry config now that the prometheus CRDs are applied. + helm upgrade --install \ --wait \ - --values ./test/acceptance/server-test/telemetry.yaml \ + --values ./test/acceptance/server-test/vault-server.yaml \ + --values ./test/acceptance/server-test/vault-telemetry.yaml \ "$(name_prefix)" . - wait_for_running $(name_prefix)-0 - - # Sealed, not initialized - wait_for_sealed_vault $(name_prefix)-0 - - # Vault Init - local token=$(kubectl exec -ti "$(name_prefix)-0" -- \ - vault operator init -format=json -n 1 -t 1 | \ - jq -r '.unseal_keys_b64[0]') - [ "${token}" != "" ] - - # Vault Unseal - local pods=($(kubectl get pods --selector='app.kubernetes.io/name=vault' -o json | jq -r '.items[].metadata.name')) - for pod in "${pods[@]}" - do - kubectl exec -ti ${pod} -- vault operator unseal ${token} - done - wait_for_ready "$(name_prefix)-0" - # Unsealed, initialized - local sealed_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | - jq -r '.sealed' ) - [ "${sealed_status}" == "false" ] + echo 'path "sys/metrics" {capabilities = ["read"]}' | kubectl exec -i vault-0 -- vault policy write metrics - - local init_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | - jq -r '.initialized') - [ "${init_status}" == "true" ] + # Store Vault's dev TLS CA and a token in a secret for prometheus to use. + kubectl create secret generic vault-metrics-client \ + --from-literal="ca.crt=$(kubectl exec $(name_prefix)-0 -- cat /var/run/tls/vault-ca.pem)" \ + --from-literal="token=$(kubectl exec vault-0 -- vault token create -policy=metrics -field=token)" # unfortunately it can take up to 2 minutes for the vault prometheus job to appear # TODO: investigate how reduce this. diff --git a/test/acceptance/server-test/telemetry.yaml b/test/acceptance/server-test/telemetry.yaml deleted file mode 100644 index 557008425..000000000 --- a/test/acceptance/server-test/telemetry.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - -server: - standalone: - config: | - ui = true - - listener "tcp" { - tls_disable = 1 - address = "[::]:8200" - cluster_address = "[::]:8201" - # Enable unauthenticated metrics access (necessary for Prometheus Operator) - telemetry { - unauthenticated_metrics_access = "true" - } - } - - storage "file" { - path = "/vault/data" - } - - telemetry { - prometheus_retention_time = "30s" - disable_hostname = true - } - -serverTelemetry: - serviceMonitor: - enabled: true - interval: 15s diff --git a/test/acceptance/server-test/vault-server.yaml b/test/acceptance/server-test/vault-server.yaml new file mode 100644 index 000000000..eb61a7632 --- /dev/null +++ b/test/acceptance/server-test/vault-server.yaml @@ -0,0 +1,26 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +global: + tlsDisable: false +server: + dev: + enabled: true + # >- to convert to a single line with no line breaks. + extraArgs: >- + -dev-tls + -dev-tls-cert-dir=/var/run/tls + -dev-tls-san=vault.default.svc.cluster.local + -dev-tls-san=vault.default.svc + -dev-tls-san=vault.default + -dev-tls-san=vault + -dev-tls-san=$POD_IP + extraEnvironmentVars: + VAULT_CACERT: /var/run/tls/vault-ca.pem + VAULT_LOCAL_CONFIG: '{"telemetry":{"prometheus_retention_time":"30s","disable_hostname":true}}' + volumes: + - name: tls + emptyDir: {} + volumeMounts: + - mountPath: /var/run/tls + name: tls diff --git a/test/acceptance/server-test/vault-telemetry.yaml b/test/acceptance/server-test/vault-telemetry.yaml new file mode 100644 index 000000000..3740b92f8 --- /dev/null +++ b/test/acceptance/server-test/vault-telemetry.yaml @@ -0,0 +1,16 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +serverTelemetry: + serviceMonitor: + enabled: true + interval: 15s + tlsConfig: + ca: + secret: + name: vault-metrics-client + key: ca.crt + authorization: + credentials: + name: vault-metrics-client + key: token diff --git a/test/unit/prometheus-servicemonitor.bats b/test/unit/prometheus-servicemonitor.bats index 290733eb7..2dffb050c 100755 --- a/test/unit/prometheus-servicemonitor.bats +++ b/test/unit/prometheus-servicemonitor.bats @@ -145,23 +145,23 @@ load _helpers [ "$(echo "$output" | yq -r '.spec.endpoints[0].tlsConfig.ca')" = "ca.crt" ] } -@test "prometheus/ServiceMonitor-server: bearerTokenFile default" { +@test "prometheus/ServiceMonitor-server: authorization default" { cd `chart_dir` local output=$( (helm template \ --show-only templates/prometheus-servicemonitor.yaml \ --set 'serverTelemetry.serviceMonitor.enabled=true' \ . ) | tee /dev/stderr) - [ "$(echo "$output" | yq -r '.spec.endpoints[0] | has("bearerTokenFile")')" = "false" ] + [ "$(echo "$output" | yq -r '.spec.endpoints[0].authorization')" = "null" ] } -@test "prometheus/ServiceMonitor-server: bearerTokenFile set" { +@test "prometheus/ServiceMonitor-server: authorization override" { cd `chart_dir` local output=$( (helm template \ --show-only templates/prometheus-servicemonitor.yaml \ + --set 'serverTelemetry.serviceMonitor.authorization.credentials.name=a-secret' \ --set 'serverTelemetry.serviceMonitor.enabled=true' \ - --set 'serverTelemetry.serviceMonitor.bearerTokenFile=tokenfile' \ . ) | tee /dev/stderr) - [ "$(echo "$output" | yq -r '.spec.endpoints[0].bearerTokenFile')" = "tokenfile" ] + [ "$(echo "$output" | yq -r '.spec.endpoints[0].authorization.credentials.name')" = "a-secret" ] } diff --git a/values.yaml b/values.yaml index cd900315f..8bed7ab80 100644 --- a/values.yaml +++ b/values.yaml @@ -1277,11 +1277,13 @@ serverTelemetry: # Timeout for Prometheus scrapes scrapeTimeout: 10s - # tlsConfig used for connecting to the Vault API + # tlsConfig used for scraping the Vault metrics API. + # See API reference: https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.TLSConfig tlsConfig: {} - # bearerTokenfile used for authentication to the Vault metrics API - bearerTokenFile: "" + # authorization used for scraping the Vault metrics API. + # See API reference: https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} prometheusRules: # The Prometheus operator *must* be installed before enabling this feature,