From dfe73ecce96f4ba7763870af0a36658a6cab5cad Mon Sep 17 00:00:00 2001 From: Marcio Silva Date: Thu, 1 Feb 2024 11:56:19 -0500 Subject: [PATCH] Support for multiple `verifiers` and `registrars` Includes an example on README Includes also an example on how to use multiple `verifiers` with `kt` Signed-off-by: Marcio Silva --- Makefile | 2 + README.md | 41 +++++++++++++++---- .../keylime-registrar/templates/_helpers.tpl | 6 +-- .../templates/deployment.yaml | 2 +- .../keylime-tenant/templates/deployment.yaml | 2 +- .../keylime-verifier/templates/_helpers.tpl | 6 +-- .../templates/statefulset.yaml | 7 +++- build/helm/keylime/templates/configmap.yaml | 2 +- build/helm/keylime/values.yaml | 2 +- hack/k8s-poc/admin/kt | 32 ++++++++++++++- 10 files changed, 80 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 5641df5..f89398c 100644 --- a/Makefile +++ b/Makefile @@ -263,6 +263,8 @@ helm-keylime-deploy: ## Deploy the keylime helm chart helm-keylime-update: ## Update the deployed keylime helm chart { \ touch $(HELM_CHART_CUSTOM_VALUES);\ + cat $(HACK_DIR)/k8s-poc/admin/kt | sed -e "s/#export/export/g" -e "s^REPLACE_HELM_CHART_KUBECONFIG^$(HELM_CHART_KUBECONFIG)^g" -e "s/REPLACE_KEYLIME_NAMESPACE/$(HELM_CHART_NAMESPACE)/g" > $(MKFILE_DIR)/kt;\ + chmod +x $(MKFILE_DIR)/kt;\ helm upgrade $(HELM_CHART_RELEASE_NAME) $(BUILD_ARTIFACTS_DIR)/keylime-$(HELM_CHART_KEYLIME_VERSION).tgz --namespace $(HELM_CHART_NAMESPACE) --create-namespace --kubeconfig $(HELM_CHART_KUBECONFIG) -f $(HELM_CHART_CUSTOM_VALUES);\ } diff --git a/README.md b/README.md index 8718c2c..da6b311 100644 --- a/README.md +++ b/README.md @@ -170,10 +170,10 @@ global: ### Keylime agent: deploy agents in privileged pods. -This configuration deploys Keylime agents in the cluster in a daemon -set. The pods running the keylime agent are privileged. We do not +This configuration deploys Keylime `agents` in the cluster in a daemon +set. The pods running the keylime `agent` are privileged. We do not recommend running this configuration in production mode, but may help -with debugging keylime agents. +with debugging keylime. ``` @@ -188,13 +188,13 @@ global: ### Keylime agent: deploy agents in unprivileged pods -This configuration deploys Keylime agents in the cluster, but the pods -running the agent are unprivileged. The keylime agent needs access to +This configuration deploys Keylime `agents` in the cluster, but the pods +running it are unprivileged. The keylime `agent` needs access to the TPM device and to parts of the `securityfs` file system. Both of these are provided by the TPM device plugin, which is turned on -automatically with unprivileged agent pods. +automatically with unprivileged `agent` pods. -In addition, the effective group ID of the keylime agent pod has to +In addition, the effective group ID of the keylime `agent` pod has to match the group ID of the TPM device. Kubernetes does not allow `runAsGroup` to take symbolic values. @@ -220,8 +220,6 @@ keylime-agent: runAsGroup: 109 <---- make this match the group ID of group on the hosts running the agent. ``` - - ### Deploy with custom images (e.g. from a local registry) This configuration is for those of us debugging custom (self-built) @@ -251,3 +249,28 @@ global: repository: localhost/custom-tenant-image tag: latest ``` + +### Deploy multiple Verifiers and registrars + +For both availability and scalability reasons (scale-out), multiple `verifiers` +and `registrars` can be deployed. Please do notice that this configuration +cannot operate with a `sqlite` backend (i.e. either `mysql:external` or +`mysql:enable` has to be set to `true `). When adding a new `agent` to a +`verifier`, one of the multiple ones (exposed via multiple services, one per +pod in the `StatefulSet`) has to be selected. The utility `kt`, used by `make +helm-keylime-test` shows an example on how to pick a `verifier` based on an +`agent` UUID. + +``` +global: + database: + mysql: + enable: true + service: + registrar: + replicas: 2 + type: "LoadBalancer" + verifier: + replicas: 3 + type: "LoadBalancer" +``` diff --git a/build/helm/keylime/charts/keylime-registrar/templates/_helpers.tpl b/build/helm/keylime/charts/keylime-registrar/templates/_helpers.tpl index cdce10c..cefd4b2 100644 --- a/build/helm/keylime/charts/keylime-registrar/templates/_helpers.tpl +++ b/build/helm/keylime/charts/keylime-registrar/templates/_helpers.tpl @@ -102,10 +102,10 @@ Expand to the replica count, which is conditional on both the value set on the " and "database" sections of global values */}} {{- define "registrar.replicaCount" -}} -{{- if .Values.global.database.sqlite.enable }} -{{- 1 }} -{{- else }} +{{- if or (eq .Values.global.database.mysql.external true) (eq .Values.global.database.mysql.enable true) }} {{- default 1 .Values.global.service.registrar.replicas }} +{{- else }} +{{- 1 }} {{- end }} {{- end }} diff --git a/build/helm/keylime/charts/keylime-registrar/templates/deployment.yaml b/build/helm/keylime/charts/keylime-registrar/templates/deployment.yaml index a250948..db5a6f9 100644 --- a/build/helm/keylime/charts/keylime-registrar/templates/deployment.yaml +++ b/build/helm/keylime/charts/keylime-registrar/templates/deployment.yaml @@ -33,7 +33,7 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: '{{- include "registrar.image.repository" . }}:{{- include "registrar.image.tag" .}}' - imagePullPolicy: {{ .Values.image.pullPolicy }} + imagePullPolicy: {{ include "registrar.image.pullPolicy" . }} ports: - name: registrar containerPort: {{ .Values.service.nontlsPort }} diff --git a/build/helm/keylime/charts/keylime-tenant/templates/deployment.yaml b/build/helm/keylime/charts/keylime-tenant/templates/deployment.yaml index ce1c44b..f65763a 100644 --- a/build/helm/keylime/charts/keylime-tenant/templates/deployment.yaml +++ b/build/helm/keylime/charts/keylime-tenant/templates/deployment.yaml @@ -35,7 +35,7 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: '{{- include "tenant.image.repository" . }}:{{- include "tenant.image.tag" .}}' - imagePullPolicy: {{ .Values.image.pullPolicy }} + imagePullPolicy: {{ include "tenant.image.pullPolicy" . }} command: - /bin/bash - -c diff --git a/build/helm/keylime/charts/keylime-verifier/templates/_helpers.tpl b/build/helm/keylime/charts/keylime-verifier/templates/_helpers.tpl index 249d860..5d22fae 100644 --- a/build/helm/keylime/charts/keylime-verifier/templates/_helpers.tpl +++ b/build/helm/keylime/charts/keylime-verifier/templates/_helpers.tpl @@ -102,10 +102,10 @@ Expand to the replica count, which is conditional on both the value set on the " and "database" sections of global values */}} {{- define "verifier.replicaCount" -}} -{{- if .Values.global.database.sqlite.enable }} -{{- 1 }} -{{- else }} +{{- if or (eq .Values.global.database.mysql.external true) (eq .Values.global.database.mysql.enable true) }} {{- default 1 .Values.global.service.verifier.replicas }} +{{- else }} +{{- 1 }} {{- end }} {{- end }} diff --git a/build/helm/keylime/charts/keylime-verifier/templates/statefulset.yaml b/build/helm/keylime/charts/keylime-verifier/templates/statefulset.yaml index e775e17..c713e33 100644 --- a/build/helm/keylime/charts/keylime-verifier/templates/statefulset.yaml +++ b/build/helm/keylime/charts/keylime-verifier/templates/statefulset.yaml @@ -31,10 +31,15 @@ spec: envFrom: - configMapRef: name: {{ include "verifier.configMap" . }} + env: + - name: KEYLIME_VERIFIER_UUID + valueFrom: + fieldRef: + fieldPath: metadata.name securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: '{{- include "verifier.image.repository" . }}:{{- include "verifier.image.tag" .}}' - imagePullPolicy: {{ .Values.image.pullPolicy }} + imagePullPolicy: {{ include "verifier.image.pullPolicy" . }} ports: - name: verifier containerPort: {{ .Values.service.port }} diff --git a/build/helm/keylime/templates/configmap.yaml b/build/helm/keylime/templates/configmap.yaml index b28e2d7..55b3679 100644 --- a/build/helm/keylime/templates/configmap.yaml +++ b/build/helm/keylime/templates/configmap.yaml @@ -7,7 +7,7 @@ metadata: {{- include "keylime.labels" . | nindent 4 }} data: {{- if or .Values.global.database.mysql.enable .Values.global.database.mysql.external }} - {{- if .Values.global.database.mysql.external }} + {{- if .Values.global.database.mysql.external }} KEYLIME_REGISTRAR_DATABASE_URL: "mysql+pymysql://{{ .Values.mysql.auth.externalUser }}:{{ .Values.mysql.auth.externalPassword }}@{{ .Values.mysql.auth.externalIP }}:3306/{{ .Values.mysql.auth.database }}?charset=utf8" KEYLIME_VERIFIER_DATABASE_URL: "mysql+pymysql://{{ .Values.mysql.auth.externalUser }}:{{ .Values.mysql.auth.externalPassword }}@{{ .Values.mysql.auth.externalIP }}:3306/{{ .Values.mysql.auth.database }}?charset=utf8" {{- else }} diff --git a/build/helm/keylime/values.yaml b/build/helm/keylime/values.yaml index 980c969..3534a69 100644 --- a/build/helm/keylime/values.yaml +++ b/build/helm/keylime/values.yaml @@ -131,7 +131,7 @@ global: # mysql enables a MySQL database backend mysql: # enable activates the MySQL database backend - # This will pull in a MySQL helm chart for deployment. + # This will pull in a MySQL helm chart for deployment. IMPORTANT: this will override sqlite (enabled by default) enable: false # leave it empty, and a new password, maintained accross multiple upgrades, will be generated password: "" diff --git a/hack/k8s-poc/admin/kt b/hack/k8s-poc/admin/kt index a8ccc39..2ce8732 100755 --- a/hack/k8s-poc/admin/kt +++ b/hack/k8s-poc/admin/kt @@ -11,6 +11,9 @@ export KEYLIME_NAMESPACE=keylime #export KEYLIME_NAMESPACE=REPLACE_KEYLIME_NAMESPACE #export KUBECONFIG=REPLACE_HELM_CHART_KUBECONFIG +KEYLIME_VERIFIER_SERVICE_PREFIX=$(kubectl get services --no-headers -l '!statefulset.kubernetes.io/pod-name,app.kubernetes.io/name=keylime-verifier' -n $KEYLIME_NAMESPACE -o json | jq -r '.items[].metadata.name') +KEYLIME_VERIFIER_SERVICE_DOMAIN=$(kubectl get cm coredns -n kube-system -o jsonpath="{.data.Corefile}" | grep ".local " | awk -F ' ' '{print $2}') + KEYLIME_TENANT_POD=$(kubectl get pods --namespace ${KEYLIME_NAMESPACE} | grep tenant | awk '{ print $1 }') if [[ -z $KEYLIME_TENANT_POD ]] then @@ -18,6 +21,24 @@ then exit 1 fi +KEYLIME_NUMBER_OF_VERIFIERS=$(kubectl get services --no-headers -l 'app.kubernetes.io/name=keylime-verifier,statefulset.kubernetes.io/pod-name' -n $KEYLIME_NAMESPACE | wc -l) + +function verifier_from_agent_uuid { + local _number_of_verifiers=$1 + local _agent_uuid=$2 + + _agent_hash=$(echo $((0x$(sha1sum <<<"${_agent_uuid}")0))) + + if [[ ${_agent_hash} -lt 0 ]] + then + _agent_hash=$((-_agent_hash)) + fi + + _verifier_nr=$((_agent_hash%_number_of_verifiers)) + + echo $KEYLIME_VERIFIER_SERVICE_PREFIX-${_verifier_nr}.${KEYLIME_NAMESPACE}.svc.${KEYLIME_VERIFIER_SERVICE_DOMAIN} +} + KEYLIME_RECEIVED_COMMAND="$*" while [[ $# -gt 0 ]] @@ -85,7 +106,7 @@ then echo "#### Uploading files $KEYLIME_FILE_LIST to pod..." for klf in $(echo $KEYLIME_FILE_LIST | sed 's/,/ /g') do - kubectl cp $klf ${KEYLIME_NAMESPACE}/$KEYLIME_TENANT_POD:$klf + kubectl cp $klf ${KEYLIME_NAMESPACE}/$KEYLIME_TENANT_POD:$klf done fi @@ -96,7 +117,14 @@ then KEYLIME_TENANT_COMMAND="for uuid in $(echo ${_all_agent_uuids} | sed 's/,/ /g'); do /usr/local/bin/keylime_tenant $(echo $KEYLIME_RECEIVED_COMMAND | sed 's/deleteall/delete/g') -u \\\${uuid}; done" elif echo $KEYLIME_RECEIVED_COMMAND | grep -q addall then - KEYLIME_TENANT_COMMAND="for uuid in $(echo ${_all_agent_uuids} | sed 's/,/ /g'); do /usr/local/bin/keylime_tenant $(echo $KEYLIME_RECEIVED_COMMAND | sed 's/addall/add/g') -u \\\${uuid}; done" + for uuid in $(echo ${_all_agent_uuids} | sed 's/,/ /g') + do + _ktc="/usr/local/bin/keylime_tenant $(echo $KEYLIME_RECEIVED_COMMAND | sed 's/addall/add/g') -u ${uuid} -v $(verifier_from_agent_uuid $KEYLIME_NUMBER_OF_VERIFIERS $KEYLIME_TENANT_AGENT_UUID); "${_ktc} + done + KEYLIME_TENANT_COMMAND=${_ktc}/bin/true + elif echo $KEYLIME_RECEIVE_COMMAND | grep -q add + then + KEYLIME_TENANT_COMMAND=$KEYLIME_TENANT_COMMAND" -v $(verifier_from_agent_uuid $KEYLIME_NUMBER_OF_VERIFIERS $KEYLIME_TENANT_AGENT_UUID)" else KEYLIME_TENANT_COMMAND="/usr/local/bin/keylime_tenant $KEYLIME_RECEIVED_COMMAND" fi