Support for multiple verifiers and registrars

Includes an example on README

Includes also an example on how to use multiple `verifiers` with `kt`

Signed-off-by: Marcio Silva <[email protected]>

Makefile

@@ -263,6 +263,8 @@ helm-keylime-deploy: ## Deploy the keylime helm chart
 helm-keylime-update: ## Update the deployed keylime helm chart
 	{ \
 	touch $(HELM_CHART_CUSTOM_VALUES);\
+	cat $(HACK_DIR)/k8s-poc/admin/kt | sed -e "s/#export/export/g" -e "s^REPLACE_HELM_CHART_KUBECONFIG^$(HELM_CHART_KUBECONFIG)^g" -e "s/REPLACE_KEYLIME_NAMESPACE/$(HELM_CHART_NAMESPACE)/g" > $(MKFILE_DIR)/kt;\
+	chmod +x $(MKFILE_DIR)/kt;\
 	helm upgrade $(HELM_CHART_RELEASE_NAME) $(BUILD_ARTIFACTS_DIR)/keylime-$(HELM_CHART_KEYLIME_VERSION).tgz --namespace $(HELM_CHART_NAMESPACE) --create-namespace --kubeconfig $(HELM_CHART_KUBECONFIG) -f $(HELM_CHART_CUSTOM_VALUES);\
 	}
 

README.md

@@ -170,10 +170,10 @@ global:
 
 ### Keylime agent: deploy agents in privileged pods.
 
-This configuration deploys Keylime agents in the cluster in a daemon
-set. The pods running the keylime agent are privileged. We do not
+This configuration deploys Keylime `agents` in the cluster in a daemon
+set. The pods running the keylime `agent` are privileged. We do not
 recommend running this configuration in production mode, but may help
-with debugging keylime agents.
+with debugging keylime.
 
 
 ```
@@ -188,13 +188,13 @@ global:
 
 ### Keylime agent: deploy agents in unprivileged pods
 
-This configuration deploys Keylime agents in the cluster, but the pods
-running the agent are unprivileged. The keylime agent needs access to
+This configuration deploys Keylime `agents` in the cluster, but the pods
+running it are unprivileged. The keylime `agent` needs access to
 the TPM device and to parts of the `securityfs` file system. Both of
 these are provided by the TPM device plugin, which is turned on
-automatically with unprivileged agent pods.
+automatically with unprivileged `agent` pods.
 
-In addition, the effective group ID of the keylime agent pod has to
+In addition, the effective group ID of the keylime `agent` pod has to
 match the group ID of the TPM device. Kubernetes does not allow
 `runAsGroup` to take symbolic values.
 
@@ -220,8 +220,6 @@ keylime-agent:
     runAsGroup: 109    <---- make this match the group ID of group <tss> on the hosts running the agent.
 ```
 
-
-
 ### Deploy with custom images (e.g. from a local registry)
 
 This configuration is for those of us debugging custom (self-built)
@@ -251,3 +249,28 @@ global:
         repository: localhost/custom-tenant-image
         tag: latest
 ```
+
+### Deploy multiple Verifiers and registrars
+
+For both availability and scalability reasons (scale-out), multiple `verifiers`
+and `registrars` can be deployed. Please do notice that this configuration
+cannot operate with a `sqlite` backend (i.e. either `mysql:external` or
+`mysql:enable` has to be set to `true `). When adding a new `agent` to a
+`verifier`, one of the multiple ones (exposed via multiple services, one per
+pod in the `StatefulSet`) has to be selected. The utility `kt`, used by `make
+helm-keylime-test` shows an example on how to pick a `verifier` based on an
+`agent` UUID.
+
+```
+global:
+  database:
+    mysql:
+      enable: true
+  service:
+    registrar:
+      replicas: 2
+      type: "LoadBalancer"
+    verifier:
+      replicas: 3
+      type: "LoadBalancer"    
+```

build/helm/keylime/charts/keylime-registrar/templates/_helpers.tpl

@@ -102,10 +102,10 @@ Expand to the replica count, which is conditional on both the value set on the "
 and "database" sections of global values
 */}}
 {{- define "registrar.replicaCount" -}}
-{{- if .Values.global.database.sqlite.enable }}
-{{- 1 }}
-{{- else }}
+{{- if or (eq .Values.global.database.mysql.external true) (eq .Values.global.database.mysql.enable true) }}
 {{- default 1 .Values.global.service.registrar.replicas }}
+{{- else }}
+{{- 1 }}
 {{- end }}
 {{- end }}
 

build/helm/keylime/charts/keylime-registrar/templates/deployment.yaml

@@ -33,7 +33,7 @@ spec:
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: '{{- include "registrar.image.repository" . }}:{{- include "registrar.image.tag" .}}'
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          imagePullPolicy: {{ include "registrar.image.pullPolicy" . }}
           ports:
             - name: registrar
               containerPort: {{ .Values.service.nontlsPort }}

build/helm/keylime/charts/keylime-tenant/templates/deployment.yaml

@@ -35,7 +35,7 @@ spec:
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: '{{- include "tenant.image.repository" . }}:{{- include "tenant.image.tag" .}}'
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          imagePullPolicy: {{ include "tenant.image.pullPolicy" . }}
           command:
             - /bin/bash
             - -c

build/helm/keylime/charts/keylime-verifier/templates/_helpers.tpl

@@ -102,10 +102,10 @@ Expand to the replica count, which is conditional on both the value set on the "
 and "database" sections of global values
 */}}
 {{- define "verifier.replicaCount" -}}
-{{- if .Values.global.database.sqlite.enable }}
-{{- 1 }}
-{{- else }}
+{{- if or (eq .Values.global.database.mysql.external true) (eq .Values.global.database.mysql.enable true) }}
 {{- default 1 .Values.global.service.verifier.replicas }}
+{{- else }}
+{{- 1 }}
 {{- end }}
 {{- end }}
 

build/helm/keylime/charts/keylime-verifier/templates/statefulset.yaml

@@ -31,10 +31,15 @@ spec:
           envFrom:
             - configMapRef:
                 name: {{ include "verifier.configMap" . }}
+          env:
+            - name: KEYLIME_VERIFIER_UUID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name                          
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: '{{- include "verifier.image.repository" . }}:{{- include "verifier.image.tag" .}}'
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          imagePullPolicy: {{ include "verifier.image.pullPolicy" . }}
           ports:
             - name: verifier
               containerPort: {{ .Values.service.port }}

build/helm/keylime/templates/configmap.yaml

@@ -7,7 +7,7 @@ metadata:
     {{- include "keylime.labels" . | nindent 4 }}
 data:
 {{- if or .Values.global.database.mysql.enable .Values.global.database.mysql.external }}
-  {{- if .Values.global.database.mysql.external }}  
+  {{- if .Values.global.database.mysql.external }}
   KEYLIME_REGISTRAR_DATABASE_URL: "mysql+pymysql://{{ .Values.mysql.auth.externalUser }}:{{ .Values.mysql.auth.externalPassword }}@{{ .Values.mysql.auth.externalIP }}:3306/{{ .Values.mysql.auth.database }}?charset=utf8"
   KEYLIME_VERIFIER_DATABASE_URL: "mysql+pymysql://{{ .Values.mysql.auth.externalUser }}:{{ .Values.mysql.auth.externalPassword }}@{{ .Values.mysql.auth.externalIP }}:3306/{{ .Values.mysql.auth.database }}?charset=utf8"
   {{- else }}

build/helm/keylime/values.yaml

@@ -131,7 +131,7 @@ global:
     # mysql enables a MySQL database backend
     mysql:
       # enable activates the MySQL database backend
-      # This will pull in a MySQL helm chart for deployment.
+      # This will pull in a MySQL helm chart for deployment. IMPORTANT: this will override sqlite (enabled by default)
       enable: false
       # leave it empty, and a new password, maintained accross multiple upgrades, will be generated
       password: ""

hack/k8s-poc/admin/kt

@@ -11,13 +11,34 @@ export KEYLIME_NAMESPACE=keylime
 #export KEYLIME_NAMESPACE=REPLACE_KEYLIME_NAMESPACE
 #export KUBECONFIG=REPLACE_HELM_CHART_KUBECONFIG
 
+KEYLIME_VERIFIER_SERVICE_PREFIX=$(kubectl get services --no-headers -l '!statefulset.kubernetes.io/pod-name,app.kubernetes.io/name=keylime-verifier' -n $KEYLIME_NAMESPACE -o json | jq -r '.items[].metadata.name')
+KEYLIME_VERIFIER_SERVICE_DOMAIN=$(kubectl get cm coredns -n kube-system -o jsonpath="{.data.Corefile}" | grep ".local " | awk -F ' ' '{print $2}')
+
 KEYLIME_TENANT_POD=$(kubectl get pods --namespace ${KEYLIME_NAMESPACE} | grep tenant | awk '{ print $1 }')
 if [[ -z $KEYLIME_TENANT_POD ]]
 then
     echo "ERROR: unable to find tenant pod on namespace ${KEYLIME_NAMESPACE}"
     exit 1
 fi
 
+KEYLIME_NUMBER_OF_VERIFIERS=$(kubectl get services --no-headers -l 'app.kubernetes.io/name=keylime-verifier,statefulset.kubernetes.io/pod-name' -n $KEYLIME_NAMESPACE | wc -l)
+
+function verifier_from_agent_uuid {
+    local _number_of_verifiers=$1
+    local _agent_uuid=$2
+
+    _agent_hash=$(echo $((0x$(sha1sum <<<"${_agent_uuid}")0)))
+
+    if [[ ${_agent_hash} -lt 0 ]]
+    then
+        _agent_hash=$((-_agent_hash))
+    fi
+
+    _verifier_nr=$((_agent_hash%_number_of_verifiers))
+
+    echo $KEYLIME_VERIFIER_SERVICE_PREFIX-${_verifier_nr}.${KEYLIME_NAMESPACE}.svc.${KEYLIME_VERIFIER_SERVICE_DOMAIN}
+}
+
 KEYLIME_RECEIVED_COMMAND="$*"
 
 while [[ $# -gt 0 ]]
@@ -85,7 +106,7 @@ then
     echo "#### Uploading files $KEYLIME_FILE_LIST to pod..." 
     for klf in $(echo $KEYLIME_FILE_LIST | sed 's/,/ /g')
     do
-    kubectl cp $klf ${KEYLIME_NAMESPACE}/$KEYLIME_TENANT_POD:$klf
+        kubectl cp $klf ${KEYLIME_NAMESPACE}/$KEYLIME_TENANT_POD:$klf
     done
 fi
 
@@ -96,7 +117,14 @@ then
         KEYLIME_TENANT_COMMAND="for uuid in $(echo ${_all_agent_uuids} | sed 's/,/ /g'); do /usr/local/bin/keylime_tenant $(echo $KEYLIME_RECEIVED_COMMAND | sed 's/deleteall/delete/g') -u \\\${uuid}; done"
     elif echo $KEYLIME_RECEIVED_COMMAND | grep -q addall
     then
-        KEYLIME_TENANT_COMMAND="for uuid in $(echo ${_all_agent_uuids} | sed 's/,/ /g'); do /usr/local/bin/keylime_tenant $(echo $KEYLIME_RECEIVED_COMMAND | sed 's/addall/add/g') -u \\\${uuid}; done"
+        for uuid in $(echo ${_all_agent_uuids} | sed 's/,/ /g')
+        do
+            _ktc="/usr/local/bin/keylime_tenant $(echo $KEYLIME_RECEIVED_COMMAND | sed 's/addall/add/g') -u ${uuid} -v $(verifier_from_agent_uuid $KEYLIME_NUMBER_OF_VERIFIERS $KEYLIME_TENANT_AGENT_UUID); "${_ktc}
+        done
+        KEYLIME_TENANT_COMMAND=${_ktc}/bin/true
+    elif echo $KEYLIME_RECEIVE_COMMAND | grep -q add
+    then
+        KEYLIME_TENANT_COMMAND=$KEYLIME_TENANT_COMMAND" -v $(verifier_from_agent_uuid $KEYLIME_NUMBER_OF_VERIFIERS $KEYLIME_TENANT_AGENT_UUID)"
     else
         KEYLIME_TENANT_COMMAND="/usr/local/bin/keylime_tenant $KEYLIME_RECEIVED_COMMAND"
     fi