Merge pull request #2481 from uselagoon/rootless

Implement rootless workloads
uselagoon · Oct 29, 2021 · 6068c29 · 6068c29
2 parents 604b600 + c85d083
commit 6068c29
Show file tree

Hide file tree

Showing 15 changed files with 70 additions and 23 deletions.
diff --git a/Makefile b/Makefile
@@ -1079,6 +1079,7 @@ kind/test: kind/cluster helm/repos $(addprefix local-dev/,$(KIND_TOOLS)) $(addpr
 			SKIP_INSTALL_REGISTRY=true \
 			LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
 			USE_CALICO_CNI=true \
+			LAGOON_FEATURE_FLAG_DEFAULT_ROOTLESS_WORKLOAD=enabled \
 		&& docker run --rm --network host --name ct-$(CI_BUILD_TAG) \
 			--volume "$$(pwd)/test-suite-run.ct.yaml:/etc/ct/ct.yaml" \
 			--volume "$$(pwd):/workdir" \
@@ -1202,6 +1203,7 @@ kind/retest:
 			SKIP_ALL_DEPS=true \
 			LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
 			USE_CALICO_CNI=true \
+			LAGOON_FEATURE_FLAG_DEFAULT_ROOTLESS_WORKLOAD=enabled \
 		&& docker run --rm --network host --name ct-$(CI_BUILD_TAG) \
 			--volume "$$(pwd)/test-suite-run.ct.yaml:/etc/ct/ct.yaml" \
 			--volume "$$(pwd):/workdir" \

diff --git a/docs/administering-lagoon/feature-flags.md b/docs/administering-lagoon/feature-flags.md
@@ -7,12 +7,10 @@ This is designed to assist users and administrators to roll out new platform fea
 
 The following environment variables can be set on an environment or project to toggle feature flags.
 
-| Environment Variable Name                      | Active scope\* | Version introduced | Version removed | Default Value | Description                                                                                                                                                                                                                                                                                                                                       |
-| ---                                            | ---            | ---                | ---             | ---           | ---                                                                                                                                                                                                                                                                                                                                               |
-| `LAGOON_FEATURE_FLAG_ROOTLESS_WORKLOAD`        | `global`        | 2.2.0              | -               | `disabled`    | Set to `enabled` to set a non-root pod security context on the pods in this environment or project.<br><br>This flag will eventually be deprecated, at which point non-root workloads will be enforced.                                                                                                                                                                                              |
-| `LAGOON_FEATURE_FLAG_ISOLATION_NETWORK_POLICY` | `global`        | 2.2.0              | -               | `disabled`    | Set to `enabled` to add a default namespace isolation network policy to each environment on deployment.<br><br>This flag will eventually be deprecated, at which point the namespace isolation network policy will be enforced.<br><br>NOTE: enabling and then disabling this feature will _not_ remove any existing network policy from previous deployments. Those must be removed manually. |
-
-\* Active scope indicates whether the variable must be set as `build` or `runtime` scope to take effect. `global` sets the variable in both scopes, so that will work too.
+| Environment Variable Name                      | Active scope | Version introduced | Version removed | Default Value | Description                                                                                                                                                                                                                                                                                                                                                                                    |
+| ---                                            | ---          | ---                | ---             | ---           | ---                                                                                                                                                                                                                                                                                                                                                                                            |
+| `LAGOON_FEATURE_FLAG_ROOTLESS_WORKLOAD`        | `global`     | 2.2.0              | -               | `disabled`    | Set to `enabled` to set a non-root pod security context on the pods in this environment or project.<br><br>This flag will eventually be deprecated, at which point non-root workloads will be enforced.                                                                                                                                                                                        |
+| `LAGOON_FEATURE_FLAG_ISOLATION_NETWORK_POLICY` | `global`     | 2.2.0              | -               | `disabled`    | Set to `enabled` to add a default namespace isolation network policy to each environment on deployment.<br><br>This flag will eventually be deprecated, at which point the namespace isolation network policy will be enforced.<br><br>NOTE: enabling and then disabling this feature will _not_ remove any existing network policy from previous deployments. Those must be removed manually. |
 
 ## Cluster-level controls
 

diff --git a/images/kubectl-build-deploy-dind/Dockerfile b/images/kubectl-build-deploy-dind/Dockerfile
@@ -13,6 +13,7 @@ WORKDIR /kubectl-build-deploy/git
 COPY docker-entrypoint.sh /lagoon/entrypoints/100-docker-entrypoint.sh
 COPY build-deploy.sh /kubectl-build-deploy/build-deploy.sh
 COPY build-deploy-docker-compose.sh /kubectl-build-deploy/build-deploy-docker-compose.sh
+COPY rootless.values.yaml /kubectl-build-deploy/rootless.values.yaml
 
 COPY scripts /kubectl-build-deploy/scripts
 

diff --git a/images/kubectl-build-deploy-dind/build-deploy-docker-compose.sh b/images/kubectl-build-deploy-dind/build-deploy-docker-compose.sh
@@ -29,22 +29,23 @@ function contains() {
 # and order:
 #
 # 1. The cluster-force feature flag, prefixed with LAGOON_FEATURE_FLAG_FORCE_,
-#    in the build environment. This is set via a flag on the build-deploy
-#    controller. This overrides the other variables and allows policy
-#    enforcement at the cluster level.
+#    as a build pod environment variable. This is set via a flag on the
+#    build-deploy controller. This overrides the other variables and allows
+#    policy enforcement at the cluster level.
 #
 # 2. The regular feature flag, prefixed with LAGOON_FEATURE_FLAG_, in the
-#    Lagoon environment env-vars. This allows policy control at the environment
-#    level.
+#    Lagoon environment global scoped env-vars. This allows policy control at
+#    the environment level.
 #
 # 3. The regular feature flag, prefixed with LAGOON_FEATURE_FLAG_, in the
-#    Lagoon project env-vars. This allows policy control at the project level.
+#    Lagoon project global scoped env-vars. This allows policy control at the
+#    project level.
 #
 # 4. The cluster-default feature flag, prefixed with
-#    LAGOON_FEATURE_FLAG_DEFAULT_, in the build environment. This is set via a
-#    flag on the build-deploy controller. This allows default policy to be set
-#    at the cluster level, but maintains the ability to selectively override at
-#    the project or environment level.
+#    LAGOON_FEATURE_FLAG_DEFAULT_, as a build pod environment variable. This is
+#    set via a flag on the build-deploy controller. This allows default policy
+#    to be set at the cluster level, but maintains the ability to selectively
+#    override at the project or environment level.
 #
 # The value of the first variable found is printed to stdout. If the variable
 # is not found, print an empty string. Additional arguments are ignored.
@@ -54,16 +55,16 @@ function featureFlag() {
 
 	local forceFlagVar defaultFlagVar flagVar
 
-	# check build environment for the force policy first
+	# check build pod environment for the force policy first
 	forceFlagVar="LAGOON_FEATURE_FLAG_FORCE_$1"
 	[ "${!forceFlagVar}" ] && echo "${!forceFlagVar}" && return
 
 	flagVar="LAGOON_FEATURE_FLAG_$1"
 	# check Lagoon environment variables
-	flagValue=$(jq -r '.[] | select((.scope as $scope | ["build", "global"] | index($scope)) and .name == "'"$flagVar"'") | .value' <<<"$LAGOON_ENVIRONMENT_VARIABLES")
+	flagValue=$(jq -r '.[] | select(.scope == "global" and .name == "'"$flagVar"'") | .value' <<<"$LAGOON_ENVIRONMENT_VARIABLES")
 	[ "$flagValue" ] && echo "$flagValue" && return
 	# check Lagoon project variables
-	flagValue=$(jq -r '.[] | select((.scope as $scope | ["build", "global"] | index($scope)) and .name == "'"$flagVar"'") | .value' <<<"$LAGOON_PROJECT_VARIABLES")
+	flagValue=$(jq -r '.[] | select(.scope == "global" and .name == "'"$flagVar"'") | .value' <<<"$LAGOON_PROJECT_VARIABLES")
 	[ "$flagValue" ] && echo "$flagValue" && return
 
 	# fall back to the default, if set.
@@ -610,6 +611,10 @@ yq write -i -- /kubectl-build-deploy/values.yaml 'routesAutogenerateShortSuffix'
 for i in $ROUTES_AUTOGENERATE_PREFIXES; do yq write -i -- /kubectl-build-deploy/values.yaml 'routesAutogeneratePrefixes[+]' $i; done
 yq write -i -- /kubectl-build-deploy/values.yaml 'kubernetes' $KUBERNETES
 yq write -i -- /kubectl-build-deploy/values.yaml 'lagoonVersion' $LAGOON_VERSION
+# check for ROOTLESS_WORKLOAD feature flag, disabled by default
+if [ "$(featureFlag ROOTLESS_WORKLOAD)" = enabled ]; then
+	yq merge -ix -- /kubectl-build-deploy/values.yaml /kubectl-build-deploy/rootless.values.yaml
+fi
 
 
 echo -e "\

diff --git a/images/kubectl-build-deploy-dind/helmcharts/kibana/templates/cronjob.yaml b/images/kubectl-build-deploy-dind/helmcharts/kibana/templates/cronjob.yaml
@@ -1,4 +1,5 @@
 {{- range $cronjobName, $cronjobConfig := .Values.nativeCronjobs }}
+---
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:

diff --git a/images/kubectl-build-deploy-dind/helmcharts/mariadb-single/templates/cronjob.yaml b/images/kubectl-build-deploy-dind/helmcharts/mariadb-single/templates/cronjob.yaml
@@ -35,7 +35,7 @@ spec:
           priorityClassName: {{ include "mariadb-single.lagoonPriority" $ }}
           enableServiceLinks: false
           securityContext:
-            {{- toYaml $.Values.podSecurityContext | nindent 12 }}
+            {{- toYaml .Values.podSecurityContext | nindent 12 }}
           volumes:
             - name: {{ include "mariadb-single.fullname" $ }}
               persistentVolumeClaim:

diff --git a/images/kubectl-build-deploy-dind/helmcharts/mariadb-single/templates/deployment.yaml b/images/kubectl-build-deploy-dind/helmcharts/mariadb-single/templates/deployment.yaml
@@ -35,6 +35,8 @@ spec:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       containers:
         - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image }}"
           imagePullPolicy: {{ .Values.imagePullPolicy }}
           env:

diff --git a/images/kubectl-build-deploy-dind/helmcharts/mongodb-single/templates/cronjob.yaml b/images/kubectl-build-deploy-dind/helmcharts/mongodb-single/templates/cronjob.yaml
@@ -35,7 +35,7 @@ spec:
           priorityClassName: {{ include "mongodb-single.lagoonPriority" $ }}
           enableServiceLinks: false
           securityContext:
-            {{- toYaml $.Values.podSecurityContext | nindent 12 }}
+            {{- toYaml .Values.podSecurityContext | nindent 12 }}
           volumes:
             - name: {{ include "mongodb-single.fullname" $ }}
               persistentVolumeClaim:

diff --git a/images/kubectl-build-deploy-dind/helmcharts/mongodb-single/templates/deployment.yaml b/images/kubectl-build-deploy-dind/helmcharts/mongodb-single/templates/deployment.yaml
@@ -35,6 +35,8 @@ spec:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       containers:
         - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image }}"
           imagePullPolicy: {{ .Values.imagePullPolicy }}
           env:

diff --git a/images/kubectl-build-deploy-dind/helmcharts/nginx-php-persistent/templates/deployment.yaml b/images/kubectl-build-deploy-dind/helmcharts/nginx-php-persistent/templates/deployment.yaml
@@ -34,6 +34,32 @@ spec:
       enableServiceLinks: false
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{/* This is a test for "rootless" configuration. It will be true if the ROOTLESS_WORKLOAD feature flag is enabled. */}}
+      {{- if eq (int .Values.podSecurityContext.fsGroup) 10001 }}
+      # This initContainer sets permissions on the RWX storage compatible with
+      # the pod securityContext.
+      initContainers:
+        - command:
+            - "sh"
+            - "-c"
+            - |
+              set -e
+              SENTINEL="/storage/.lagoon-rootless-migration-complete"
+              if ! [ -f "$SENTINEL" ]; then
+                find /storage -mindepth 1 -exec chgrp $(stat -c "%g" /storage) {} +
+                find /storage -mindepth 1 -exec chmod g+rw {} +
+                find /storage -mindepth 1 -type d -exec chmod g+x {} +
+              fi
+              touch "$SENTINEL"
+          image: busybox:musl
+          imagePullPolicy: IfNotPresent
+          name: fix-storage-permissions
+          securityContext:
+            runAsUser: 0
+          volumeMounts:
+            - mountPath: /storage
+              name: {{ include "nginx-php-persistent.persistentStorageName" . }}
+      {{- end }}
       containers:
         - image: {{ .Values.images.nginx | quote }}
           name: "nginx"

diff --git a/images/kubectl-build-deploy-dind/helmcharts/postgres-single/templates/cronjob.yaml b/images/kubectl-build-deploy-dind/helmcharts/postgres-single/templates/cronjob.yaml
@@ -35,7 +35,7 @@ spec:
           priorityClassName: {{ include "postgres-single.lagoonPriority" $ }}
           enableServiceLinks: false
           securityContext:
-            {{- toYaml $.Values.podSecurityContext | nindent 12 }}
+            {{- toYaml .Values.podSecurityContext | nindent 12 }}
           volumes:
             - name: {{ include "postgres-single.fullname" $ }}
               persistentVolumeClaim:

diff --git a/images/kubectl-build-deploy-dind/helmcharts/postgres-single/templates/deployment.yaml b/images/kubectl-build-deploy-dind/helmcharts/postgres-single/templates/deployment.yaml
@@ -35,6 +35,8 @@ spec:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       containers:
         - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image }}"
           imagePullPolicy: {{ .Values.imagePullPolicy }}
           env:

diff --git a/images/kubectl-build-deploy-dind/helmcharts/solr/templates/cronjob.yaml b/images/kubectl-build-deploy-dind/helmcharts/solr/templates/cronjob.yaml
@@ -35,7 +35,7 @@ spec:
           priorityClassName: {{ include "solr.lagoonPriority" $ }}
           enableServiceLinks: false
           securityContext:
-            {{- toYaml $.Values.podSecurityContext | nindent 12 }}
+            {{- toYaml .Values.podSecurityContext | nindent 12 }}
           containers:
             - image: {{ $.Values.image | quote }}
               name: cronjob-{{ $.Release.Name }}-{{ $cronjobName }}

diff --git a/images/kubectl-build-deploy-dind/helmcharts/solr/templates/deployment.yaml b/images/kubectl-build-deploy-dind/helmcharts/solr/templates/deployment.yaml
@@ -38,6 +38,8 @@ spec:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       containers:
         - image: {{ .Values.image | quote }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
           name: {{ .Chart.Name }}
           imagePullPolicy: {{ .Values.imagePullPolicy }}
           ports:

diff --git a/images/kubectl-build-deploy-dind/rootless.values.yaml b/images/kubectl-build-deploy-dind/rootless.values.yaml
@@ -0,0 +1,6 @@
+# This is used in build-deploy-docker-compose.sh to specify the rootless
+# podSecurityContext for Lagoon helmcharts.
+podSecurityContext:
+  fsGroup: 10001
+  runAsGroup: 0
+  runAsUser: 10000