Make Pod securityContext configurable in antrea Helm chart (#5718)

When running on some K8s distributions, users may want to adjust the securityContext for antrea-agent containers. This is reserved for "power users", and most users should not modify the default values. When modifying the securityContext, some Antrea functions may break. For #5707 Signed-off-by: Antonin Bas <[email protected]>
antrea-io · Nov 20, 2023 · ac314f0 · ac314f0
1 parent ed362ce
commit ac314f0
Show file tree

Hide file tree

Showing 8 changed files with 83 additions and 25 deletions.
diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md
@@ -25,16 +25,25 @@ Kubernetes: `>= 1.16.0-0`
 | agent.antreaAgent.logFileMaxNum | int | `4` | Max number of log files. |
 | agent.antreaAgent.logFileMaxSize | int | `100` | Max size in MBs of any single log file. |
 | agent.antreaAgent.resources | object | `{"requests":{"cpu":"200m"}}` | Resource requests and limits for the antrea-agent container. |
+| agent.antreaAgent.securityContext.capabilities | list | `[]` | Capabilities for the antrea-agent container. |
+| agent.antreaAgent.securityContext.privileged | bool | `true` | Run the antrea-agent container as privileged. Currently we require this to be true (for sysctl configurations), but we may support running as non-privileged in the future. |
 | agent.antreaIPsec.resources | object | `{"requests":{"cpu":"50m"}}` | Resource requests and limits for the antrea-ipsec container (when IPsec is enabled). |
+| agent.antreaIPsec.securityContext.capabilities | list | `["NET_ADMIN"]` | Capabilities for the antrea-ipsec container. |
+| agent.antreaIPsec.securityContext.privileged | bool | `false` | Run the antrea-ipsec container as privileged. |
 | agent.antreaOVS.extraArgs | list | `[]` | Extra command-line arguments for antrea-ovs. |
 | agent.antreaOVS.logFileMaxNum | int | `4` | Max number of log files. |
 | agent.antreaOVS.logFileMaxSize | int | `100` | Max size in MBs of any single log file. |
 | agent.antreaOVS.resources | object | `{"requests":{"cpu":"200m"}}` | Resource requests and limits for the antrea-ovs container. |
+| agent.antreaOVS.securityContext.capabilities | list | `["SYS_NICE","NET_ADMIN","SYS_ADMIN","IPC_LOCK"]` | Capabilities for the antrea-ovs container. |
+| agent.antreaOVS.securityContext.privileged | bool | `false` | Run the antrea-ovs container as privileged. |
 | agent.apiPort | int | `10350` | Port for the antrea-agent APIServer to serve on. |
 | agent.dnsPolicy | string | `""` | DNS Policy for the antrea-agent Pods. If empty, the Kubernetes default will be used. |
 | agent.enablePrometheusMetrics | bool | `true` | Enable metrics exposure via Prometheus. |
 | agent.extraVolumes | list | `[]` | Additional volumes for antrea-agent Pods. |
+| agent.installCNI.extraEnv | object | `{}` | Extra environment variables to be injected into install-cni. |
 | agent.installCNI.resources | object | `{"requests":{"cpu":"100m"}}` | Resource requests and limits for the install-cni initContainer. |
+| agent.installCNI.securityContext.capabilities | list | `["SYS_MODULE"]` | Capabilities for the install-cni initContainer. |
+| agent.installCNI.securityContext.privileged | bool | `false` | Run the install-cni container as privileged. |
 | agent.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for the antrea-agent Pods. |
 | agent.podAnnotations | object | `{}` | Annotations to be added to antrea-agent Pods. |
 | agent.podLabels | object | `{}` | Labels to be added to antrea-agent Pods. |

diff --git a/build/charts/antrea/templates/agent/daemonset.yaml b/build/charts/antrea/templates/agent/daemonset.yaml
@@ -93,12 +93,23 @@ spec:
           {{- else }}
           command: ["install_cni"]
           {{- end }}
+          {{- with .Values.agent.installCNI.securityContext }}
           securityContext:
+            {{- if .privileged }}
+            privileged: true
+            {{- else }}
+            {{- with .capabilities }}
             capabilities:
               add:
-                # SYS_MODULE is required to load the OVS kernel module.
-                - SYS_MODULE
+                {{- toYaml . | nindent 16 }}
+            {{- end }}
+            {{- end }}
+          {{- end }}
           env:
+            {{- range $k, $v := .Values.agent.installCNI.extraEnv }}
+            - name: {{ $k | quote }}
+              value: {{ $v | quote }}
+            {{- end }}
             # SKIP_CNI_BINARIES takes in values as a comma separated list of
             # binaries that need to be skipped for installation, e.g. "portmap, bandwidth".
             - name: SKIP_CNI_BINARIES
@@ -200,9 +211,18 @@ spec:
             # scenario, otherwise the DaemonSet controller would restart all agents at once, as opposed to performing a
             # rolling update. Set failureThreshold to 8 so it can tolerate 70s of disconnection.
             failureThreshold: 8
+          {{- with .Values.agent.antreaAgent.securityContext }}
           securityContext:
-            # antrea-agent needs to perform sysctl configuration.
+            {{- if .privileged }}
             privileged: true
+            {{- else }}
+            {{- with .capabilities }}
+            capabilities:
+              add:
+                {{- toYaml . | nindent 16 }}
+            {{- end }}
+            {{- end }}
+          {{- end }}
           volumeMounts:
           - name: antrea-config
             mountPath: /etc/antrea/antrea-agent.conf
@@ -263,14 +283,18 @@ spec:
             {{- with .Values.agent.antreaOVS.extraArgs }}
             {{- toYaml . | trim | nindent 12 }}
             {{- end }}
+          {{- with .Values.agent.antreaOVS.securityContext }}
           securityContext:
-            # capabilities required by OVS daemons
+            {{- if .privileged }}
+            privileged: true
+            {{- else }}
+            {{- with .capabilities }}
             capabilities:
               add:
-                - SYS_NICE
-                - NET_ADMIN
-                - SYS_ADMIN
-                - IPC_LOCK
+                {{- toYaml . | nindent 16 }}
+            {{- end }}
+            {{- end }}
+          {{- end }}
           livenessProbe:
             exec:
               # docker CRI doesn't honor timeoutSeconds, add "timeout" to the command as a workaround.
@@ -304,10 +328,18 @@ spec:
                 - timeout 5 container_liveness_probe ovs-ipsec
             initialDelaySeconds: 5
             periodSeconds: 5
+          {{- with .Values.agent.antreaIPsec.securityContext }}
           securityContext:
+            {{- if .privileged }}
+            privileged: true
+            {{- else }}
+            {{- with .capabilities }}
             capabilities:
               add:
-              - NET_ADMIN
+                {{- toYaml . | nindent 16 }}
+            {{- end }}
+            {{- end }}
+          {{- end }}
           volumeMounts:
           - name: host-var-run-antrea
             mountPath: /var/run/openvswitch

diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml
@@ -235,10 +235,19 @@ agent:
   # -- Additional volumes for antrea-agent Pods.
   extraVolumes: []
   installCNI:
+    # -- Extra environment variables to be injected into install-cni.
+    extraEnv: {}
     # -- Resource requests and limits for the install-cni initContainer.
     resources:
       requests:
         cpu: "100m"
+    securityContext:
+      # -- Run the install-cni container as privileged.
+      privileged: false
+      # -- Capabilities for the install-cni initContainer.
+      capabilities:
+        # Used to load the Wireguard and OVS kernel modules.
+        - SYS_MODULE
   antreaAgent:
     # -- Extra environment variables to be injected into antrea-agent.
     extraEnv: {}
@@ -254,6 +263,13 @@ agent:
     resources:
       requests:
         cpu: "200m"
+    securityContext:
+      # -- Run the antrea-agent container as privileged. Currently we require
+      # this to be true (for sysctl configurations), but we may support running
+      # as non-privileged in the future.
+      privileged: true
+      # -- Capabilities for the antrea-agent container.
+      capabilities: []
   antreaOVS:
     # -- Max size in MBs of any single log file.
     logFileMaxSize: 100
@@ -265,12 +281,28 @@ agent:
     resources:
       requests:
         cpu: "200m"
+    securityContext:
+      # -- Run the antrea-ovs container as privileged.
+      privileged: false
+      # -- Capabilities for the antrea-ovs container.
+      capabilities:
+        # These are the capabilities required by the OVS daemons.
+        - SYS_NICE
+        - NET_ADMIN
+        - SYS_ADMIN
+        - IPC_LOCK
   antreaIPsec:
     # -- Resource requests and limits for the antrea-ipsec container (when IPsec
     # is enabled).
     resources:
       requests:
         cpu: "50m"
+    securityContext:
+      # -- Run the antrea-ipsec container as privileged.
+      privileged: false
+      # -- Capabilities for the antrea-ipsec container.
+      capabilities:
+        - NET_ADMIN
 
 controller:
   # -- Port for the antrea-controller APIServer to serve on.

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -6899,7 +6899,6 @@ spec:
           securityContext:
             capabilities:
               add:
-                # SYS_MODULE is required to load the OVS kernel module.
                 - SYS_MODULE
           env:
             # SKIP_CNI_BINARIES takes in values as a comma separated list of
@@ -6980,7 +6979,6 @@ spec:
             # rolling update. Set failureThreshold to 8 so it can tolerate 70s of disconnection.
             failureThreshold: 8
           securityContext:
-            # antrea-agent needs to perform sysctl configuration.
             privileged: true
           volumeMounts:
           - name: antrea-config
@@ -7025,7 +7023,6 @@ spec:
             - "--log_file_max_size=100"
             - "--log_file_max_num=4"
           securityContext:
-            # capabilities required by OVS daemons
             capabilities:
               add:
                 - SYS_NICE

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -6898,7 +6898,6 @@ spec:
           securityContext:
             capabilities:
               add:
-                # SYS_MODULE is required to load the OVS kernel module.
                 - SYS_MODULE
           env:
             # SKIP_CNI_BINARIES takes in values as a comma separated list of
@@ -6981,7 +6980,6 @@ spec:
             # rolling update. Set failureThreshold to 8 so it can tolerate 70s of disconnection.
             failureThreshold: 8
           securityContext:
-            # antrea-agent needs to perform sysctl configuration.
             privileged: true
           volumeMounts:
           - name: antrea-config
@@ -7026,7 +7024,6 @@ spec:
             - "--log_file_max_size=100"
             - "--log_file_max_num=4"
           securityContext:
-            # capabilities required by OVS daemons
             capabilities:
               add:
                 - SYS_NICE

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -6896,7 +6896,6 @@ spec:
           securityContext:
             capabilities:
               add:
-                # SYS_MODULE is required to load the OVS kernel module.
                 - SYS_MODULE
           env:
             # SKIP_CNI_BINARIES takes in values as a comma separated list of
@@ -6978,7 +6977,6 @@ spec:
             # rolling update. Set failureThreshold to 8 so it can tolerate 70s of disconnection.
             failureThreshold: 8
           securityContext:
-            # antrea-agent needs to perform sysctl configuration.
             privileged: true
           volumeMounts:
           - name: antrea-config
@@ -7023,7 +7021,6 @@ spec:
             - "--log_file_max_size=100"
             - "--log_file_max_num=4"
           securityContext:
-            # capabilities required by OVS daemons
             capabilities:
               add:
                 - SYS_NICE

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -6910,7 +6910,6 @@ spec:
           securityContext:
             capabilities:
               add:
-                # SYS_MODULE is required to load the OVS kernel module.
                 - SYS_MODULE
           env:
             # SKIP_CNI_BINARIES takes in values as a comma separated list of
@@ -6998,7 +6997,6 @@ spec:
             # rolling update. Set failureThreshold to 8 so it can tolerate 70s of disconnection.
             failureThreshold: 8
           securityContext:
-            # antrea-agent needs to perform sysctl configuration.
             privileged: true
           volumeMounts:
           - name: antrea-config
@@ -7046,7 +7044,6 @@ spec:
             - "--log_file_max_size=100"
             - "--log_file_max_num=4"
           securityContext:
-            # capabilities required by OVS daemons
             capabilities:
               add:
                 - SYS_NICE
@@ -7090,7 +7087,7 @@ spec:
           securityContext:
             capabilities:
               add:
-              - NET_ADMIN
+                - NET_ADMIN
           volumeMounts:
           - name: host-var-run-antrea
             mountPath: /var/run/openvswitch

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -6896,7 +6896,6 @@ spec:
           securityContext:
             capabilities:
               add:
-                # SYS_MODULE is required to load the OVS kernel module.
                 - SYS_MODULE
           env:
             # SKIP_CNI_BINARIES takes in values as a comma separated list of
@@ -6978,7 +6977,6 @@ spec:
             # rolling update. Set failureThreshold to 8 so it can tolerate 70s of disconnection.
             failureThreshold: 8
           securityContext:
-            # antrea-agent needs to perform sysctl configuration.
             privileged: true
           volumeMounts:
           - name: antrea-config
@@ -7023,7 +7021,6 @@ spec:
             - "--log_file_max_size=100"
             - "--log_file_max_num=4"
           securityContext:
-            # capabilities required by OVS daemons
             capabilities:
               add:
                 - SYS_NICE