Merge pull request #8568 from zalando-incubator/beta-to-stable

beta to stable

cluster/config-defaults.yaml

@@ -677,6 +677,11 @@ teapot_admission_controller_configmap_deletion_protection_factories_enabled: "tr
 # enable the rolebinding admission-controller webhook which validates rolebindings and clusterrolebindings
 teapot_admission_controller_enable_rolebinding_webhook: "true"
 
+# enable the generic admission-controller webhook which catches all resources
+teapot_admission_controller_enable_generic_webhook: "false"
+# prevent write operations for non-admin users in protected namespaces
+teapot_admission_controller_prevent_write_operations: "false"
+
 # Enable and configure Pod Security Policy rules implemented in admission-controller.
 teapot_admission_controller_pod_security_policy_enabled: "true"
 
@@ -1165,11 +1170,33 @@ control_plane_load_balancer_internal: "none"
 #  fs.inotify.max_user_watches = 100000
 sysctl_settings: ""
 
-
+# kube-janitor configuration
+{{if eq .Cluster.Environment "production"}}
+# This makes kube-janitor opt-in for production clusters
+
+# IMPORTANT:
+# Please note that before enabling kube-janitor for a production cluster, you
+# must ensure that no existing resources should be annotated with a TTL.
+# This can happen in the case where a test deployment is deployed to production
+# as is. Currently, it's a no-op since kube-janitor doesn't run in production.
+# 
+# This is needed until we can implement namespace prefix matching to reduce
+# the scope of kube-janitor to a set of namespace names that aren't known
+# at the time of enaling kube-janitor. Once the feature is in place, it would
+# be easier to limit the scope.
+kube_janitor_enabled: "false"
+{{else}}
+kube_janitor_enabled: "true"
+{{end}}
 
 # scheduling_controls
 teapot_admission_controller_scheduling_controls_enabled: "false"
 teapot_admission_controller_scheduling_controls_default_architecture: "amd64"
 
 # role-sync-controller configs
+# Enabled by default only on Zalando EKS clusters
+{{ if eq .Cluster.Provider "zalando-eks" }}
+role_sync_controller_enabled: "true"
+{{ else }}
 role_sync_controller_enabled: "false"
+{{ end }}

cluster/manifests/01-admission-control/config.yaml

@@ -12,6 +12,8 @@ data:
 
   dns.default.subdomain-max-length: "{{ .Cluster.ConfigItems.subdomain_max_length }}"
 
+  generic.prevent-write-operations.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_prevent_write_operations }}"
+
   pod.container-resource-control.min-memory-request: "25Mi"
   pod.container-resource-control.default-cpu-request: "{{ .Cluster.ConfigItems.teapot_admission_controller_default_cpu_request }}"
   pod.container-resource-control.default-memory-request: "{{ .Cluster.ConfigItems.teapot_admission_controller_default_memory_request }}"

cluster/manifests/01-admission-control/teapot.yaml

@@ -267,3 +267,41 @@ webhooks:
         apiVersions: ["v1"]
         resources: ["rolebindings", "clusterrolebindings"]
 {{- end }}
+{{- if eq .Cluster.ConfigItems.teapot_admission_controller_enable_generic_webhook "true" }}
+  - name: generic-namespaced-admitter.teapot.zalan.do
+    clientConfig:
+      url: "https://localhost:8085/generic"
+      caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}"
+    admissionReviewVersions: ["v1beta1"]
+    failurePolicy: Fail
+    sideEffects: "NoneOnDryRun"
+    matchPolicy: Equivalent
+    namespaceSelector:
+      matchExpressions:
+        - key: kubernetes.io/metadata.name
+          operator: In
+          values: [ "kube-system", "visibility", "kubenurse" ]
+    rules:
+      - operations: [ "*" ]
+        apiGroups: ["*"]
+        apiVersions: ["*"]
+        resources: ["*/*"]
+        scope: "Namespaced"
+  - name: generic-cluster-admitter.teapot.zalan.do
+    clientConfig:
+      url: "https://localhost:8085/generic"
+      caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}"
+    admissionReviewVersions: ["v1beta1"]
+    failurePolicy: Fail
+    sideEffects: "NoneOnDryRun"
+    matchPolicy: Equivalent
+    objectSelector:
+      matchLabels:
+        admission.zalando.org/infrastructure-component: "true"
+    rules:
+      - operations: [ "*" ]
+        apiGroups: ["*"]
+        apiVersions: ["*"]
+        resources: ["*/*"]
+        scope: "Cluster"
+{{- end }}

cluster/manifests/deletions.yaml

@@ -330,3 +330,21 @@ post_apply:
   kind: ServiceAccount
   namespace: kube-system
 {{- end }}
+{{- if ne .Cluster.ConfigItems.kube_janitor_enabled "true" }}
+- name: kube-janitor
+  kind: Deployment
+  namespace: kube-system
+- name: kube-janitor
+  kind: ConfigMap
+  namespace: kube-system
+- name: kube-janitor
+  kind: VerticalPodAutoscaler
+  namespace: kube-system
+- name: kube-janitor
+  kind: ServiceAccount
+  namespace: kube-system
+- name: kube-janitor
+  kind: ClusterRole
+- name: kube-janitor
+  kind: ClusterRoleBinding
+{{- end }}

cluster/manifests/deployment-service/controller-statefulset.yaml

@@ -29,7 +29,7 @@ spec:
       terminationGracePeriodSeconds: 300
       containers:
         - name: "deployment-service-controller"
-          image: "container-registry.zalando.net/teapot/deployment-controller:master-230"
+          image: "container-registry.zalando.net/teapot/deployment-controller:master-232"
           args:
             - "--config-namespace=kube-system"
             - "--decrypt-kms-alias-arn=arn:aws:kms:{{ .Cluster.Region }}:{{ .Cluster.InfrastructureAccount | getAWSAccountID }}:alias/deployment-secret"

cluster/manifests/deployment-service/status-service-deployment.yaml

@@ -1,4 +1,4 @@
-# {{ $image   := "container-registry.zalando.net/teapot/deployment-status-service:master-230" }}
+# {{ $image   := "container-registry.zalando.net/teapot/deployment-status-service:master-232" }}
 # {{ $version := index (split $image ":") 1 }}
 
 apiVersion: apps/v1

cluster/manifests/kube-janitor/deployment.yaml

@@ -1,4 +1,4 @@
-# {{ if ne .Cluster.Environment "production" }}
+# {{ if eq .Cluster.ConfigItems.kube_janitor_enabled "true" }}
 # {{ $image := "container-registry.zalando.net/teapot/kube-janitor:23.7.0-main-2" }}
 # {{ $version := index (split (index (split $image ":") 1) "-") 0 }}
 apiVersion: apps/v1

cluster/manifests/kube-janitor/rbac.yaml

@@ -1,4 +1,4 @@
-{{ if ne .Cluster.Environment "production" }}
+{{ if eq .Cluster.ConfigItems.kube_janitor_enabled "true" }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:

cluster/manifests/kube-janitor/rules-config.yaml

@@ -1,4 +1,4 @@
-# {{ if ne .Cluster.Environment "production" }}
+# {{ if eq .Cluster.ConfigItems.kube_janitor_enabled "true" }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

cluster/manifests/kube-janitor/vpa.yaml

@@ -1,4 +1,4 @@
-{{ if ne .Cluster.Environment "production" }}
+{{ if eq .Cluster.ConfigItems.kube_janitor_enabled "true" }}
 apiVersion: autoscaling.k8s.io/v1
 kind: VerticalPodAutoscaler
 metadata:

cluster/manifests/prometheus/rbac.yaml

@@ -8,6 +8,8 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: prometheus
+  labels:
+    admission.zalando.org/infrastructure-component: "true"
 rules:
 - apiGroups: [""]
   resources:
@@ -37,6 +39,8 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: prometheus
+  labels:
+    admission.zalando.org/infrastructure-component: "true"
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

cluster/manifests/role-sync-controller/cronjob.yaml

@@ -26,5 +26,18 @@ spec:
           restartPolicy: OnFailure
           containers:
           - name: role-sync-controller
-            image: container-registry.zalando.net/teapot/role-sync-controller:main-1
+            image: container-registry.zalando.net/teapot/role-sync-controller:main-2
+            args:
+              - --subject-group=PowerUser
+              - --subject-group=Manual
+              - --subject-group=Emergency
+              - --subject-group=okta:common/engineer
+              - --subject-serviceaccount=default/cdp
+              - --subject-user=zalando-iam:zalando:service:k8sapi-local_deployment-service-executor
+              {{- if eq .Cluster.Environment "test"}}
+              - --subject-group=CollaboratorPowerUser
+              {{- end}}
+              {{- if eq .Cluster.Provider "zalando-eks"}}
+              - --subject-serviceaccount=kube-system/deployment-service-controller
+              {{- end}}
 {{ end }}

cluster/manifests/roles/collaborator-roles.yaml

@@ -14,6 +14,16 @@ rules:
   - update
   - patch
   - delete
+{{ if eq .Cluster.ConfigItems.role_sync_controller_enabled "true" }}
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+{{ end }}
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

cluster/manifests/roles/poweruser-role.yaml

@@ -58,6 +58,7 @@ rules:
   - services/proxy
   verbs:
   - get
+{{ if ne .Cluster.ConfigItems.role_sync_controller_enabled "true" }}
 - apiGroups:
   - ''
   resources:
@@ -71,6 +72,7 @@ rules:
   - patch
   - update
   - watch
+{{ end }}
 - apiGroups:
   - ''
   - extensions

cluster/node-pools/master-default/userdata.yaml

@@ -206,7 +206,7 @@ write_files:
             limits:
               memory: {{ .Values.InstanceInfo.MemoryFraction (parseInt64 .Cluster.ConfigItems.apiserver_memory_limit_percent)}}
 {{- end }}
-        - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-222
+        - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-224
           name: admission-controller
           lifecycle:
             preStop: