Merge pull request #8584 from zalando-incubator/beta-to-stable

beta to stable

cluster/config-defaults.yaml

@@ -484,9 +484,6 @@ kubernetes_lifecycle_metrics_mem_min: "120Mi"
 kube_node_ready_controller_cpu: "50m"
 kube_node_ready_controller_memory: "200Mi"
 
-# Enable kube-node-ready ASG lifecycle hook feature.
-kube_node_ready_enabled: "true"
-
 # Enable deployment of aws-cloud-controller-manager
 aws_cloud_controller_manager_enabled: "true"
 aws_cloud_controller_manager_cpu: "125m"
@@ -677,9 +674,9 @@ teapot_admission_controller_configmap_deletion_protection_factories_enabled: "tr
 # enable the rolebinding admission-controller webhook which validates rolebindings and clusterrolebindings
 teapot_admission_controller_enable_rolebinding_webhook: "true"
 
-# enable the generic admission-controller webhook which catches all resources
-teapot_admission_controller_enable_generic_webhook: "false"
-# prevent write operations for non-admin users in protected namespaces
+# enable the generic deny-all admission webhook which rejects all requests it receives
+teapot_admission_controller_enable_write_protection_webhook: "false"
+# configure the behaviour of the deny-all admission webhook, `true` blocks everything, `false` allows everything
 teapot_admission_controller_prevent_write_operations: "false"
 
 # Enable and configure Pod Security Policy rules implemented in admission-controller.
@@ -778,8 +775,6 @@ kube2iam_cpu: "25m"
 kube2iam_memory: "100Mi"
 
 # configure whether kube2iam should only run on worker nodes.
-# This depends on control_plane_asg_lifecycle_hook=false as kube-node-ready
-# doesn't work without kube2iam.
 kube2iam_worker_only: "true"
 
 # CIDR configuration for nodes and pods
@@ -1135,9 +1130,6 @@ apiserver_memory_limit_percent: "80"
 
 apiserver_max_requests_inflight: "400"
 
-# specify if control plane nodes should rely on ASG Lifecycle Hook or not
-control_plane_asg_lifecycle_hook: "false"
-
 # enable graceful shutdown on the control_plane nodes
 control_plane_graceful_shutdown: "true"
 

cluster/manifests/01-admission-control/teapot.yaml

@@ -267,10 +267,34 @@ webhooks:
         apiVersions: ["v1"]
         resources: ["rolebindings", "clusterrolebindings"]
 {{- end }}
-{{- if eq .Cluster.ConfigItems.teapot_admission_controller_enable_generic_webhook "true" }}
-  - name: generic-namespaced-admitter.teapot.zalan.do
+{{- if eq .Cluster.ConfigItems.teapot_admission_controller_enable_write_protection_webhook "true" }}
+  - name: pod-exec-admitter.teapot.zalan.do
     clientConfig:
-      url: "https://localhost:8085/generic"
+      url: "https://localhost:8085/pod/exec"
+      caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}"
+    admissionReviewVersions: ["v1beta1"]
+    failurePolicy: Fail
+    sideEffects: "NoneOnDryRun"
+    matchPolicy: Equivalent
+    namespaceSelector:
+      matchExpressions:
+        - key: kubernetes.io/metadata.name
+          operator: NotIn
+          values: [ "kube-system", "visibility", "kubenurse" ]
+    rules:
+      - operations: [ "CONNECT" ]
+        apiGroups: [""]
+        apiVersions: ["v1"]
+        resources: ["pods/exec"]
+        scope: "Namespaced"
+    matchConditions:
+      - name: 'exclude-privileged-groups'
+        expression: 'request.userInfo.groups.all(g, !(g in ["okta:common/administrator", "zalando:administrator"]))'
+      - name: 'exclude-postgres-admins'
+        expression: 'request.userInfo.groups.all(g, !(g in ["okta:common/postgres-admin"]))'
+  - name: namespaced-deny-admitter.teapot.zalan.do
+    clientConfig:
+      url: "https://localhost:8085/deny"
       caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}"
     admissionReviewVersions: ["v1beta1"]
     failurePolicy: Fail
@@ -287,9 +311,16 @@ webhooks:
         apiVersions: ["*"]
         resources: ["*/*"]
         scope: "Namespaced"
-  - name: generic-cluster-admitter.teapot.zalan.do
+    matchConditions:
+      - name: 'exclude-privileged-groups'
+        expression: 'request.userInfo.groups.all(g, !(g in ["system:masters", "system:nodes", "system:serviceaccounts:kube-system", "okta:common/administrator", "zalando:administrator"]))'
+      - name: 'exclude-privileged-usernames'
+        expression: '!(request.userInfo.username in ["system:kube-controller-manager", "system:kube-scheduler", "zalando-iam:zalando:service:k8sapi_credentials-provider"])'
+      - name: 'exclude-eks-components'
+        expression: '!request.userInfo.username.startsWith("eks:")'
+  - name: global-deny-admitter.teapot.zalan.do
     clientConfig:
-      url: "https://localhost:8085/generic"
+      url: "https://localhost:8085/deny"
       caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}"
     admissionReviewVersions: ["v1beta1"]
     failurePolicy: Fail
@@ -304,4 +335,11 @@ webhooks:
         apiVersions: ["*"]
         resources: ["*/*"]
         scope: "Cluster"
+    matchConditions:
+      - name: 'exclude-privileged-groups'
+        expression: 'request.userInfo.groups.all(g, !(g in ["system:masters", "system:nodes", "system:serviceaccounts:kube-system", "okta:common/administrator", "zalando:administrator"]))'
+      - name: 'exclude-privileged-usernames'
+        expression: '!(request.userInfo.username in ["system:kube-controller-manager", "system:kube-scheduler", "zalando-iam:zalando:service:k8sapi_credentials-provider"])'
+      - name: 'exclude-eks-components'
+        expression: '!request.userInfo.username.startsWith("eks:")'
 {{- end }}

cluster/manifests/aws-cloud-controller-manager/daemonset.yaml

@@ -27,7 +27,7 @@ spec:
         - --cloud-provider=aws
         - --use-service-account-credentials=true
         - --configure-cloud-routes=false
-        image: container-registry.zalando.net/teapot/aws-cloud-controller-manager-internal:v1.31.1-master-132
+        image: container-registry.zalando.net/teapot/aws-cloud-controller-manager-internal:v1.31.1-master-133
         name: aws-cloud-controller-manager
         resources:
           requests:

cluster/manifests/coredns-local/daemonset-coredns.yaml

@@ -39,7 +39,7 @@ spec:
       containers:
 {{ if eq .Cluster.ConfigItems.dns_cache "unbound" }}
       - name: unbound
-        image: container-registry.zalando.net/teapot/unbound:1.19.2-master-9
+        image: container-registry.zalando.net/teapot/unbound:1.22.0-master-10
         args:
         - -d
         - -c

cluster/manifests/deletions.yaml

@@ -309,17 +309,6 @@ post_apply:
   kind: DaemonSet
   namespace: kube-system
 {{- end }}
-{{- if ne .Cluster.ConfigItems.kube_node_ready_enabled "true" }}
-- name: kube-node-ready
-  kind: DaemonSet
-  namespace: kube-system
-- name: kube-node-ready
-  kind: ServiceAccount
-  namespace: kube-system
-- name: kube-node-ready
-  kind: Service
-  namespace: kube-system
-{{- end }}
 {{- if ne .Cluster.ConfigItems.role_sync_controller_enabled "true" }}
 - name: role-sync-controller
   kind: CronJob

cluster/manifests/deployment-service/controller-statefulset.yaml

@@ -29,7 +29,7 @@ spec:
       terminationGracePeriodSeconds: 300
       containers:
         - name: "deployment-service-controller"
-          image: "container-registry.zalando.net/teapot/deployment-controller:master-232"
+          image: "container-registry.zalando.net/teapot/deployment-controller:master-233"
           args:
             - "--config-namespace=kube-system"
             - "--decrypt-kms-alias-arn=arn:aws:kms:{{ .Cluster.Region }}:{{ .Cluster.InfrastructureAccount | getAWSAccountID }}:alias/deployment-secret"

cluster/manifests/deployment-service/status-service-deployment.yaml

@@ -1,4 +1,4 @@
-# {{ $image   := "container-registry.zalando.net/teapot/deployment-status-service:master-232" }}
+# {{ $image   := "container-registry.zalando.net/teapot/deployment-status-service:master-233" }}
 # {{ $version := index (split $image ":") 1 }}
 
 apiVersion: apps/v1

cluster/manifests/kube-metrics-adapter/deployment.yaml

@@ -27,7 +27,7 @@ spec:
       serviceAccountName: custom-metrics-apiserver
       containers:
       - name: kube-metrics-adapter
-        image: container-registry.zalando.net/teapot/kube-metrics-adapter:kube-metrics-adapter-0.2.3-31-g9ff49a9
+        image: container-registry.zalando.net/teapot/kube-metrics-adapter:kube-metrics-adapter-0.2.3-35-g498f85e
         env:
         - name: AWS_REGION
           value: {{ .Cluster.Region }}

cluster/manifests/kube2iam/daemonset.yaml

@@ -21,7 +21,7 @@ spec:
       annotations:
         logging/destination: "{{.Cluster.ConfigItems.log_destination_infra}}"
     spec:
-{{- if and (eq .Cluster.ConfigItems.kube2iam_worker_only "true") (eq .Cluster.ConfigItems.control_plane_asg_lifecycle_hook "false") }}
+{{- if eq .Cluster.ConfigItems.kube2iam_worker_only "true" }}
       nodeSelector:
         node.kubernetes.io/role: worker
 {{- end }}

cluster/node-pools/master-default/stack.yaml

@@ -101,13 +101,3 @@ Resources:
       Roles:
       - !ImportValue '{{ .Cluster.ID }}:master-iam-role'
     Type: 'AWS::IAM::InstanceProfile'
-{{- if eq .Cluster.ConfigItems.control_plane_asg_lifecycle_hook "true" }}
-  AutoscalingLifecycleHook:
-    Properties:
-      AutoScalingGroupName: !Ref AutoScalingGroup
-      LifecycleHookName: "kube-node-ready-lifecycle-hook"
-      DefaultResult: CONTINUE
-      HeartbeatTimeout: '600'
-      LifecycleTransition: 'autoscaling:EC2_INSTANCE_LAUNCHING'
-    Type: 'AWS::AutoScaling::LifecycleHook'
-{{- end }}

cluster/node-pools/master-default/userdata.yaml

@@ -4,7 +4,7 @@ write_files:
     path: /etc/kubernetes/secrets.env
     content: |
       NODEPOOL_TAINTS=node.kubernetes.io/role=master:NoSchedule{{if index .NodePool.ConfigItems "taints"}},{{.NodePool.ConfigItems.taints}}{{end}}
-      NODE_LABELS=master=true,node.kubernetes.io/exclude-from-external-load-balancers,node.kubernetes.io/distro=ubuntu,cluster-lifecycle-controller.zalan.do/decommission-priority=999,lifecycle-status=ready{{if index .NodePool.ConfigItems "labels"}},{{.NodePool.ConfigItems.labels}}{{end}}{{if eq .Cluster.ConfigItems.control_plane_asg_lifecycle_hook "true" }},asg-lifecycle-hook=true{{end}}
+      NODE_LABELS=master=true,node.kubernetes.io/exclude-from-external-load-balancers,node.kubernetes.io/distro=ubuntu,cluster-lifecycle-controller.zalan.do/decommission-priority=999,lifecycle-status=ready{{if index .NodePool.ConfigItems "labels"}},{{.NodePool.ConfigItems.labels}}{{end}}
       NODEPOOL_NAME={{ .NodePool.Name }}
       KUBELET_ROLE=master
 
@@ -206,7 +206,7 @@ write_files:
             limits:
               memory: {{ .Values.InstanceInfo.MemoryFraction (parseInt64 .Cluster.ConfigItems.apiserver_memory_limit_percent)}}
 {{- end }}
-        - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-224
+        - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-228
           name: admission-controller
           lifecycle:
             preStop:
@@ -357,7 +357,7 @@ write_files:
           - mountPath: /etc/kubernetes/k8s-authnz-webhook-kubeconfig
             name: k8s-authnz-webhook-kubeconfig
             readOnly: true
-        - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/foundation/platform-iam-tokeninfo:master-130
+        - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/foundation/platform-iam-tokeninfo:master-131
           name: tokeninfo
           ports:
             - containerPort: 9021
@@ -388,7 +388,7 @@ write_files:
               value: {{ .Cluster.ConfigItems.apiserver_business_partner_ids }}
 {{ if ne .Cluster.Environment "production" }}
         - name: tokeninfo-sandbox
-          image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/foundation/platform-iam-tokeninfo:master-130
+          image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/foundation/platform-iam-tokeninfo:master-131
           ports:
           - containerPort: 9022
           lifecycle:
@@ -600,7 +600,7 @@ write_files:
         containers:
         - name: kube-controller-manager
 {{- if eq .Cluster.ConfigItems.kubernetes_controller_manager_image "zalando" }}
-          image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/kube-controller-manager-internal:v1.31.2-master-132
+          image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/kube-controller-manager-internal:v1.31.3-master-133
 {{- else }}
           image: nonexistent.zalan.do/teapot/kube-controller-manager:fixed
 {{- end }}

cluster/node-pools/worker-combined/stack.yaml

@@ -174,13 +174,3 @@ Resources:
       Roles:
       - !ImportValue '{{ .Cluster.ID }}:worker-iam-role'
     Type: 'AWS::IAM::InstanceProfile'
-# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }}
-  AutoscalingLifecycleHook:
-    Properties:
-      AutoScalingGroupName: !Ref AutoScalingGroup
-      LifecycleHookName: "kube-node-ready-lifecycle-hook"
-      DefaultResult: CONTINUE
-      HeartbeatTimeout: '600'
-      LifecycleTransition: 'autoscaling:EC2_INSTANCE_LAUNCHING'
-    Type: 'AWS::AutoScaling::LifecycleHook'
-# {{ end }}

cluster/node-pools/worker-karpenter/provisioners.yaml

@@ -124,7 +124,7 @@ spec:
 #  {{ end}}
 #{{ end}}
 
-#{{ if and (eq (len .NodePool.InstanceTypes) 1) (eq (index .NodePool.InstanceTypes 0) "default-for-karpenter") }}
+#{{ if (eq .NodePool.KarpenterInstanceTypeStrategy "default-for-karpenter" )  }}
         - key: "karpenter.k8s.aws/instance-family"
           operator: In
           values:
@@ -155,7 +155,7 @@ spec:
             - "c7in"
             - "m7in"
             - "r7in"
-#{{ else if (gt (len .NodePool.InstanceTypes) 0) }}
+#{{ else if (eq .NodePool.KarpenterInstanceTypeStrategy "custom" ) }}
         - key: "node.kubernetes.io/instance-type"
           operator: In
           values:
@@ -165,7 +165,7 @@ spec:
 #{{ end }}
 
 # safety guards to prevent the use of unwanted instance types in case the user has not specified any specific instance types
-#{{ if or (eq .NodePool.KarpenterInstanceTypeStrategy "default-for-karpenter") (eq .NodePool.KarpenterInstanceTypeStrategy "not-specified") }}
+#{{ if ne .NodePool.KarpenterInstanceTypeStrategy "custom" }}
         # exclude unwanted sizes
         - key: "karpenter.k8s.aws/instance-size"
           operator: "NotIn"

cluster/node-pools/worker-splitaz/stack.yaml

@@ -125,16 +125,6 @@ Resources:
       VPCZoneIdentifier:
         - "{{ index $data.Values.subnets $az }}"
     Type: 'AWS::AutoScaling::AutoScalingGroup'
-# {{ if eq $data.Cluster.ConfigItems.kube_node_ready_enabled "true" }}
-  AutoscalingLifecycleHook{{$azID}}:
-    Properties:
-      AutoScalingGroupName: !Ref AutoScalingGroup{{$azID}}
-      LifecycleHookName: "kube-node-ready-lifecycle-hook"
-      DefaultResult: CONTINUE
-      HeartbeatTimeout: '600'
-      LifecycleTransition: 'autoscaling:EC2_INSTANCE_LAUNCHING'
-    Type: 'AWS::AutoScaling::LifecycleHook'
-# {{ end }}
 {{ end }}
 {{ end }}
 {{ end }}