From b8e6808ead7a2f48f9f9ddfd76f519632c6aee6b Mon Sep 17 00:00:00 2001 From: Avi Biton Date: Sun, 11 Aug 2024 15:01:53 +0300 Subject: [PATCH] update manifests for controller deployment This update handles both RHTAPWATCH-1208 and RHTAPWATCH-1209 Update the manifests for the controller deployment: - update the manager manifests - update rbac resources - patch to add kube-rbac-proxy to the manager - add service monitor to read the metrics Signed-off-by: Avi Biton --- config/default/kustomization.yaml | 144 ++---------------- config/default/manager_auth_proxy_patch.yaml | 36 +++++ config/default/manager_metrics_patch.yaml | 4 - config/manager/kustomization.yaml | 5 + config/manager/manager.yaml | 43 ++---- config/prometheus/kustomization.yaml | 1 + config/prometheus/monitor.yaml | 17 ++- config/prometheus/secret.yaml | 8 + .../rbac/auth_proxy_client_clusterrole.yaml | 9 ++ config/rbac/auth_proxy_role.yaml | 17 +++ config/rbac/auth_proxy_role_binding.yaml | 12 ++ .../auth_proxy_service.yaml} | 8 +- config/rbac/kustomization.yaml | 16 +- config/rbac/leader_election_role.yaml | 40 ----- config/rbac/leader_election_role_binding.yaml | 15 -- config/rbac/prometheus_role_binding.yaml | 12 ++ config/rbac/prometheus_service_account.yaml | 5 + config/rbac/prometheus_viewer_role.yaml | 9 ++ config/rbac/role_binding.yaml | 3 - config/rbac/service_account.yaml | 3 - 20 files changed, 164 insertions(+), 243 deletions(-) create mode 100644 config/default/manager_auth_proxy_patch.yaml delete mode 100644 config/default/manager_metrics_patch.yaml create mode 100644 config/prometheus/secret.yaml create mode 100644 config/rbac/auth_proxy_client_clusterrole.yaml create mode 100644 config/rbac/auth_proxy_role.yaml create mode 100644 config/rbac/auth_proxy_role_binding.yaml rename config/{default/metrics_service.yaml => rbac/auth_proxy_service.yaml} (61%) delete mode 100644 config/rbac/leader_election_role.yaml delete mode 100644 config/rbac/leader_election_role_binding.yaml create mode 100644 config/rbac/prometheus_role_binding.yaml create mode 100644 config/rbac/prometheus_service_account.yaml create mode 100644 config/rbac/prometheus_viewer_role.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 95e9784..7e2a2de 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,147 +1,21 @@ # Adds namespace to all resources. -namespace: notification-service-system +namespace: notification-controller # Value of this field is prepended to the # names of all resources, e.g. a deployment named # "wordpress" becomes "alices-wordpress". # Note that it should also match with the prefix (text before '-') of the namespace # field above. -namePrefix: notification-service- - -# Labels to add to all resources and selectors. -#labels: -#- includeSelectors: true -# pairs: -# someName: someValue +namePrefix: notification-controller- resources: -#- ../crd +# Add rbac resources - ../rbac +# Add manager manifest - ../manager -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- ../webhook -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager -# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus -# [METRICS] To enable the controller manager metrics service, uncomment the following line. -- metrics_service.yaml - -# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager -#patches: -# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. -# More info: https://book.kubebuilder.io/reference/metrics -# If you want to expose the metric endpoint of your controller-manager uncomment the following line. -- path: manager_metrics_patch.yaml - target: - kind: Deployment - -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- path: manager_webhook_patch.yaml - -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. -# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. -# 'CERTMANAGER' needs to be enabled to use ca injection -#- path: webhookcainjection_patch.yaml +# Enable prometheus monitor +- ../prometheus -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -# Uncomment the following replacements to add the cert-manager CA injection annotations -#replacements: -# - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.namespace # namespace of the certificate CR -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: # Add cert-manager annotation to the webhook Service -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true +patchesStrategicMerge: +# This patch inject a sidecar container which is a HTTP proxy for the controller manager +- manager_auth_proxy_patch.yaml diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 0000000..b291a02 --- /dev/null +++ b/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,36 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--http2-disable" + - "--v=0" + ports: + - containerPort: 8443 + protocol: TCP + name: https + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + securityContext: + readOnlyRootFilesystem: true + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" diff --git a/config/default/manager_metrics_patch.yaml b/config/default/manager_metrics_patch.yaml deleted file mode 100644 index 6c546ae..0000000 --- a/config/default/manager_metrics_patch.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# This patch adds the args to allow exposing the metrics endpoint securely -- op: add - path: /spec/template/spec/containers/0/args/0 - value: --metrics-bind-address=:8080 diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b8..71f3473 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,7 @@ resources: - manager.yaml + +images: +- name: controller + newName: quay.io/konflux-ci/notification-service + newTag: latest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 75672e4..a78931c 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -3,7 +3,7 @@ kind: Namespace metadata: labels: control-plane: controller-manager - app.kubernetes.io/name: notification-service + app.kubernetes.io/name: notification-controller app.kubernetes.io/managed-by: kustomize name: system --- @@ -14,7 +14,7 @@ metadata: namespace: system labels: control-plane: controller-manager - app.kubernetes.io/name: notification-service + app.kubernetes.io/name: notification-controller app.kubernetes.io/managed-by: kustomize spec: selector: @@ -28,45 +28,24 @@ spec: labels: control-plane: controller-manager spec: - # TODO(user): Uncomment the following code to configure the nodeAffinity expression - # according to the platforms which are supported by your solution. - # It is considered best practice to support multiple architectures. You can - # build your manager image using the makefile target docker-buildx. - # affinity: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: kubernetes.io/arch - # operator: In - # values: - # - amd64 - # - arm64 - # - ppc64le - # - s390x - # - key: kubernetes.io/os - # operator: In - # values: - # - linux + volumes: + - name: vol-secret + secret: + secretName: aws-sns-secret securityContext: runAsNonRoot: true - # TODO(user): For common cases that do not require escalating privileges - # it is recommended to ensure that all your Pods/Containers are restrictive. - # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted - # Please uncomment the following code if your project does NOT have to work on old Kubernetes - # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). - # seccompProfile: - # type: RuntimeDefault containers: - command: - /manager - args: - - --leader-elect - - --health-probe-bind-address=:8081 image: controller:latest name: manager + volumeMounts: + - name: vol-secret + mountPath: /.aws securityContext: + readOnlyRootFilesystem: true allowPrivilegeEscalation: false + runAsNonRoot: true capabilities: drop: - "ALL" diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index ed13716..93a7146 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,2 +1,3 @@ resources: - monitor.yaml +- secret.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index b63b64c..704eb8d 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -3,16 +3,25 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: labels: - control-plane: controller-manager - app.kubernetes.io/name: notification-service + app.kubernetes.io/name: servicemonitor + app.kubernetes.io/instance: controller-manager-metrics-monitor + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: notification-controller + app.kubernetes.io/part-of: notification-controller app.kubernetes.io/managed-by: kustomize + control-plane: controller-manager name: controller-manager-metrics-monitor namespace: system spec: endpoints: - path: /metrics - port: http # Ensure this is the name of the port that exposes HTTP metrics - scheme: http + port: https + scheme: https + bearerTokenSecret: + key: token + name: "notification-controller-metrics-reader" + tlsConfig: + insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager diff --git a/config/prometheus/secret.yaml b/config/prometheus/secret.yaml new file mode 100644 index 0000000..c2aee2f --- /dev/null +++ b/config/prometheus/secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: metrics-reader + namespace: system + annotations: + kubernetes.io/service-account.name: notification-controller-metrics-reader +type: kubernetes.io/service-account-token diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml new file mode 100644 index 0000000..51a75db --- /dev/null +++ b/config/rbac/auth_proxy_client_clusterrole.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml new file mode 100644 index 0000000..80e1857 --- /dev/null +++ b/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 0000000..ec7acc0 --- /dev/null +++ b/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/default/metrics_service.yaml b/config/rbac/auth_proxy_service.yaml similarity index 61% rename from config/default/metrics_service.yaml rename to config/rbac/auth_proxy_service.yaml index ebaeff0..71f1797 100644 --- a/config/default/metrics_service.yaml +++ b/config/rbac/auth_proxy_service.yaml @@ -3,15 +3,13 @@ kind: Service metadata: labels: control-plane: controller-manager - app.kubernetes.io/name: notification-service - app.kubernetes.io/managed-by: kustomize name: controller-manager-metrics-service namespace: system spec: ports: - - name: http - port: 8080 + - name: https + port: 8443 protocol: TCP - targetPort: 8080 + targetPort: https selector: control-plane: controller-manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 166fe79..74c9df3 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -7,5 +7,17 @@ resources: - service_account.yaml - role.yaml - role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml + +# auth proxy protects the /metrics endpoint +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml + +# prometheus +- prometheus_service_account.yaml +- prometheus_viewer_role.yaml +- prometheus_role_binding.yaml + +# - leader_election_role.yaml +# - leader_election_role_binding.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml deleted file mode 100644 index 7f8485f..0000000 --- a/config/rbac/leader_election_role.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# permissions to do leader election. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/name: notification-service - app.kubernetes.io/managed-by: kustomize - name: leader-election-role -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml deleted file mode 100644 index e375cd6..0000000 --- a/config/rbac/leader_election_role_binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/name: notification-service - app.kubernetes.io/managed-by: kustomize - name: leader-election-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: leader-election-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/prometheus_role_binding.yaml b/config/rbac/prometheus_role_binding.yaml new file mode 100644 index 0000000..e737bf0 --- /dev/null +++ b/config/rbac/prometheus_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-viewer-role +subjects: + - kind: ServiceAccount + name: metrics-reader + namespace: system diff --git a/config/rbac/prometheus_service_account.yaml b/config/rbac/prometheus_service_account.yaml new file mode 100644 index 0000000..deddda9 --- /dev/null +++ b/config/rbac/prometheus_service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-reader + namespace: system diff --git a/config/rbac/prometheus_viewer_role.yaml b/config/rbac/prometheus_viewer_role.yaml new file mode 100644 index 0000000..0f63614 --- /dev/null +++ b/config/rbac/prometheus_viewer_role.yaml @@ -0,0 +1,9 @@ +# permissions for prometheus to view custom metrics. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-viewer-role +rules: + - nonResourceURLs: + - /metrics + verbs: [get] diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 90fb70d..2070ede 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -1,9 +1,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - labels: - app.kubernetes.io/name: notification-service - app.kubernetes.io/managed-by: kustomize name: manager-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index af8a41a..7cd6025 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -1,8 +1,5 @@ apiVersion: v1 kind: ServiceAccount metadata: - labels: - app.kubernetes.io/name: notification-service - app.kubernetes.io/managed-by: kustomize name: controller-manager namespace: system