From fc80f2712cb12f7e9d570f21e720b3c8f1725691 Mon Sep 17 00:00:00 2001 From: Ljupco Vangelski Date: Tue, 25 Oct 2022 09:03:39 +0200 Subject: [PATCH] Revert " [#3764] Upgrade to Kafka 3.3.1 and SASL connection (#3852)" (#3879) This reverts commit b97b335a3d17ff8b5e6fba1a297791d7d7f9e863. --- .../charts/prerequisites/charts/kafka/BUILD | 2 +- .../kafka/charts/kafka/templates/_helpers.tpl | 35 ++-- .../charts/kafka/templates/configmap.yaml | 14 -- .../kafka/charts/kafka/templates/service.yaml | 34 ---- .../charts/kafka/templates/statefulset.yaml | 59 ++----- .../charts/kafka/charts/kafka/values.yaml | 17 +- .../charts/kafka/charts/zookeeper/Chart.yaml | 5 + .../charts/zookeeper/templates/_helpers.tpl | 48 ++++++ .../zookeeper/templates/headless-service.yaml | 19 +++ .../templates/poddisruptionbudget.yaml | 19 +++ .../charts/zookeeper/templates/service.yaml | 17 ++ .../zookeeper/templates/statefulset.yaml | 158 ++++++++++++++++++ .../charts/kafka/charts/zookeeper/values.yaml | 25 +++ infrastructure/images/kafka/Dockerfile | 4 +- infrastructure/images/kafka/Makefile | 4 +- .../images/kafka/config/server.properties | 4 +- .../images/kafka/config/zookeeper.properties | 9 + .../images/kafka/scripts/configure-kafka.sh | 24 +-- .../kafka/scripts/configure-zookeeper.sh | 21 +++ .../images/kafka/scripts/entrypoint.sh | 3 + 20 files changed, 390 insertions(+), 131 deletions(-) delete mode 100644 infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/configmap.yaml create mode 100644 infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/Chart.yaml create mode 100644 infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/_helpers.tpl create mode 100644 infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/headless-service.yaml create mode 100644 infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml create mode 100644 infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/service.yaml create mode 100644 infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/statefulset.yaml create mode 100644 infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/values.yaml create mode 100644 infrastructure/images/kafka/config/zookeeper.properties create mode 100755 infrastructure/images/kafka/scripts/configure-zookeeper.sh diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/BUILD b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/BUILD index 81948f5a43..374bb095f5 100644 --- a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/BUILD +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/BUILD @@ -1,5 +1,5 @@ load("//tools/build:helm.bzl", "helm_ruleset_version") helm_ruleset_version( - version = "3.3.1", + version = "2.7.0", ) diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/_helpers.tpl b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/_helpers.tpl index afaf6cdab9..fe026c0740 100644 --- a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/_helpers.tpl +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/_helpers.tpl @@ -31,6 +31,30 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Create a default fully qualified zookeeper name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "kafka.zookeeper.fullname" -}} +{{- $name := default "zookeeper" (index .Values "zookeeper" "nameOverride") -}} +{{- printf "%s-headless" $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Form the Zookeeper URL. If zookeeper is installed as part of this chart, use k8s service discovery, +else use user-provided URL +*/}} +{{- define "kafka.zookeeper.service-name" }} +{{- if (index .Values "zookeeper" "enabled") -}} +{{- $clientPort := default 2181 (index .Values "zookeeper" "clientPort") | int -}} +{{- printf "%s:%d" (include "kafka.zookeeper.fullname" .) $clientPort }} +{{- else -}} +{{- $zookeeperConnect := printf "%s" (index .Values "zookeeper" "url") }} +{{- $zookeeperConnectOverride := (index .Values "configurationOverrides" "zookeeper.connect") }} +{{- default $zookeeperConnect $zookeeperConnectOverride }} +{{- end -}} +{{- end -}} + {{/* Form the Advertised Listeners. We will use the value of nodeport.firstListenerPort to create the external advertised listeners if configurationOverrides.advertised.listeners is not set. @@ -67,14 +91,3 @@ Create a variable containing all the datadirs created. {{- $name := "prometheus-prometheus-kafka-exporter" -}} {{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} {{- end -}} - -{{- define "kafka.controller-quorum-voters" -}} - {{- $firstControllerPort := .Values.firstControllerPort -}} - {{- $brokers := .Values.brokers -}} - {{- range $k, $v := until ($brokers|int) -}} - {{- printf "%d@kafka-%d:%d" $k $k (add ($firstControllerPort|int) $k) -}} - {{- if (lt $k (sub $brokers 1 )) -}} - {{- printf "," -}} - {{- end -}} - {{- end -}} -{{- end -}} diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/configmap.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/configmap.yaml deleted file mode 100644 index e0be5ae6a6..0000000000 --- a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/configmap.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if .Values.sasl }} -kind: ConfigMap -apiVersion: v1 -metadata: - name: kafka-jaas -data: - kafka_jaas.conf: | - KafkaServer { - org.apache.kafka.common.security.plain.PlainLoginModule required - username={{- default "admin" .Values.jaasUsername | quote }} - password={{- default (randAlphaNum 32) .Values.jaasPassword | quote }} - user_admin={{- default (randAlphaNum 64) .Values.jaasAdminPassword | quote }}; - }; -{{- end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/service.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/service.yaml index 910e001101..b83bb2b90f 100644 --- a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/service.yaml +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/service.yaml @@ -15,40 +15,6 @@ spec: app: {{ template "kafka.name" . }} release: {{ .Release.Name }} --- -{{- $brokers := .Values.brokers | int }} -{{- $firstControllerPort := .Values.firstControllerPort }} -{{- range $k, $v := until ($brokers|int) }} -apiVersion: v1 -kind: Service -metadata: - name: kafka-{{ $k }} - labels: - app: kafka-controller -spec: - type: NodePort - ports: - - port: {{ add ($firstControllerPort|int) $k }} - protocol: TCP - selector: - statefulset.kubernetes.io/pod-name: kafka-{{ $k }} ---- -{{- end }} -{{- if .Values.sasl }} -apiVersion: v1 -kind: Service -metadata: - name: {{ template "kafka.fullname" . }}-external - labels: - app: {{ template "kafka.name" . }} -spec: - type: LoadBalancer - ports: - - port: {{ .Values.saslPort }} - name: broker - selector: - app: {{ template "kafka.name" . }} ---- -{{- end }} {{- if .Values.prometheus.enabled }} apiVersion: v1 kind: Service diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/statefulset.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/statefulset.yaml index 9d9b103116..1521c11565 100644 --- a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/statefulset.yaml +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/statefulset.yaml @@ -3,7 +3,6 @@ apiVersion: apps/v1 {{- else }} apiVersion: apps/v1beta1 {{- end }} -{{- $brokers := .Values.brokers | int }} kind: StatefulSet metadata: name: {{ template "kafka.fullname" . }} @@ -21,7 +20,7 @@ spec: {{- end }} serviceName: {{ template "kafka.fullname" . }}-headless podManagementPolicy: {{ .Values.podManagementPolicy }} - replicas: {{ $brokers }} + replicas: {{ default 3 .Values.brokers }} updateStrategy: type: {{ .Values.updateStrategy }} template: @@ -54,21 +53,13 @@ spec: - containerPort: {{ .Values.port }} name: kafka {{- if .Values.nodeport.enabled }} + {{- $brokers := .Values.brokers | int }} {{- $root := . }} {{- range $i, $e := until $brokers }} - - containerPort: {{ add $root.Values.firstListenerPort $i }} + - containerPort: {{ add $root.Values.nodeport.firstListenerPort $i }} name: nodeport-{{ $i }} {{- end }} {{- end }} - {{- $root := . }} - {{- range $i, $e := until $brokers }} - - containerPort: {{ add $root.Values.firstControllerPort $i }} - name: controller-{{ $i }} - {{- end }} - {{- if .Values.sasl }} - - containerPort: {{ .Values.saslPort }} - name: kafka-sasl - {{- end }} resources: {{ toYaml .Values.resources | indent 10 }} env: @@ -89,34 +80,19 @@ spec: fieldRef: fieldPath: metadata.namespace - name: AIRY_LISTENERS - value: PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:AIRY_BROKER_PORT,CONTROLLER://0.0.0.0:AIRY_CONTROLLER_PORT{{- if .Values.sasl }},SASL_PLAINTEXT://0.0.0.0:{{ .Values.saslPort }}{{- end }} + value: PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:AIRY_BROKER_PORT - name: AIRY_ADVERTISED_LISTENERS - value: PLAINTEXT://AIRY_POD_NAME.{{ template "kafka.fullname" . }}-headless.AIRY_NAMESPACE:9092,EXTERNAL://AIRY_HOST_IP:AIRY_BROKER_PORT{{- if .Values.sasl }},SASL_PLAINTEXT://AIRY_POD_NAME.kafka-headless.AIRY_NAMESPACE:{{ .Values.saslPort }}{{- end }} - - name: AIRY_CONTROLLER_QUORUM_VOTERS - value: {{ template "kafka.controller-quorum-voters" . }} + value: PLAINTEXT://AIRY_POD_NAME.{{ template "kafka.fullname" . }}-headless.AIRY_NAMESPACE:9092,EXTERNAL://AIRY_HOST_IP:AIRY_BROKER_PORT - name: AIRY_FIRST_LISTENER_PORT value: "{{ .Values.firstListenerPort }}" - - name: AIRY_FIRST_CONTROLLER_PORT - value: "{{ .Values.firstControllerPort }}" - name: KAFKA_HEAP_OPTS value: {{ .Values.heapOptions }} + - name: KAFKA_ZOOKEEPER_CONNECT + value: {{ include "kafka.zookeeper.service-name" . | quote }} {{- range $key, $value := .Values.configurationOverrides }} - name: {{ printf "KAFKA_%s" $key | replace "." "_" | upper | quote }} value: {{ $value | quote }} {{- end }} - {{- if .Values.sasl }} - - name: KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND - value: "true" - - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP - value: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT - - name: KAFKA_SASL_ENABLED_MECHANISMS - value: PLAIN - - name: KAFKA_OPTS - value: -Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf - {{- else }} - - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP - value: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - {{- end }} args: ["kafka-server-start.sh", "/etc/kafka/server.properties"] volumeMounts: {{- if .Values.persistence.enabled }} @@ -126,16 +102,21 @@ spec: mountPath: /opt/kafka/data-{{$k}} {{- end }} {{- end }} - {{- if .Values.sasl }} - - name: kafka-jaas - mountPath: /etc/kafka/kafka_server_jaas.conf - subPath: kafka_jaas.conf - {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.imagePullSecrets | indent 8 }} {{- end }} initContainers: + - name: wait + image: "{{ .Values.global.busyboxImage }}" + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "/opt/provisioning/wait-for-service-url.sh"] + env: + - name: SERVICE_URL + value: {{ include "kafka.zookeeper.service-name" . | quote }} + volumeMounts: + - name: kafka-helper-scripts + mountPath: /opt/provisioning - name: fix-permissions image: "{{ .Values.global.busyboxImage }}" imagePullPolicy: IfNotPresent @@ -156,12 +137,6 @@ spec: - name: datadir-0 emptyDir: {} {{- end }} - {{- if .Values.sasl }} - - name: kafka-jaas - configMap: - name: kafka-jaas - name: kafka-jaas - {{- end }} - name: kafka-helper-scripts configMap: name: kafka-helper-scripts diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/values.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/values.yaml index 732f966572..24f1408db7 100644 --- a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/values.yaml +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/values.yaml @@ -1,18 +1,17 @@ brokers: 1 image: ghcr.io/airyhq/infrastructure/kafka -imageTag: 3.3.1 +imageTag: 2.7.0 imagePullPolicy: IfNotPresent imagePullSecrets: podManagementPolicy: OrderedReady updateStrategy: RollingUpdate -sasl: false configurationOverrides: "offsets.topic.replication.factor": "1" "log.retention.hours": "-1" + "listener.security.protocol.map": |- + PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT port: 9092 -saslPort: 9093 firstListenerPort: 31090 -firstControllerPort: 19092 persistence: enabled: true size: 10Gi @@ -24,6 +23,16 @@ nodeSelector: {} tolerations: [] nodeport: enabled: true + servicePort: 19092 + firstListenerPort: 31090 +zookeeper: + enabled: true + servers: 1 + persistence: + enabled: true + dataDirSize: 5Gi + dataLogDirSize: 5Gi + url: "" prometheus: enabled: false exporterImage: danielqsj/kafka-exporter diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/Chart.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/Chart.yaml new file mode 100644 index 0000000000..9dcc049857 --- /dev/null +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: "1.0" +description: A Helm chart for Confluent Zookeeper on Kubernetes +name: zookeeper +version: 1.0 \ No newline at end of file diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/_helpers.tpl b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/_helpers.tpl new file mode 100644 index 0000000000..dff32ff458 --- /dev/null +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/_helpers.tpl @@ -0,0 +1,48 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "cp-zookeeper.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cp-zookeeper.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cp-zookeeper.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a server list string based on fullname, namespace, # of servers +in a format like "zkhost1:port:port;zkhost2:port:port" +*/}} +{{- define "cp-zookeeper.serverlist" -}} +{{- $namespace := .Release.Namespace }} +{{- $name := include "cp-zookeeper.fullname" . -}} +{{- $serverPort := .Values.serverPort -}} +{{- $leaderElectionPort := .Values.leaderElectionPort -}} +{{- $zk := dict "servers" (list) -}} +{{- range $idx, $v := until (int .Values.servers) }} +{{- $noop := printf "%s-%d.%s-headless.%s:%d:%d" $name $idx $name $namespace (int $serverPort) (int $leaderElectionPort) | append $zk.servers | set $zk "servers" -}} +{{- end }} +{{- printf "%s" (join ";" $zk.servers) | quote -}} +{{- end -}} diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/headless-service.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/headless-service.yaml new file mode 100644 index 0000000000..6e354b9020 --- /dev/null +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/headless-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "cp-zookeeper.fullname" . }}-headless + labels: + app: {{ template "cp-zookeeper.name" . }} + chart: {{ template "cp-zookeeper.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + ports: + - port: {{ .Values.serverPort }} + name: server + - port: {{ .Values.leaderElectionPort }} + name: leader-election + clusterIP: None + selector: + app: {{ template "cp-zookeeper.name" . }} + release: {{ .Release.Name }} diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..855a11b967 --- /dev/null +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml @@ -0,0 +1,19 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ template "cp-zookeeper.fullname" . }}-pdb + labels: + app: {{ template "cp-zookeeper.name" . }} + chart: {{ template "cp-zookeeper.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + matchLabels: + app: {{ template "cp-zookeeper.name" . }} + release: {{ .Release.Name }} + {{- if .Values.minAvailable }} + minAvailable: {{ .Values.minAvailable }} + {{- else }} + maxUnavailable: 1 + {{- end }} diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/service.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/service.yaml new file mode 100644 index 0000000000..f19bef6314 --- /dev/null +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "cp-zookeeper.fullname" . }} + labels: + app: {{ template "cp-zookeeper.name" . }} + chart: {{ template "cp-zookeeper.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + type: {{ .Values.serviceType }} + ports: + - port: {{ .Values.clientPort }} + name: client + selector: + app: {{ template "cp-zookeeper.name" . }} + release: {{ .Release.Name }} diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/statefulset.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/statefulset.yaml new file mode 100644 index 0000000000..2356f50d97 --- /dev/null +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/statefulset.yaml @@ -0,0 +1,158 @@ +{{- if .Capabilities.APIVersions.Has "apps/v1" }} +apiVersion: apps/v1 +{{- else }} +apiVersion: apps/v1beta1 +{{- end }} +kind: StatefulSet +metadata: + name: {{ template "cp-zookeeper.fullname" . }} + labels: + app: {{ template "cp-zookeeper.name" . }} + chart: {{ template "cp-zookeeper.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + {{- if .Capabilities.APIVersions.Has "apps/v1" }} + selector: + matchLabels: + app: {{ template "cp-zookeeper.name" . }} + release: {{ .Release.Name }} + {{- end }} + serviceName: {{ template "cp-zookeeper.fullname" . }}-headless + podManagementPolicy: {{ .Values.podManagementPolicy }} + replicas: {{ default 3 .Values.servers }} + updateStrategy: + type: {{ .Values.updateStrategy }} + template: + metadata: + labels: + app: {{ template "cp-zookeeper.name" . }} + release: {{ .Release.Name }} + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - {{ template "cp-zookeeper.name" . }} + - key: "release" + operator: In + values: + - {{ .Release.Name }} + topologyKey: "kubernetes.io/hostname" + initContainers: + - name: fix-permissions + image: "{{ .Values.global.busyboxImage }}" + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - "chown -R 1000 /var/lib/zookeeper/" + volumeMounts: + - name: datadir + mountPath: /var/lib/zookeeper/data + - name: datalogdir + mountPath: /var/lib/zookeeper/log + containers: + - name: {{ template "cp-zookeeper.name" . }}-server + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: "{{ .Values.imagePullPolicy }}" + ports: + - containerPort: {{ .Values.clientPort }} + name: client + - containerPort: {{ .Values.serverPort }} + name: server + - containerPort: {{ .Values.leaderElectionPort }} + name: leader-election + resources: +{{ toYaml .Values.resources | indent 10 }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name : KAFKA_HEAP_OPTS + value: "{{ .Values.heapOptions }}" + - name : ZOOKEEPER_TICK_TIME + value: "{{ .Values.tickTime }}" + - name : ZOOKEEPER_SYNC_LIMIT + value: "{{ .Values.syncLimit }}" + - name : ZOOKEEPER_INIT_LIMIT + value: "{{ .Values.initLimit }}" + - name : ZOOKEEPER_MAX_CLIENT_CNXNS + value: "{{ .Values.maxClientCnxns }}" + - name : ZOOKEEPER_AUTOPURGE_SNAP_RETAIN_COUNT + value: "{{ .Values.autoPurgeSnapRetainCount}}" + - name : ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL + value: "{{ .Values.autoPurgePurgeInterval}}" + - name: ZOOKEEPER_CLIENT_PORT + value: "{{ .Values.clientPort }}" + - name : ZOOKEEPER_SERVERS + value: {{ template "cp-zookeeper.serverlist" . }} + # ZOOKEEPER_SERVER_ID is required just to pass cp-zookeeper ensure script for env check, + # the value(metadata.mame) is not used and will be overwritten in command part + - name: ZOOKEEPER_SERVER_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + args: ["zookeeper-server-start.sh", "/etc/kafka/zookeeper.properties"] + volumeMounts: + - name: datadir + mountPath: /var/lib/zookeeper/data + - name: datalogdir + mountPath: /var/lib/zookeeper/log + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + volumes: + {{ if not .Values.persistence.enabled }} + - name: datadir + emptyDir: {} + - name: datalogdir + emptyDir: {} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: datadir + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: "{{ .Values.persistence.dataDirSize }}" + {{- if .Values.persistence.dataDirStorageClass }} + {{- if (eq "-" .Values.persistence.dataDirStorageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistence.dataDirStorageClass }}" + {{- end }} + {{- end }} + - metadata: + name: datalogdir + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: "{{ .Values.persistence.dataLogDirSize }}" + {{- if .Values.persistence.dataLogDirStorageClass }} + {{- if (eq "-" .Values.persistence.dataLogDirStorageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistence.dataLogDirStorageClass }}" + {{- end }} + {{- end }} + {{- end }} diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/values.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/values.yaml new file mode 100644 index 0000000000..ad682bb02b --- /dev/null +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/values.yaml @@ -0,0 +1,25 @@ +servers: 1 +image: ghcr.io/airyhq/infrastructure/kafka +imageTag: 2.7.0 +imagePullPolicy: IfNotPresent +imagePullSecrets: +podManagementPolicy: OrderedReady +updateStrategy: RollingUpdate +tickTime: 2000 +syncLimit: 5 +initLimit: 10 +maxClientCnxns: 60 +autoPurgeSnapRetainCount: 3 +autoPurgePurgeInterval: 24 +heapOptions: "-Xms512M -Xmx512M" +serverPort: 2888 +leaderElectionPort: 3888 +clientPort: 2181 +persistence: + enabled: true + dataDirSize: 5Gi + dataLogDirSize: 5Gi +resources: {} +podAnnotations: {} +nodeSelector: {} +tolerations: [] diff --git a/infrastructure/images/kafka/Dockerfile b/infrastructure/images/kafka/Dockerfile index bd3c572941..64416471f5 100644 --- a/infrastructure/images/kafka/Dockerfile +++ b/infrastructure/images/kafka/Dockerfile @@ -3,7 +3,7 @@ FROM openjdk:8-jre-alpine LABEL maintainer "https://github.com/airyhq" LABEL name="kafka" version=${INSTALL_KAFKA_VERSION} -ENV INSTALL_KAFKA_VERSION 3.3.1 +ENV INSTALL_KAFKA_VERSION 2.7.0 ENV INSTALL_SCALA_VERSION 2.13 ENV INSTALL_MIRROR="https://downloads.apache.org/" @@ -23,7 +23,9 @@ ENV PATH /sbin:/opt/kafka/bin/:$PATH WORKDIR /opt/kafka COPY config/server.properties /etc/kafka/server.properties +COPY config/zookeeper.properties /etc/kafka/zookeeper.properties COPY /scripts/configure-kafka.sh /root/configure-kafka.sh +COPY /scripts/configure-zookeeper.sh /root/configure-zookeeper.sh COPY /scripts/entrypoint.sh /root/entrypoint.sh ENTRYPOINT ["/root/entrypoint.sh"] diff --git a/infrastructure/images/kafka/Makefile b/infrastructure/images/kafka/Makefile index b370df017e..6da969fb57 100644 --- a/infrastructure/images/kafka/Makefile +++ b/infrastructure/images/kafka/Makefile @@ -2,5 +2,5 @@ build: docker build -t airy-kafka . release: build - docker tag airy-kafka ghcr.io/airyhq/infrastructure/kafka:3.3.1 - docker push ghcr.io/airyhq/infrastructure/kafka:3.3.1 + docker tag airy-kafka ghcr.io/airyhq/infrastructure/kafka:2.7.0 + docker push ghcr.io/airyhq/infrastructure/kafka:2.7.0 diff --git a/infrastructure/images/kafka/config/server.properties b/infrastructure/images/kafka/config/server.properties index f864f9f5dd..b358841589 100644 --- a/infrastructure/images/kafka/config/server.properties +++ b/infrastructure/images/kafka/config/server.properties @@ -1,6 +1,4 @@ # Default settings -process.roles=broker,controller -inter.broker.listener.name=PLAINTEXT -controller.listener.names=CONTROLLER +listener.security.protocol.map=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT log.dirs=/opt/kafka/data-0/logs # Injected settings diff --git a/infrastructure/images/kafka/config/zookeeper.properties b/infrastructure/images/kafka/config/zookeeper.properties new file mode 100644 index 0000000000..1f07480930 --- /dev/null +++ b/infrastructure/images/kafka/config/zookeeper.properties @@ -0,0 +1,9 @@ +dataDir=/var/lib/zookeeper/data +dataLogDir=/var/lib/zookeeper/log +clientPort=2181 +maxClientCnxns=0 +initLimit=10 +autopurge.purgeInterval=24 +syncLimit=5 +autopurge.snapRetainCount=3 +tickTime=2000 diff --git a/infrastructure/images/kafka/scripts/configure-kafka.sh b/infrastructure/images/kafka/scripts/configure-kafka.sh index fe79139266..431c204c63 100755 --- a/infrastructure/images/kafka/scripts/configure-kafka.sh +++ b/infrastructure/images/kafka/scripts/configure-kafka.sh @@ -3,29 +3,24 @@ set -eo pipefail BROKER_ID=$(echo "${POD_NAME}" | awk -F "-" '{ print $NF; }') BROKER_PORT=$((AIRY_FIRST_LISTENER_PORT + BROKER_ID)) -CONTROLLER_PORT=$((AIRY_FIRST_CONTROLLER_PORT + BROKER_ID)) -LISTENERS=$(echo "${AIRY_LISTENERS}" | \ - sed "s/AIRY_BROKER_PORT/${BROKER_PORT}/g" | \ - sed "s/AIRY_CONTROLLER_PORT/${CONTROLLER_PORT}/g") +LISTENERS="${AIRY_LISTENERS//AIRY_BROKER_PORT/${BROKER_PORT}}" ADVERTISED_LISTENERS=$(echo "${AIRY_ADVERTISED_LISTENERS}" | \ sed "s/AIRY_POD_NAME/${POD_NAME}/g" | \ sed "s/AIRY_NAMESPACE/${POD_NAMESPACE}/g" | \ sed "s/AIRY_HOST_IP/${HOST_IP}/g" | \ sed "s/AIRY_BROKER_PORT/${BROKER_PORT}/g") -CONTROLLER_QUORUM_VOTERS=${AIRY_CONTROLLER_QUORUM_VOTERS} # Insert runtime configuration -{ - echo "node.id=${BROKER_ID}" +{ + echo "broker.id=${BROKER_ID}" echo "listeners=${LISTENERS}" echo "advertised.listeners=${ADVERTISED_LISTENERS}" - echo "controller.quorum.voters=${CONTROLLER_QUORUM_VOTERS}" } >> /etc/kafka/server.properties - + # Insert all other KAFKA_* env variables as settings for VAR in $(env) do - if [[ $VAR =~ ^KAFKA_ && ! $VAR =~ ^KAFKA_VERSION && ! $VAR =~ ^KAFKA_[0-9] && ! $VAR =~ ^KAFKA_PORT && ! $VAR =~ ^KAFKA_OPTS && ! $VAR =~ ^KAFKA_SERVICE_ ]]; then + if [[ $VAR =~ ^KAFKA_ && ! $VAR =~ ^KAFKA_VERSION && ! $VAR =~ ^KAFKA_[0-9] && ! $VAR =~ ^KAFKA_PORT && ! $VAR =~ ^KAFKA_SERVICE_ ]]; then kafka_name=$(echo "$VAR" | sed -r "s/KAFKA_(.*)=.*/\1/g" | tr '[:upper:]' '[:lower:]' | tr _ .) env_var=$(echo "$VAR" | sed -r "s/(.*)=.*/\1/g") echo "$kafka_name=${!env_var}" >> /etc/kafka/server.properties @@ -34,12 +29,3 @@ done echo "Printing out the broker configuration" cat /etc/kafka/server.properties - -if ! kafka-storage.sh info -c /etc/kafka/server.properties > /dev/null; then - echo "Cleaning up Kafka..." - rm -rf /opt/kafka/data-0/logs/__cluster_metadata-0/ - CLUSTER_ID=$(/opt/kafka/bin/kafka-storage.sh random-uuid) - /opt/kafka/bin/kafka-storage.sh format -t "${CLUSTER_ID}" -c /etc/kafka/server.properties --ignore-formatted - chown -R 1000 /opt/kafka/ -echo "Configuration done, starting Kafka..." -fi diff --git a/infrastructure/images/kafka/scripts/configure-zookeeper.sh b/infrastructure/images/kafka/scripts/configure-zookeeper.sh new file mode 100755 index 0000000000..6e3ad8e5bb --- /dev/null +++ b/infrastructure/images/kafka/scripts/configure-zookeeper.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -eo pipefail + +ZOOKEEPER_SERVERS=$(echo "${ZOOKEEPER_SERVERS}" | tr ";" " ") +ZOOKEEPER_ID=$(echo "${POD_NAME}" | awk -F "-" '{ print $NF; }') +ZOOKEEPER_SERVER_ID=$((ZOOKEEPER_ID+1)) + +echo "${ZOOKEEPER_SERVER_ID}" > /var/lib/zookeeper/data/myid + +# Insert runtime configuration +COUNT=1 +for SERVER in ${ZOOKEEPER_SERVERS}; do + if [ ${COUNT} -eq ${ZOOKEEPER_SERVER_ID} ]; then + echo "server.${COUNT}=0.0.0.0:2888:3888" >> /etc/kafka/zookeeper.properties + else + echo "server.${COUNT}=${SERVER}" >> /etc/kafka/zookeeper.properties + fi + COUNT=$((COUNT+1)) +done + +cat /etc/kafka/zookeeper.properties diff --git a/infrastructure/images/kafka/scripts/entrypoint.sh b/infrastructure/images/kafka/scripts/entrypoint.sh index 5355421103..0b4974582c 100755 --- a/infrastructure/images/kafka/scripts/entrypoint.sh +++ b/infrastructure/images/kafka/scripts/entrypoint.sh @@ -3,6 +3,9 @@ set -e if [[ "$1" == kafka-server-start.sh && "$2" == /etc/kafka/server.properties ]];then /root/configure-kafka.sh +elif + [[ "$1" == zookeeper-server-start.sh && "$2" == /etc/kafka/zookeeper.properties ]];then + /root/configure-zookeeper.sh else echo "Unsupported arguments to docker entrypoint" exit 1