Revert " [#3764] Upgrade to Kafka 3.3.1 and SASL connection (#3852)"

This reverts commit b97b335.

infrastructure/helm-chart/charts/prerequisites/charts/kafka/BUILD

@@ -1,5 +1,5 @@
 load("//tools/build:helm.bzl", "helm_ruleset_version")
 
 helm_ruleset_version(
-    version = "3.3.1",
+    version = "2.7.0",
 )

infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/_helpers.tpl

@@ -31,6 +31,30 @@ Create chart name and version as used by the chart label.
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+Create a default fully qualified zookeeper name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "kafka.zookeeper.fullname" -}}
+{{- $name := default "zookeeper" (index .Values "zookeeper" "nameOverride") -}}
+{{- printf "%s-headless" $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Form the Zookeeper URL. If zookeeper is installed as part of this chart, use k8s service discovery,
+else use user-provided URL
+*/}}
+{{- define "kafka.zookeeper.service-name" }}
+{{- if (index .Values "zookeeper" "enabled") -}}
+{{- $clientPort := default 2181 (index .Values "zookeeper" "clientPort") | int -}}
+{{- printf "%s:%d" (include "kafka.zookeeper.fullname" .) $clientPort }}
+{{- else -}}
+{{- $zookeeperConnect := printf "%s" (index .Values "zookeeper" "url") }}
+{{- $zookeeperConnectOverride := (index .Values "configurationOverrides" "zookeeper.connect") }}
+{{- default $zookeeperConnect $zookeeperConnectOverride }}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Form the Advertised Listeners. We will use the value of nodeport.firstListenerPort to create the
 external advertised listeners if configurationOverrides.advertised.listeners is not set.
@@ -67,14 +91,3 @@ Create a variable containing all the datadirs created.
 {{- $name := "prometheus-prometheus-kafka-exporter" -}}
 {{- printf "%s" $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
-
-{{- define "kafka.controller-quorum-voters" -}}
-  {{- $firstControllerPort := .Values.firstControllerPort -}}
-  {{- $brokers := .Values.brokers -}}
-  {{- range $k, $v := until ($brokers|int) -}}
-    {{- printf "%d@kafka-%d:%d" $k $k (add ($firstControllerPort|int) $k) -}}
-    {{- if (lt $k (sub $brokers  1 )) -}}
-      {{- printf "," -}}
-    {{- end -}}
-  {{- end -}}
-{{- end -}}

infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/service.yaml

@@ -15,40 +15,6 @@ spec:
     app: {{ template "kafka.name" . }}
     release: {{ .Release.Name }}
 ---
-{{- $brokers := .Values.brokers | int }}
-{{- $firstControllerPort := .Values.firstControllerPort }}
-{{- range $k, $v := until ($brokers|int) }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: kafka-{{ $k }}
-  labels:
-    app: kafka-controller
-spec:
-  type: NodePort
-  ports:
-  - port: {{ add ($firstControllerPort|int) $k }}
-    protocol: TCP
-  selector:
-    statefulset.kubernetes.io/pod-name: kafka-{{ $k }}
----
-{{- end }}
-{{- if .Values.sasl }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ template "kafka.fullname" . }}-external
-  labels:
-    app: {{ template "kafka.name" . }}
-spec:
-  type: LoadBalancer
-  ports:
-    - port: {{ .Values.saslPort }}
-      name: broker
-  selector:
-    app: {{ template "kafka.name" . }}
----
-{{- end }}
 {{- if .Values.prometheus.enabled }}
 apiVersion: v1
 kind: Service

infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/templates/statefulset.yaml

@@ -3,7 +3,6 @@ apiVersion: apps/v1
 {{- else }}
 apiVersion: apps/v1beta1
 {{- end }}
-{{- $brokers := .Values.brokers | int }}
 kind: StatefulSet
 metadata:
   name: {{ template "kafka.fullname" . }}
@@ -21,7 +20,7 @@ spec:
   {{- end }}
   serviceName: {{ template "kafka.fullname" . }}-headless
   podManagementPolicy: {{ .Values.podManagementPolicy }}
-  replicas: {{ $brokers }}
+  replicas: {{ default 3 .Values.brokers }}
   updateStrategy:
     type: {{ .Values.updateStrategy }}
   template:
@@ -54,21 +53,13 @@ spec:
         - containerPort: {{ .Values.port }}
           name: kafka
         {{- if .Values.nodeport.enabled }}
+          {{- $brokers := .Values.brokers | int }}
           {{- $root := . }}
           {{- range $i, $e := until $brokers }}
-        - containerPort: {{ add $root.Values.firstListenerPort $i }}
+        - containerPort: {{ add $root.Values.nodeport.firstListenerPort $i }}
           name: nodeport-{{ $i }}
           {{- end }}
         {{- end }}
-        {{- $root := . }}
-        {{- range $i, $e := until $brokers }}
-        - containerPort: {{ add $root.Values.firstControllerPort $i }}
-          name: controller-{{ $i }}
-        {{- end }}
-        {{- if .Values.sasl }}
-        - containerPort: {{ .Values.saslPort }}
-          name: kafka-sasl
-        {{- end }}
         resources:
 {{ toYaml .Values.resources | indent 10 }}
         env:
@@ -89,34 +80,19 @@ spec:
             fieldRef:
               fieldPath: metadata.namespace
         - name: AIRY_LISTENERS
-          value: PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:AIRY_BROKER_PORT,CONTROLLER://0.0.0.0:AIRY_CONTROLLER_PORT{{- if .Values.sasl }},SASL_PLAINTEXT://0.0.0.0:{{ .Values.saslPort }}{{- end }}
+          value: PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:AIRY_BROKER_PORT
         - name: AIRY_ADVERTISED_LISTENERS
-          value: PLAINTEXT://AIRY_POD_NAME.{{ template "kafka.fullname" . }}-headless.AIRY_NAMESPACE:9092,EXTERNAL://AIRY_HOST_IP:AIRY_BROKER_PORT{{- if .Values.sasl }},SASL_PLAINTEXT://AIRY_POD_NAME.kafka-headless.AIRY_NAMESPACE:{{ .Values.saslPort }}{{- end }}
-        - name: AIRY_CONTROLLER_QUORUM_VOTERS
-          value: {{ template "kafka.controller-quorum-voters" . }}
+          value: PLAINTEXT://AIRY_POD_NAME.{{ template "kafka.fullname" . }}-headless.AIRY_NAMESPACE:9092,EXTERNAL://AIRY_HOST_IP:AIRY_BROKER_PORT
         - name: AIRY_FIRST_LISTENER_PORT
           value: "{{ .Values.firstListenerPort }}"
-        - name: AIRY_FIRST_CONTROLLER_PORT
-          value: "{{ .Values.firstControllerPort }}"
         - name: KAFKA_HEAP_OPTS
           value: {{ .Values.heapOptions }}
+        - name: KAFKA_ZOOKEEPER_CONNECT
+          value: {{ include "kafka.zookeeper.service-name" . | quote }}
         {{- range $key, $value := .Values.configurationOverrides }}
         - name: {{ printf "KAFKA_%s" $key | replace "." "_" | upper | quote }}
           value: {{ $value | quote }}
         {{- end }}
-        {{- if .Values.sasl }}
-        - name: KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND
-          value: "true"
-        - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
-          value: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT
-        - name: KAFKA_SASL_ENABLED_MECHANISMS
-          value: PLAIN
-        - name: KAFKA_OPTS
-          value: -Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf
-        {{- else }}
-        - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
-          value: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
-        {{- end }}
         args: ["kafka-server-start.sh", "/etc/kafka/server.properties"]
         volumeMounts:
         {{- if .Values.persistence.enabled }}
@@ -126,16 +102,21 @@ spec:
             mountPath: /opt/kafka/data-{{$k}}
           {{- end }}
         {{- end }}
-        {{- if .Values.sasl }}
-          - name: kafka-jaas
-            mountPath: /etc/kafka/kafka_server_jaas.conf
-            subPath: kafka_jaas.conf
-        {{- end }}
       {{- if .Values.imagePullSecrets }}
       imagePullSecrets:
 {{ toYaml .Values.imagePullSecrets | indent 8 }}
       {{- end }}
       initContainers:
+      - name: wait
+        image: "{{ .Values.global.busyboxImage }}"
+        imagePullPolicy: IfNotPresent
+        command: ["/bin/sh", "/opt/provisioning/wait-for-service-url.sh"]
+        env:
+        - name: SERVICE_URL
+          value: {{ include "kafka.zookeeper.service-name" . | quote }}
+        volumeMounts:
+        - name: kafka-helper-scripts
+          mountPath: /opt/provisioning
       - name: fix-permissions
         image: "{{ .Values.global.busyboxImage }}"
         imagePullPolicy: IfNotPresent
@@ -156,12 +137,6 @@ spec:
       - name: datadir-0
         emptyDir: {}
       {{- end }}
-      {{- if .Values.sasl }}
-      - name: kafka-jaas
-        configMap:
-          name: kafka-jaas
-        name: kafka-jaas
-      {{- end }}
       - name: kafka-helper-scripts
         configMap:
           name: kafka-helper-scripts

infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/kafka/values.yaml

@@ -1,18 +1,17 @@
 brokers: 1
 image: ghcr.io/airyhq/infrastructure/kafka
-imageTag: 3.3.1
+imageTag: 2.7.0
 imagePullPolicy: IfNotPresent
 imagePullSecrets:
 podManagementPolicy: OrderedReady
 updateStrategy: RollingUpdate
-sasl: false
 configurationOverrides:
   "offsets.topic.replication.factor": "1"
   "log.retention.hours": "-1"
+  "listener.security.protocol.map": |-
+    PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
 port: 9092
-saslPort: 9093
 firstListenerPort: 31090
-firstControllerPort: 19092
 persistence:
   enabled: true
   size: 10Gi
@@ -24,6 +23,16 @@ nodeSelector: {}
 tolerations: []
 nodeport:
   enabled: true
+  servicePort: 19092
+  firstListenerPort: 31090
+zookeeper:
+  enabled: true
+  servers: 1
+  persistence:
+    enabled: true
+    dataDirSize: 5Gi
+    dataLogDirSize: 5Gi
+  url: ""
 prometheus:
   enabled: false
   exporterImage: danielqsj/kafka-exporter

infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+appVersion: "1.0"
+description: A Helm chart for Confluent Zookeeper on Kubernetes
+name: zookeeper
+version: 1.0

infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/_helpers.tpl

@@ -0,0 +1,48 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "cp-zookeeper.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "cp-zookeeper.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "cp-zookeeper.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a server list string based on fullname, namespace, # of servers
+in a format like "zkhost1:port:port;zkhost2:port:port"
+*/}}
+{{- define "cp-zookeeper.serverlist" -}}
+{{- $namespace := .Release.Namespace }}
+{{- $name := include "cp-zookeeper.fullname" . -}}
+{{- $serverPort := .Values.serverPort -}}
+{{- $leaderElectionPort := .Values.leaderElectionPort -}}
+{{- $zk := dict "servers" (list) -}}
+{{- range $idx, $v := until (int .Values.servers) }}
+{{- $noop := printf "%s-%d.%s-headless.%s:%d:%d" $name $idx $name $namespace (int $serverPort) (int $leaderElectionPort) | append $zk.servers | set $zk "servers" -}}
+{{- end }}
+{{- printf "%s" (join ";" $zk.servers) | quote -}}
+{{- end -}}

infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/headless-service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "cp-zookeeper.fullname" . }}-headless
+  labels:
+    app: {{ template "cp-zookeeper.name" . }}
+    chart: {{ template "cp-zookeeper.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  ports:
+    - port: {{ .Values.serverPort }}
+      name: server
+    - port: {{ .Values.leaderElectionPort }}
+      name: leader-election
+  clusterIP: None
+  selector:
+    app: {{ template "cp-zookeeper.name" . }}
+    release: {{ .Release.Name }}

infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml

@@ -0,0 +1,19 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ template "cp-zookeeper.fullname" . }}-pdb
+  labels:
+    app: {{ template "cp-zookeeper.name" . }}
+    chart: {{ template "cp-zookeeper.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ template "cp-zookeeper.name" . }}
+      release: {{ .Release.Name }}
+  {{- if .Values.minAvailable }}
+  minAvailable: {{ .Values.minAvailable }}
+  {{- else }}
+  maxUnavailable: 1
+  {{- end }}

infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/zookeeper/templates/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "cp-zookeeper.fullname" . }}
+  labels:
+    app: {{ template "cp-zookeeper.name" . }}
+    chart: {{ template "cp-zookeeper.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  type: {{ .Values.serviceType }}
+  ports:
+    - port: {{ .Values.clientPort }}
+      name: client
+  selector:
+    app: {{ template "cp-zookeeper.name" . }}
+    release: {{ .Release.Name }}