diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02c8436e4a..699da3a78a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,6 +81,7 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} GITHUB_BRANCH: ${{ github.ref }} - name: Publish helm charts diff --git a/VERSION b/VERSION index b930054648..316ba4bd9e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.55.0-alpha +0.55.0 diff --git a/backend/components/chat-plugin/helm/templates/backend/deployment.yaml b/backend/components/chat-plugin/helm/templates/backend/deployment.yaml index 1f8995b817..a82f25fd2f 100644 --- a/backend/components/chat-plugin/helm/templates/backend/deployment.yaml +++ b/backend/components/chat-plugin/helm/templates/backend/deployment.yaml @@ -51,6 +51,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.backend.resources | indent 10 }} initContainers: @@ -68,3 +73,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/contacts/helm/templates/deployment.yaml b/backend/components/contacts/helm/templates/deployment.yaml index 5e4c6846a8..b0d2e68d9b 100644 --- a/backend/components/contacts/helm/templates/deployment.yaml +++ b/backend/components/contacts/helm/templates/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/facebook/helm/templates/deployments.yaml b/backend/components/facebook/helm/templates/deployments.yaml index 4d72ef5174..a530a8122f 100644 --- a/backend/components/facebook/helm/templates/deployments.yaml +++ b/backend/components/facebook/helm/templates/deployments.yaml @@ -59,6 +59,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -76,6 +81,11 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} --- apiVersion: apps/v1 kind: Deployment @@ -124,6 +134,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.eventsRouter.resources | indent 10 }} initContainers: @@ -141,3 +156,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/google/helm/templates/deployments.yaml b/backend/components/google/helm/templates/deployments.yaml index 7af611787e..80c2af2376 100644 --- a/backend/components/google/helm/templates/deployments.yaml +++ b/backend/components/google/helm/templates/deployments.yaml @@ -54,6 +54,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -71,6 +76,11 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} --- apiVersion: apps/v1 kind: Deployment @@ -122,6 +132,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.eventsRouter.resources | indent 10 }} initContainers: @@ -139,3 +154,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/media-resolver/helm/templates/deployment.yaml b/backend/components/media-resolver/helm/templates/deployment.yaml index b31af97ffd..3484e0b71f 100644 --- a/backend/components/media-resolver/helm/templates/deployment.yaml +++ b/backend/components/media-resolver/helm/templates/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 12 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/sources-api/helm/templates/deployment.yaml b/backend/components/sources-api/helm/templates/deployment.yaml index 9ce7cda665..4d23b16dea 100644 --- a/backend/components/sources-api/helm/templates/deployment.yaml +++ b/backend/components/sources-api/helm/templates/deployment.yaml @@ -50,6 +50,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 10 }} initContainers: @@ -67,3 +72,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/streams/helm/templates/deployment.yaml b/backend/components/streams/helm/templates/deployment.yaml index 067e9bf6c9..b0cd1dfb3f 100644 --- a/backend/components/streams/helm/templates/deployment.yaml +++ b/backend/components/streams/helm/templates/deployment.yaml @@ -48,6 +48,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 10 }} initContainers: @@ -67,3 +72,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/twilio/helm/templates/deployments.yaml b/backend/components/twilio/helm/templates/deployments.yaml index 0e3a6a5706..461dd6c30d 100644 --- a/backend/components/twilio/helm/templates/deployments.yaml +++ b/backend/components/twilio/helm/templates/deployments.yaml @@ -54,6 +54,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -141,3 +146,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/viber/helm/templates/deployments.yaml b/backend/components/viber/helm/templates/deployments.yaml index ebb0d5254b..55fbb10d72 100644 --- a/backend/components/viber/helm/templates/deployments.yaml +++ b/backend/components/viber/helm/templates/deployments.yaml @@ -49,6 +49,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -66,3 +71,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/webhook/helm/templates/deployments.yaml b/backend/components/webhook/helm/templates/deployments.yaml index 814a41c9ae..6c6cb5b1b1 100644 --- a/backend/components/webhook/helm/templates/deployments.yaml +++ b/backend/components/webhook/helm/templates/deployments.yaml @@ -55,6 +55,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.consumer.resources | indent 10 }} initContainers: @@ -157,3 +162,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/whatsapp/helm/templates/deployments.yaml b/backend/components/whatsapp/helm/templates/deployments.yaml index 1fcbff4b11..4c5e04b48a 100644 --- a/backend/components/whatsapp/helm/templates/deployments.yaml +++ b/backend/components/whatsapp/helm/templates/deployments.yaml @@ -113,6 +113,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.eventsRouter.resources | indent 10 }} initContainers: @@ -130,3 +135,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/docs/docs/changelog.md b/docs/docs/changelog.md index 0b0cb793ab..9d4aa94e5c 100644 --- a/docs/docs/changelog.md +++ b/docs/docs/changelog.md @@ -3,6 +3,64 @@ title: Changelog sidebar_label: 📝 Changelog --- +## 0.55.0 + +#### Changes + +- [[#4167](https://github.com/airyhq/airy/issues/4167)] Kafka certificate authentication [[#4168](https://github.com/airyhq/airy/pull/4168)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Change name for Pinecone connector [[#4127](https://github.com/airyhq/airy/pull/4127)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Add description for components [[#4126](https://github.com/airyhq/airy/pull/4126)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Add LLM filter in Catalog [[#4125](https://github.com/airyhq/airy/pull/4125)] + +#### 🚀 Features + +- [[#4139](https://github.com/airyhq/airy/issues/4139)] Add Flink connector [[#4147](https://github.com/airyhq/airy/pull/4147)] +- [[#3945](https://github.com/airyhq/airy/issues/3945)] Slack connector [[#4146](https://github.com/airyhq/airy/pull/4146)] +- [[#4144](https://github.com/airyhq/airy/issues/4144)] Adapt Frontend with Schema Manager [[#4145](https://github.com/airyhq/airy/pull/4145)] +- [[#4141](https://github.com/airyhq/airy/issues/4141)] Schema Registry Manager [[#4143](https://github.com/airyhq/airy/pull/4143)] +- [[#4137](https://github.com/airyhq/airy/issues/4137)] Improve Kafka Sections [[#4138](https://github.com/airyhq/airy/pull/4138)] +- [[#4132](https://github.com/airyhq/airy/issues/4132)] Added Copilot source to libs and apps [[#4133](https://github.com/airyhq/airy/pull/4133)] + +#### 🐛 Bug Fixes + +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Fix selecting fields in streams [[#4123](https://github.com/airyhq/airy/pull/4123)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Fix symbol [[#4122](https://github.com/airyhq/airy/pull/4122)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Update icons for LLM components [[#4120](https://github.com/airyhq/airy/pull/4120)] +- [[#4108](https://github.com/airyhq/airy/issues/4108)] Fix schemas and topics screens [[#4109](https://github.com/airyhq/airy/pull/4109)] + +#### 📚 Documentation + +- [[#4134](https://github.com/airyhq/airy/issues/4134)] Update the diagram in the README file [[#4136](https://github.com/airyhq/airy/pull/4136)] +- [[#4134](https://github.com/airyhq/airy/issues/4134)] Update main diagram [[#4135](https://github.com/airyhq/airy/pull/4135)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Improve docs [[#4131](https://github.com/airyhq/airy/pull/4131)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Improve docs [[#4128](https://github.com/airyhq/airy/pull/4128)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Docs - Remove Solutions link [[#4115](https://github.com/airyhq/airy/pull/4115)] + +#### 🧰 Maintenance + +- Bump google.golang.org/protobuf from 1.28.0 to 1.33.0 [[#4155](https://github.com/airyhq/airy/pull/4155)] +- Bump webpack-dev-middleware from 5.3.3 to 5.3.4 in /docs [[#4154](https://github.com/airyhq/airy/pull/4154)] +- Bump google.golang.org/protobuf from 1.28.0 to 1.33.0 in /infrastructure/lib/go/k8s/handler [[#4152](https://github.com/airyhq/airy/pull/4152)] +- Bump webpack-dev-middleware from 5.3.3 to 5.3.4 [[#4153](https://github.com/airyhq/airy/pull/4153)] +- Bump express from 4.18.2 to 4.19.2 [[#4151](https://github.com/airyhq/airy/pull/4151)] +- Bump express from 4.18.2 to 4.19.2 in /docs [[#4150](https://github.com/airyhq/airy/pull/4150)] +- Bump follow-redirects from 1.15.2 to 1.15.6 [[#4149](https://github.com/airyhq/airy/pull/4149)] +- Bump follow-redirects from 1.15.2 to 1.15.6 in /docs [[#4148](https://github.com/airyhq/airy/pull/4148)] +- Bump github.com/cyphar/filepath-securejoin from 0.2.3 to 0.2.4 [[#4118](https://github.com/airyhq/airy/pull/4118)] +- Bump @adobe/css-tools from 4.0.1 to 4.3.1 [[#4116](https://github.com/airyhq/airy/pull/4116)] +- Bump @svgr/plugin-svgo from 6.5.1 to 8.1.0 [[#4114](https://github.com/airyhq/airy/pull/4114)] +- Bump semver from 5.7.1 to 5.7.2 in /docs [[#4110](https://github.com/airyhq/airy/pull/4110)] +- Bump sass-loader from 13.1.0 to 13.3.2 [[#4103](https://github.com/airyhq/airy/pull/4103)] +- Bump word-wrap from 1.2.3 to 1.2.4 [[#4112](https://github.com/airyhq/airy/pull/4112)] + +#### Airy CLI + +You can download the Airy CLI for your operating system from the following links: + +[MacOS](https://airy-core-binaries.s3.amazonaws.com/0.55.0/darwin/amd64/airy) +[Linux](https://airy-core-binaries.s3.amazonaws.com/0.55.0/linux/amd64/airy) +[Windows](https://airy-core-binaries.s3.amazonaws.com/0.55.0/windows/amd64/airy.exe) + ## 0.54.0 #### 🚀 Features @@ -1348,38 +1406,3 @@ You can download the Airy CLI for your operating system from the following links [Linux](https://airy-core-binaries.s3.amazonaws.com/0.35.0/linux/amd64/airy) [Windows](https://airy-core-binaries.s3.amazonaws.com/0.35.0/windows/amd64/airy.exe) -## 0.34.0 - -#### Changes - -#### 🚀 Features - -- [[#2518](https://github.com/airyhq/airy/issues/2518)] Add fargate annotation [[#2540](https://github.com/airyhq/airy/pull/2540)] -- [[#2305](https://github.com/airyhq/airy/issues/2305)] Add CLI outdated version warning [[#2529](https://github.com/airyhq/airy/pull/2529)] - -#### 🐛 Bug Fixes - -- [[#2434](https://github.com/airyhq/airy/issues/2434)] Fix broken instagram Facebook inbox ingestion [[#2535](https://github.com/airyhq/airy/pull/2535)] -- [[#2457](https://github.com/airyhq/airy/issues/2457)] Fix upgrade to same version [[#2538](https://github.com/airyhq/airy/pull/2538)] -- [[#2510](https://github.com/airyhq/airy/issues/2510)] Improve error logging for helm install [[#2522](https://github.com/airyhq/airy/pull/2522)] -- [[#2255](https://github.com/airyhq/airy/issues/2255)] Fix helm chart url [[#2525](https://github.com/airyhq/airy/pull/2525)] -- [[#2523](https://github.com/airyhq/airy/issues/2523)] Fix VERSION and add changelog [[#2524](https://github.com/airyhq/airy/pull/2524)] -- [[#2473](https://github.com/airyhq/airy/issues/2473)] fix failing cypress test [[#2507](https://github.com/airyhq/airy/pull/2507)] - -#### 🧰 Maintenance - -- Bump react-redux from 7.2.5 to 7.2.6 [[#2539](https://github.com/airyhq/airy/pull/2539)] -- Bump reselect from 4.0.0 to 4.1.1 [[#2533](https://github.com/airyhq/airy/pull/2533)] -- Bump sass-loader from 12.1.0 to 12.3.0 [[#2534](https://github.com/airyhq/airy/pull/2534)] -- Bump @types/react-dom from 17.0.9 to 17.0.10 [[#2526](https://github.com/airyhq/airy/pull/2526)] -- Bump react-markdown from 7.0.1 to 7.1.0 [[#2527](https://github.com/airyhq/airy/pull/2527)] -- Bump webpack from 5.54.0 to 5.59.1 [[#2517](https://github.com/airyhq/airy/pull/2517)] - -#### Airy CLI - -You can download the Airy CLI for your operating system from the following links: - -[MacOS](https://airy-core-binaries.s3.amazonaws.com/0.34.0/darwin/amd64/airy) -[Linux](https://airy-core-binaries.s3.amazonaws.com/0.34.0/linux/amd64/airy) -[Windows](https://airy-core-binaries.s3.amazonaws.com/0.34.0/windows/amd64/airy.exe) - diff --git a/docs/docs/getting-started/installation/helm.md b/docs/docs/getting-started/installation/helm.md index e47ed9b447..4d3738cc0f 100644 --- a/docs/docs/getting-started/installation/helm.md +++ b/docs/docs/getting-started/installation/helm.md @@ -290,6 +290,39 @@ Run the following command to create the `Airy` platform without the bundled inst helm install airy airy/airy --timeout 10m --set prerequisites.kafka.enabled=false --values ./airy.yaml ``` +#### Confluent + +To connect to a Kafka instance in Confluent cloud, settings the `config.kafka.brokers` and `config.kafka.aurhJaas` is enough, prior to deploying the Helm chart. + +#### Aiven + +Aiven cloud uses a keystore and truststore certificates that need to be loaded on the workloads that are connecting to Kafka. Get the necessary certificates and connection files from Aiven using the `avn` CLI and place them in a separate directory. + +``` +avn service user-kafka-java-creds {KAFKA_INSTANCE} --username {USERNAME} -d ./aiven/ --password {PASSWORD} +``` + +Create a Kubernetes ConfigMap that contains the contents of the created directory: + +``` +kubectl create configmap kafka-config-certs --from-file aiven/ +``` + +Set the connection appropriate parameters in your `airy.yaml` file: + +```yaml +config: + kafka: + brokers: "the-aiven-kafka-broker-url" + keyTrustSecret: "the-key-trust-secret" +``` + +Then install Airy with the following command: + +```sh +helm install airy airy/airy --timeout 10m --set prerequisites.kafka.enabled=false --set global.kafkaCertAuth=true --values ./airy.yaml +``` + ### Kafka partitions per topic Currently all the default topics in the Airy instance are created with 10 partitions. To create these topics with a different number of partitions, add the following to your `airy.yaml` file before running `helm install` (before the initial creation of the topics): diff --git a/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml b/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml index 75cab9b8dc..3af2e13445 100644 --- a/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml @@ -60,6 +60,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.admin.resources | indent 10 }} initContainers: @@ -77,3 +82,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml b/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml index 362428ee8b..a461864d1d 100644 --- a/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.communication.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml b/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml index fd865b890b..363863fafc 100644 --- a/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml @@ -81,6 +81,11 @@ spec: initialDelaySeconds: 60 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.components.installer.resources | indent 10 }} initContainers: @@ -102,4 +107,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts - +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml b/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml index 6739cfe3c0..c4d95e0c57 100644 --- a/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.websocket.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml b/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml index c8650099f1..4b44212560 100644 --- a/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.unread_counter.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/config/kafka.yaml b/infrastructure/helm-chart/templates/config/kafka.yaml index a00933d800..23897e7214 100644 --- a/infrastructure/helm-chart/templates/config/kafka.yaml +++ b/infrastructure/helm-chart/templates/config/kafka.yaml @@ -13,3 +13,4 @@ data: {{- end }} KAFKA_SCHEMA_REGISTRY_URL: {{ .Values.config.kafka.schemaRegistryUrl }} KAFKA_COMMIT_INTERVAL_MS: "{{ .Values.config.kafka.commitInterval }}" + KAFKA_KEY_TRUST_SECRET: {{ .Values.config.kafka.keyTrustSecret }} diff --git a/infrastructure/helm-chart/values.yaml b/infrastructure/helm-chart/values.yaml index e4f6afa182..982a736b54 100644 --- a/infrastructure/helm-chart/values.yaml +++ b/infrastructure/helm-chart/values.yaml @@ -13,6 +13,7 @@ config: brokers: "kafka-headless:9092" zookeepers: "zookeeper:2181" authJaas: "" + keyTrustSecret: "" minimumReplicas: 1 schemaRegistryUrl: "http://schema-registry:8081" commitInterval: 1000 diff --git a/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java b/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java index b5739d9503..ca75dc902b 100644 --- a/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java +++ b/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java @@ -12,6 +12,10 @@ import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.Properties; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SslConfigs; +import java.util.HashMap; + public class KafkaConsumerWrapper { @@ -22,6 +26,7 @@ public class KafkaConsumerWrapper { private KafkaConsumer consumer; private String jaasConfig; + private String kafkaKeyTrustSecret; public KafkaConsumerWrapper(final String brokers, final String schemaRegistryUrl) { props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers); @@ -33,13 +38,22 @@ public KafkaConsumerWrapper(final String brokers, final String schemaRegistryUrl props.put(KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG, true); } - public KafkaConsumerWrapper withAuthJaas(String jaasConfig) { + public KafkaConsumerWrapper withAuthJaas(String jaasConfig, String kafkaKeyTrustSecret) { this.jaasConfig = jaasConfig; if(jaasConfig != null) { props.put("security.protocol", "SASL_SSL"); props.put("sasl.mechanism", "PLAIN"); props.put("sasl.jaas.config", jaasConfig); } + if (kafkaKeyTrustSecret != null) { + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); + props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.truststore.jks"); + props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "PKCS12"); + props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.keystore.p12"); + props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, kafkaKeyTrustSecret); + } return this; } diff --git a/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java b/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java index f44f5075aa..18ca499cab 100644 --- a/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java +++ b/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java @@ -30,6 +30,10 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SslConfigs; +import java.util.HashMap; + public class KafkaStreamsWrapper { private static final Logger log = AiryLoggerFactory.getLogger(KafkaStreamsWrapper.class); @@ -37,6 +41,7 @@ public class KafkaStreamsWrapper { private final String brokers; private final String schemaRegistryUrl; private String jaasConfig; + private String kafkaKeyTrustSecret; private long commitIntervalInMs; private long suppressIntervalInMs; private int threadCount; @@ -70,8 +75,9 @@ public KafkaStreamsWrapper(final String brokers, final String schemaRegistryUrl) healthCheckRunnerThread = new HealthCheckRunner(testMode); } - public KafkaStreamsWrapper withJaasConfig(String jaasConfig) { + public KafkaStreamsWrapper withJaasConfig(String jaasConfig, String kafkaKeyTrustSecret) { this.jaasConfig = jaasConfig; + this.kafkaKeyTrustSecret = kafkaKeyTrustSecret; return this; } @@ -227,6 +233,17 @@ public synchronized void start(final Topology topology, final String appId) thro props.put("sasl.jaas.config", jaasConfig); } + if (this.kafkaKeyTrustSecret != null) { + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); + props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.truststore.jks"); + props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "PKCS12"); + props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.keystore.p12"); + props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, kafkaKeyTrustSecret); + } + + props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, this.maxRequestSize); props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, this.fetchMaxBytes); diff --git a/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java b/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java index 0cd6c28122..f8488e2f6f 100644 --- a/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java +++ b/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java @@ -13,6 +13,9 @@ import org.springframework.context.annotation.Scope; import java.util.Properties; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SslConfigs; +import java.util.HashMap; @Configuration @PropertySource("classpath:kafka-core.properties") @@ -21,7 +24,8 @@ public class KafkaCoreConfig { @Lazy @Scope("prototype") public KafkaProducer kafkaProducer(@Value("${kafka.brokers}") final String brokers, @Value("${kafka.schema-registry-url}") final String schemaRegistryUrl, - @Value("${AUTH_JAAS:#{null}}") final String jaasConfig) { + @Value("${AUTH_JAAS:#{null}}") final String jaasConfig, + @Value("${KAFKA_KEY_TRUST_SECRET:#{null}}") final String kafkaKeyTrustSecret) { final Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers); @@ -37,6 +41,16 @@ public KafkaProducer kafkaProducer(@Value("${kafka.brokers}") final props.put("sasl.jaas.config", jaasConfig); } + if (kafkaKeyTrustSecret != null) { + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); + props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.truststore.jks"); + props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "PKCS12"); + props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.keystore.p12"); + props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, kafkaKeyTrustSecret); + } + return new KafkaProducer<>(props); } @@ -44,8 +58,9 @@ public KafkaProducer kafkaProducer(@Value("${kafka.brokers}") final @Lazy @Scope("prototype") public KafkaConsumerWrapper kafkaConsumer(@Value("${kafka.brokers}") final String brokers, @Value("${kafka.schema-registry-url}") final String schemaRegistryUrl, - @Value("${kafka.sasl.jaas.config:#{null}}") final String jaasConfig) { + @Value("${kafka.sasl.jaas.config:#{null}}") final String jaasConfig, + @Value("${KAFKA_KEY_TRUST_SECRET:#{null}}") final String kafkaKeyTrustSecret) { return new KafkaConsumerWrapper(brokers, schemaRegistryUrl) - .withAuthJaas(jaasConfig); + .withAuthJaas(jaasConfig, kafkaKeyTrustSecret); } } diff --git a/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java b/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java index f9995abfbd..217f2f9a6a 100644 --- a/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java +++ b/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java @@ -30,6 +30,9 @@ public class KafkaStreamsConfig { @Value("${AUTH_JAAS:#{null}}") private String jaasConfig; + @Value("${KAFKA_KEY_TRUST_SECRET:#{null}}") + private String kafkaKeyTrustSecret; + @Value("${kafka.rpc-port:0}") private int rpcPort; @@ -68,7 +71,7 @@ public class KafkaStreamsConfig { public KafkaStreamsWrapper airyKafkaStreams(@Value("${kafka.brokers}") final String brokers, @Value("${kafka.schema-registry-url}") final String schemaRegistryUrl) { return new KafkaStreamsWrapper(brokers, schemaRegistryUrl) .withCommitIntervalInMs(commitIntervalMs) - .withJaasConfig(jaasConfig) + .withJaasConfig(jaasConfig, kafkaKeyTrustSecret) .withSuppressIntervalInMs(suppressIntervalMs) .withThreadCount(streamsThreadCount) .withAppServerHost(rpcHost)