From 4b20fa01bd5df127b601163986e4a1f27bea5a9a Mon Sep 17 00:00:00 2001 From: Michelle Matthew Date: Wed, 4 Sep 2024 15:57:36 +0100 Subject: [PATCH] Publishing ES 11.5.0 docs Signed-off-by: Michelle Matthew --- _config.yml | 27 +- _connectors/kc-sink-ibm-mq/index.md | 2 +- _connectors/kc-sink-ibm-mq/installation.md | 1 - _connectors/kc-source-ibm-mq/index.md | 2 +- _data/navigation.yml | 237 ++- _data/reuse.yml | 3 +- _eem_11.3/02-installing/12-upgrading.md | 4 +- _ep_1.2/02-installing/12-upgrading.md | 8 +- _ep_1.2/03-getting-started/02-canvas.md | 11 +- .../04-security/06-gdpr-considerations.md | 2 +- .../05-replication/05-replication-failover.md | 2 +- .../07-administering/01-deployment-health.md | 2 +- _es_11.4/00-home.md | 2 +- _es_11.4/01-about/02-key-concepts.md | 2 +- _es_11.4/01-about/03-producing-messages.md | 4 +- _es_11.4/01-about/04-consuming-messages.md | 4 +- _es_11.4/01-about/05-partition-leadership.md | 2 +- _es_11.4/02-installing/02-planning.md | 4 +- .../02-installing/04-capacity-planning.md | 2 +- _es_11.4/02-installing/07-configuring.md | 6 +- .../03-getting-started/02-creating-topics.md | 4 +- .../04-security/03-secure-jmx-connections.md | 2 +- .../04-security/07-gdpr-considerations.md | 10 +- _es_11.4/06-connecting/02-connectors.md | 2 +- .../06-connecting/04-connecting-mq-source.md | 16 +- .../06-connecting/05-connecting-mq-sink.md | 2 +- .../06-running-connectors-on-zos.md | 2 +- .../07-administering/01-deployment-health.md | 2 +- .../07-administering/02-cluster-health.md | 6 +- .../07-administering/05a-audit-logging.md | 2 +- .../07-modifying-installation.md | 2 +- _es_11.4/07-administering/10-quotas.md | 4 +- _es_11.5/00-home.md | 92 ++ _es_11.5/01-about/01-overview.md | 67 + _es_11.5/01-about/01-whats-new.md | 31 + _es_11.5/01-about/02-key-concepts.md | 49 + _es_11.5/01-about/03-producing-messages.md | 150 ++ _es_11.5/01-about/04-consuming-messages.md | 134 ++ _es_11.5/01-about/05-partition-leadership.md | 21 + _es_11.5/02-installing/00-trying-out.md | 20 + _es_11.5/02-installing/01-prereqs.md | 221 +++ _es_11.5/02-installing/02-planning.md | 368 +++++ .../03-multizone-considerations.md | 49 + .../02-installing/04-capacity-planning.md | 111 ++ .../02-installing/05-preping-multizone.md | 110 ++ _es_11.5/02-installing/06-installing.md | 500 ++++++ .../02-installing/06a-installing-offline.md | 330 ++++ .../06b-installing-on-kubernetes.md | 205 +++ _es_11.5/02-installing/07-configuring.md | 1467 +++++++++++++++++ _es_11.5/02-installing/07b-integrating-eem.md | 80 + _es_11.5/02-installing/08-postinstallation.md | 158 ++ _es_11.5/02-installing/09-backup-restore.md | 278 ++++ .../02-installing/09-disaster-recovery.md | 72 + .../02-installing/10-moving-from-oss-kafka.md | 38 + _es_11.5/02-installing/11-uninstalling.md | 244 +++ _es_11.5/02-installing/12-upgrading.md | 396 +++++ _es_11.5/03-getting-started/01-logging-in.md | 185 +++ .../03-getting-started/02-creating-topics.md | 117 ++ .../03-getting-started/02a-managing-topics.md | 68 + .../03-generating-starter-app.md | 57 + .../03-getting-started/04-testing-loads.md | 168 ++ _es_11.5/03-getting-started/05-client.md | 57 + .../06-connecting-clients.md | 237 +++ .../06-using-kafka-console-tools.md | 102 ++ .../03-getting-started/07-sharing-topic.md | 38 + _es_11.5/04-schemas/01-schemas-overview.md | 89 + _es_11.5/04-schemas/02-creating-schemas.md | 121 ++ .../03-managing-schema-lifecycle.md | 150 ++ .../04-setting-java-apps-to-use-schemas.md | 286 ++++ .../05-setting-nonjava-apps-to-use-schemas.md | 62 + .../04-schemas/06-migrating-to-registry.md | 114 ++ .../07-using-schemas-with-rest-producer.md | 50 + ...etting-java-apps-to-use-apicurio-serdes.md | 490 ++++++ _es_11.5/04-security/01-managing-access.md | 522 ++++++ _es_11.5/04-security/02-encrypting-data.md | 18 + .../04-security/03-secure-jmx-connections.md | 119 ++ .../04-security/04-renewing-certificates.md | 86 + .../04-security/05-verifying-signature.md | 236 +++ _es_11.5/04-security/06-network-policies.md | 185 +++ .../04-security/07-gdpr-considerations.md | 139 ++ _es_11.5/04-security/08-fips.md | 34 + .../01-about-topic-mirroring.md | 101 ++ .../05-atopic-mirroring/02-mirrormaker.md | 44 + .../05-replication-failover.md | 191 +++ .../05-replication/01-about-replication.md | 43 + .../05-replication/02-planning-replication.md | 155 ++ .../03-setting-up-replication.md | 155 ++ .../05-replication/04-replication-health.md | 281 ++++ .../06-connecting/00-rest-producer-api.md | 278 ++++ .../06-connecting/00a-exposing-services.md | 86 + _es_11.5/06-connecting/01-kafka-bridge.md | 347 ++++ _es_11.5/06-connecting/02-connectors.md | 88 + .../06-connecting/02-setting-up-connectors.md | 472 ++++++ _es_11.5/06-connecting/03-connecting-mq.md | 109 ++ .../06-connecting/04-connecting-mq-source.md | 298 ++++ .../06-connecting/05-connecting-mq-sink.md | 420 +++++ .../06-running-connectors-on-zos.md | 279 ++++ .../07-administering/01-deployment-health.md | 70 + .../07-administering/02-cluster-health.md | 101 ++ _es_11.5/07-administering/03-topic-health.md | 32 + .../07-administering/04-consumer-group-lag.md | 73 + .../05-distributed-tracing.md | 93 ++ .../07-administering/05a-audit-logging.md | 314 ++++ .../06-external-monitoring.md | 15 + .../07-modifying-installation.md | 67 + .../07-administering/08-cruise-control.md | 376 +++++ _es_11.5/07-administering/09-scaling.md | 169 ++ _es_11.5/07-administering/10-quotas.md | 238 +++ .../07-administering/11-managing-multizone.md | 22 + .../07-administering/12-graceful-shutdown.md | 110 ++ .../01-EventStreams-api-reference.md | 336 ++++ ...EventStreamsGeoReplicator-api-reference.md | 34 + _es_11.5/08-troubleshooting/01-ts-intro.md | 13 + .../08-troubleshooting/02-gathering-logs.md | 9 + .../03-resources-not-available.md | 31 + .../08-troubleshooting/04-es-install-fails.md | 36 + .../08-troubleshooting/06-georep-error.md | 32 + .../08-troubleshooting/07-kakfa-producer.md | 54 + .../08-troubleshooting/08-kakfa-consumer.md | 48 + .../09-cloudctl-es-not-registered.md | 31 + .../10-cloudctl-es-fails.md | 26 + .../11-chrome-ubuntu-issue.md | 29 + .../17-error-removing-destination.md | 37 + .../08-troubleshooting/19-ui-403-error.md | 28 + .../30-default-scc-issues.md | 73 + .../32-authorization-failed-exceptions.md | 33 + .../42-pkcs12-keystore-java-client.md | 66 + .../08-troubleshooting/44-ocp-upgrade-fail.md | 57 + .../48-watched-namespace.md | 23 + .../49-apicurio-annotation.md | 50 + .../52-cruise-control-topics.md | 43 + .../53-default-psp-issues.md | 59 + .../54-georep-replicate-topic-fails.md | 41 + .../55-mq-connector-fails.md | 41 + .../08-troubleshooting/56-kafkaprotocol.md | 48 + _includes/masthead.html | 4 +- _pages/landing.md | 14 +- _support/gatherLogs.md | 4 +- _support/licensing.md | 6 +- _support/support-matrix.md | 1 + _tutorials/ep-tutorial-12.md | 2 +- images/ea-tutorials/example12-17.png | Bin 338955 -> 330496 bytes images/flowcard-status.png | Bin 0 -> 121867 bytes 143 files changed, 15569 insertions(+), 69 deletions(-) create mode 100644 _es_11.5/00-home.md create mode 100644 _es_11.5/01-about/01-overview.md create mode 100644 _es_11.5/01-about/01-whats-new.md create mode 100644 _es_11.5/01-about/02-key-concepts.md create mode 100644 _es_11.5/01-about/03-producing-messages.md create mode 100644 _es_11.5/01-about/04-consuming-messages.md create mode 100644 _es_11.5/01-about/05-partition-leadership.md create mode 100644 _es_11.5/02-installing/00-trying-out.md create mode 100644 _es_11.5/02-installing/01-prereqs.md create mode 100644 _es_11.5/02-installing/02-planning.md create mode 100644 _es_11.5/02-installing/03-multizone-considerations.md create mode 100644 _es_11.5/02-installing/04-capacity-planning.md create mode 100644 _es_11.5/02-installing/05-preping-multizone.md create mode 100644 _es_11.5/02-installing/06-installing.md create mode 100644 _es_11.5/02-installing/06a-installing-offline.md create mode 100644 _es_11.5/02-installing/06b-installing-on-kubernetes.md create mode 100644 _es_11.5/02-installing/07-configuring.md create mode 100644 _es_11.5/02-installing/07b-integrating-eem.md create mode 100644 _es_11.5/02-installing/08-postinstallation.md create mode 100644 _es_11.5/02-installing/09-backup-restore.md create mode 100644 _es_11.5/02-installing/09-disaster-recovery.md create mode 100644 _es_11.5/02-installing/10-moving-from-oss-kafka.md create mode 100644 _es_11.5/02-installing/11-uninstalling.md create mode 100644 _es_11.5/02-installing/12-upgrading.md create mode 100644 _es_11.5/03-getting-started/01-logging-in.md create mode 100644 _es_11.5/03-getting-started/02-creating-topics.md create mode 100644 _es_11.5/03-getting-started/02a-managing-topics.md create mode 100644 _es_11.5/03-getting-started/03-generating-starter-app.md create mode 100644 _es_11.5/03-getting-started/04-testing-loads.md create mode 100644 _es_11.5/03-getting-started/05-client.md create mode 100644 _es_11.5/03-getting-started/06-connecting-clients.md create mode 100644 _es_11.5/03-getting-started/06-using-kafka-console-tools.md create mode 100644 _es_11.5/03-getting-started/07-sharing-topic.md create mode 100644 _es_11.5/04-schemas/01-schemas-overview.md create mode 100644 _es_11.5/04-schemas/02-creating-schemas.md create mode 100644 _es_11.5/04-schemas/03-managing-schema-lifecycle.md create mode 100644 _es_11.5/04-schemas/04-setting-java-apps-to-use-schemas.md create mode 100644 _es_11.5/04-schemas/05-setting-nonjava-apps-to-use-schemas.md create mode 100644 _es_11.5/04-schemas/06-migrating-to-registry.md create mode 100644 _es_11.5/04-schemas/07-using-schemas-with-rest-producer.md create mode 100644 _es_11.5/04-schemas/08-setting-java-apps-to-use-apicurio-serdes.md create mode 100644 _es_11.5/04-security/01-managing-access.md create mode 100644 _es_11.5/04-security/02-encrypting-data.md create mode 100644 _es_11.5/04-security/03-secure-jmx-connections.md create mode 100644 _es_11.5/04-security/04-renewing-certificates.md create mode 100644 _es_11.5/04-security/05-verifying-signature.md create mode 100644 _es_11.5/04-security/06-network-policies.md create mode 100644 _es_11.5/04-security/07-gdpr-considerations.md create mode 100644 _es_11.5/04-security/08-fips.md create mode 100644 _es_11.5/05-atopic-mirroring/01-about-topic-mirroring.md create mode 100644 _es_11.5/05-atopic-mirroring/02-mirrormaker.md create mode 100644 _es_11.5/05-atopic-mirroring/05-replication-failover.md create mode 100644 _es_11.5/05-replication/01-about-replication.md create mode 100644 _es_11.5/05-replication/02-planning-replication.md create mode 100644 _es_11.5/05-replication/03-setting-up-replication.md create mode 100644 _es_11.5/05-replication/04-replication-health.md create mode 100644 _es_11.5/06-connecting/00-rest-producer-api.md create mode 100644 _es_11.5/06-connecting/00a-exposing-services.md create mode 100644 _es_11.5/06-connecting/01-kafka-bridge.md create mode 100644 _es_11.5/06-connecting/02-connectors.md create mode 100644 _es_11.5/06-connecting/02-setting-up-connectors.md create mode 100644 _es_11.5/06-connecting/03-connecting-mq.md create mode 100644 _es_11.5/06-connecting/04-connecting-mq-source.md create mode 100644 _es_11.5/06-connecting/05-connecting-mq-sink.md create mode 100644 _es_11.5/06-connecting/06-running-connectors-on-zos.md create mode 100644 _es_11.5/07-administering/01-deployment-health.md create mode 100644 _es_11.5/07-administering/02-cluster-health.md create mode 100644 _es_11.5/07-administering/03-topic-health.md create mode 100644 _es_11.5/07-administering/04-consumer-group-lag.md create mode 100644 _es_11.5/07-administering/05-distributed-tracing.md create mode 100644 _es_11.5/07-administering/05a-audit-logging.md create mode 100644 _es_11.5/07-administering/06-external-monitoring.md create mode 100644 _es_11.5/07-administering/07-modifying-installation.md create mode 100644 _es_11.5/07-administering/08-cruise-control.md create mode 100644 _es_11.5/07-administering/09-scaling.md create mode 100644 _es_11.5/07-administering/10-quotas.md create mode 100644 _es_11.5/07-administering/11-managing-multizone.md create mode 100644 _es_11.5/07-administering/12-graceful-shutdown.md create mode 100644 _es_11.5/07a-reference/01-EventStreams-api-reference.md create mode 100644 _es_11.5/07a-reference/02-EventStreamsGeoReplicator-api-reference.md create mode 100644 _es_11.5/08-troubleshooting/01-ts-intro.md create mode 100644 _es_11.5/08-troubleshooting/02-gathering-logs.md create mode 100644 _es_11.5/08-troubleshooting/03-resources-not-available.md create mode 100644 _es_11.5/08-troubleshooting/04-es-install-fails.md create mode 100644 _es_11.5/08-troubleshooting/06-georep-error.md create mode 100644 _es_11.5/08-troubleshooting/07-kakfa-producer.md create mode 100644 _es_11.5/08-troubleshooting/08-kakfa-consumer.md create mode 100644 _es_11.5/08-troubleshooting/09-cloudctl-es-not-registered.md create mode 100644 _es_11.5/08-troubleshooting/10-cloudctl-es-fails.md create mode 100644 _es_11.5/08-troubleshooting/11-chrome-ubuntu-issue.md create mode 100644 _es_11.5/08-troubleshooting/17-error-removing-destination.md create mode 100644 _es_11.5/08-troubleshooting/19-ui-403-error.md create mode 100644 _es_11.5/08-troubleshooting/30-default-scc-issues.md create mode 100644 _es_11.5/08-troubleshooting/32-authorization-failed-exceptions.md create mode 100644 _es_11.5/08-troubleshooting/42-pkcs12-keystore-java-client.md create mode 100644 _es_11.5/08-troubleshooting/44-ocp-upgrade-fail.md create mode 100644 _es_11.5/08-troubleshooting/48-watched-namespace.md create mode 100644 _es_11.5/08-troubleshooting/49-apicurio-annotation.md create mode 100644 _es_11.5/08-troubleshooting/52-cruise-control-topics.md create mode 100644 _es_11.5/08-troubleshooting/53-default-psp-issues.md create mode 100644 _es_11.5/08-troubleshooting/54-georep-replicate-topic-fails.md create mode 100644 _es_11.5/08-troubleshooting/55-mq-connector-fails.md create mode 100644 _es_11.5/08-troubleshooting/56-kafkaprotocol.md create mode 100644 images/flowcard-status.png diff --git a/_config.yml b/_config.yml index 9aa96642..0027dc35 100644 --- a/_config.yml +++ b/_config.yml @@ -270,11 +270,17 @@ collections: "es_11.4": output: true version: "11.4" + permalink: /es/:collection/:categories/:slug/ + order: 15 + product: es + "es_11.5": + output: true + version: "11.5" tag: Latest latest: true permalink: /es/:categories/:slug/ - order: 15 - product: es + order: 16 + product: es "eem_11.0": version: "11.0" output: true @@ -664,6 +670,23 @@ defaults: author_profile: false share: false comment: false + mastheadNavItem: Event Streams + versioned: true + sidebar: + nav: "114docs" + # _11.5 + - scope: + path: "" + type: "es_11.5" + values: + collection: "es_11.5" + version: "11.5" + product: es + layout: single + read_time: false + author_profile: false + share: false + comment: false latest: true mastheadNavItem: Event Streams versioned: true diff --git a/_connectors/kc-sink-ibm-mq/index.md b/_connectors/kc-sink-ibm-mq/index.md index 512af238..819b4a6d 100644 --- a/_connectors/kc-sink-ibm-mq/index.md +++ b/_connectors/kc-sink-ibm-mq/index.md @@ -14,4 +14,4 @@ categories: - Messaging --- -{{site.data.reuse.kafka-connect-mq-sink}} supports connecting to IBM MQ in both bindings and client mode, and offers both exactly-once and at-least-once delivery of data from IBM MQ to Apache Kafka. \ No newline at end of file +{{site.data.reuse.kafka-connect-mq-sink}} supports connecting to IBM MQ in both bindings and client mode, and offers both exactly-once and at-least-once delivery of data from IBM MQ to Apache Kafka. diff --git a/_connectors/kc-sink-ibm-mq/installation.md b/_connectors/kc-sink-ibm-mq/installation.md index abaff226..4cc3cf7d 100644 --- a/_connectors/kc-sink-ibm-mq/installation.md +++ b/_connectors/kc-sink-ibm-mq/installation.md @@ -4,7 +4,6 @@ forID: kc-sink-mq categories: [sink] --- - {{site.data.reuse.es_name}} provides additional help for setting up a Kafka Connect environment and starting the MQ sink connector. Log in to the {{site.data.reuse.es_name}} UI, click the **Toolbox** tab and scroll to the **Connectors** section. You can download the MQ sink connector from GitHub: diff --git a/_connectors/kc-source-ibm-mq/index.md b/_connectors/kc-source-ibm-mq/index.md index 1b94c638..96e95c87 100644 --- a/_connectors/kc-source-ibm-mq/index.md +++ b/_connectors/kc-source-ibm-mq/index.md @@ -14,4 +14,4 @@ categories: - Messaging --- -{{site.data.reuse.kafka-connect-mq-source}} supports connecting to IBM MQ in both bindings and client mode, and offers both exactly-once and at-least-once delivery of data from IBM MQ to Apache Kafka. \ No newline at end of file +{{site.data.reuse.kafka-connect-mq-source}} supports connecting to IBM MQ in both bindings and client mode, and offers both exactly-once and at-least-once delivery of data from IBM MQ to Apache Kafka. diff --git a/_data/navigation.yml b/_data/navigation.yml index 60408a85..0fb2da15 100644 --- a/_data/navigation.yml +++ b/_data/navigation.yml @@ -1033,7 +1033,7 @@ latest_epdocs: - title: "Event Processing fails with `Unable to find main key backup` error" url: /troubleshooting/no-main-key-backup/ -# Event Streams 11.4 docs +# Event Streams 11.5 docs latestdocs: - title: About @@ -1051,6 +1051,241 @@ latestdocs: - title: "Partition leadership" url: /about/partition-leadership/ + - title: Installing and upgrading + children: + - title: "Trying out Event Streams" + url: /installing/trying-out/ + - title: "Prerequisites" + url: /installing/prerequisites/ + - title: "Planning your installation" + url: /installing/planning/ + - title: "Considerations for multizone deployments" + url: /installing/multizone-considerations/ + - title: "Performance and capacity planning" + url: /installing/capacity-planning/ + - title: "Preparing for multizone clusters" + url: /installing/preparing-multizone/ + - title: "Installing on OpenShift Container Platform" + url: /installing/installing/ + - title: "Installing in an offline environment" + url: /installing/offline/ + - title: "Installing on other Kubernetes platforms" + url: /installing/installing-on-kubernetes/ + - title: "Configuring" + url: /installing/configuring/ + - title: "Integrating with Event Endpoint Management" + url: /installing/integrating-eem/ + - title: "Post-installation tasks" + url: /installing/post-installation/ + - title: "Backing up and restoring on OpenShift" + url: /installing/backup-restore/ + - title: "Configuring disaster recovery topologies" + url: /installing/disaster-recovery/ + - title: "Migrating from open-source Kafka" + url: /installing/moving-from-oss-kafka/ + - title: "Uninstalling" + url: /installing/uninstalling/ + - title: "Upgrading" + url: /installing/upgrading/ + - title: Getting started + children: + - title: "Logging in" + url: /getting-started/logging-in/ + - title: "Creating a Kafka topic" + url: /getting-started/creating-topics/ + - title: "Managing Kafka topics" + url: /getting-started/managing-topics/ + - title: "Running a starter application" + url: /getting-started/generating-starter-app/ + - title: "Creating and testing message loads" + url: /getting-started/testing-loads/ + - title: "Creating Kafka client applications" + url: /getting-started/client/ + - title: "Connecting clients" + url: /getting-started/connecting/ + - title: "Using Apache Kafka console tools" + url: /getting-started/using-kafka-console-tools/ + - title: "Sharing topic with Event Endpoint Management" + url: /getting-started/sharing-topic/ + - title: Schemas + children: + - title: "Schemas overview" + url: /schemas/overview/ + - title: "Creating and adding schemas" + url: /schemas/creating/ + - title: "Managing schema lifecycle" + url: /schemas/manage-lifecycle/ + - title: "Setting Java applications to use schemas" + url: /schemas/setting-java-apps/ + - title: "Setting non-Java applications to use schemas" + url: /schemas/setting-nonjava-apps/ + - title: "Migrating to Event Streams schema registry" + url: /schemas/migrating/ + - title: "Using schemas with the REST producer API" + url: /schemas/using-with-rest-producer/ + - title: "Setting Java applications to use schemas with the Apicurio Registry serdes library" + url: /schemas/setting-java-apps-apicurio-serdes/ + - title: Security + children: + - title: "Managing access" + url: /security/managing-access/ + - title: "Encrypting your data" + url: /security/encrypting-data/ + - title: "Configuring secure JMX connections" + url: /security/secure-jmx-connections/ + - title: "Renewing certificates" + url: /security/renewing-certificates/ + - title: "Verifying container image signatures" + url: /security/verifying-signature/ + - title: "Network policies" + url: /security/network-policies/ + - title: "Considerations for GDPR" + url: /security/gdpr-considerations/ + - title: "Enabling FIPS" + url: /security/fips/ + - title: Topic mirroring + children: + - title: "About topic mirroring" + url: /mirroring/about/ + - title: "About MirrorMaker" + url: /mirroring/mirrormaker/ + - title: "Switching clusters" + url: /mirroring/failover/ + - title: Geo-replication + children: + - title: "About geo-replication" + url: /georeplication/about/ + - title: "Planning for geo-replication" + url: /georeplication/planning/ + - title: "Setting up geo-replication" + url: /georeplication/setting-up/ + - title: "Monitoring and managing geo-replication" + url: /georeplication/health/ + - title: Connecting external systems + children: + - title: "Event Streams producer API" + url: /connecting/rest-api/ + - title: "Exposing services for external access" + url: /connecting/expose-service/ + - title: "Kafka Bridge" + url: /connecting/kafka-bridge/ + - title: "Kafka Connect and connectors" + url: /connecting/connectors/ + - title: "Setting up and running connectors" + url: /connecting/setting-up-connectors/ + - title: "Connecting to IBM MQ" + url: /connecting/mq/ + - title: "Running the MQ source connector" + url: /connecting/mq/source/ + - title: "Running the MQ sink connector" + url: /connecting/mq/sink/ + - title: "Running connectors on IBM z/OS" + url: /connecting/mq/zos/ + - title: Administering + children: + - title: "Monitoring deployment health" + url: /administering/deployment-health/ + - title: "Monitoring Kafka cluster health" + url: /administering/cluster-health/ + - title: "Monitoring topic health" + url: /administering/topic-health/ + - title: "Monitoring consumer group lag" + url: /administering/consumer-lag/ + - title: "Monitoring applications with distributed tracing" + url: /administering/tracing/ + - title: "Auditing Kafka" + url: /administering/auditing-kafka/ + - title: "Monitoring with external tools" + url: /administering/external-monitoring/ + - title: "Modifying installation settings" + url: /administering/modifying-installation/ + - title: "Optimizing Kafka cluster with Cruise Control" + url: /administering/cruise-control/ + - title: "Scaling" + url: /administering/scaling/ + - title: "Setting client quotas" + url: /administering/quotas/ + - title: "Managing a multizone setup" + url: /administering/managing-multizone/ + - title: "Stopping and starting Event Streams" + url: /administering/stopping-starting/ + - title: Reference + children: + - title: "API reference for the Event Streams CRDs" + url: /reference/api-reference-es/ + - title: "API reference for the geo-replicator CRDs" + url: /reference/api-reference-esgr/ + - title: REST Producer API + url: /../api/ + - title: Schema registry API + url: /../schema-api/ + - title: Troubleshooting + children: + - title: "Troubleshooting overview" + url: /troubleshooting/intro/ + - title: "Gathering logs" + url: /troubleshooting/gathering-logs/ + - title: "Resources not available" + url: /troubleshooting/resources-not-available/ + - title: "Event Streams installation reports Blocked status" + url: /troubleshooting/es-install-fails/ + - title: "Error when creating multiple geo-replicators" + url: /troubleshooting/georeplication-error/ + - title: "TimeoutException when using standard Kafka producer" + url: /troubleshooting/kafka-producer-error/ + - title: "Standard Kafka consumer hangs" + url: /troubleshooting/kafka-consumer-hangs/ + - title: "Command 'cloudctl es' fails with 'not a registered command' error" + url: /troubleshooting/cloudctl-es-not-registered/ + - title: "Command 'cloudctl es' produces 'FAILED' message" + url: /troubleshooting/cloudctl-es-fails/ + - title: "UI does not open when using Chrome on Ubuntu" + url: /troubleshooting/chrome-ubuntu-issue/ + - title: "Unable to remove destination cluster" + url: /troubleshooting/error-removing-destination/ + - title: "403 error when signing in" + url: /troubleshooting/ui-403-error/ + - title: "Event Streams not installing due to Security Context Constraint (SCC) issues" + url: /troubleshooting/default-scc-issues/ + - title: "Client receives AuthorizationException when communicating with brokers" + url: /troubleshooting/authorization-failed-exceptions/ + - title: "Client receives 'Failed to load SSL keystore' message when communicating with brokers" + url: /troubleshooting/pkcs12-keystore-java-client/ + - title: "OpenShift upgrade: fixing scheduling on node and node degraded errors" + url: /troubleshooting/ocp-upgrade-fail/ + - title: "Apicurio authentication errors due to User Operator watchedNamespace" + url: /troubleshooting/watched-namespace/ + - title: "Clients using schemas fail with Apicurio 2.5.0 or later" + url: /troubleshooting/upgrade-apicurio/ + - title: "KafkaRebalance custom resource remains in PendingProposal state" + url: /troubleshooting/kafkarebalance-pendingproposal/ + - title: "Event Streams not installing due to Pod Security Policies (PSP) issues" + url: /troubleshooting/default-psp-issues/ + - title: "Geo-replicator fails when replicating a topic" + url: /troubleshooting/georep-fails/ + - title: "Errors in IBM MQ connectors" + url: /troubleshooting/mq-connector-fails/ + - title: "Upgrading to 11.4.x fails due to `inter.broker.protocol.version`" + url: /troubleshooting/kafka-protocol-error/ + +# Sidebar navigation for Event Streams 11.4 docs + +114docs: + - title: About + children: + - title: "Introduction" + url: /about/overview/ + - title: "What's new" + url: /about/whats-new/ + - title: "Key concepts" + url: /about/key-concepts/ + - title: "Producing messages" + url: /about/producing-messages/ + - title: "Consuming messages" + url: /about/consuming-messages/ + - title: "Partition leadership" + url: /about/partition-leadership/ + - title: Installing and upgrading children: - title: "Trying out Event Streams" diff --git a/_data/reuse.yml b/_data/reuse.yml index e21981d7..96d57fa3 100644 --- a/_data/reuse.yml +++ b/_data/reuse.yml @@ -28,8 +28,9 @@ egw: "Event Gateway" # {{site.data.reuse.eem_manager}} eem_manager: "Event Manager" + # {{site.data.reuse.eem_ubp_license_id}} -eem_ubp_license_id: "L-GGQD-G7AYJD" +eem_ubp_license_id: "L-FTGN-WUM5C5" # {{site.data.reuse.egw_short}} egw_short: "Event Gateway" diff --git a/_eem_11.3/02-installing/12-upgrading.md b/_eem_11.3/02-installing/12-upgrading.md index 88b7e5cd..f871e579 100644 --- a/_eem_11.3/02-installing/12-upgrading.md +++ b/_eem_11.3/02-installing/12-upgrading.md @@ -80,7 +80,9 @@ Before you can upgrade to the latest version, make the catalog source for the ve - Specific versions: If you used the CASE bundle to install catalog source for a specific previous version, you must download and use a new CASE bundle for the version you want to upgrade to. - If you used the CASE bundle for an online install, [apply the new catalog source](../installing/#adding-specific-versions) to update the `CatalogSource`. - If you used the CASE bundle for an offline install that uses a private registry, follow the instructions in [installing offline](../offline/#download-the-case-bundle) to remirror images and update the `CatalogSource`. - - In both cases, wait for the `status.installedCSV` field in the `Subscription` to update. It should eventually reflect the latest version available in the new `CatalogSource` image for the currently selected channel in the `Subscription`. In the {{site.data.reuse.openshift_short}} web console, the current version of the operator is shown under `Installed Operators`. Using the CLI, when you check the status of the `Subscription` custom resource, the `status.installedCSV` field shows the current operator version. + - In both cases, wait for the `status.installedCSV` field in the `Subscription` to update. It eventually reflects the latest version available in the new `CatalogSource` image for the currently selected channel in the `Subscription`: + - In the {{site.data.reuse.openshift_short}} web console, the current version of the operator is displayed under `Installed Operators`. + - If you are using the CLI, check the status of the `Subscription` custom resource, the `status.installedCSV` field shows the current operator version. The change to a new Channel, if needed, would be a later step. diff --git a/_ep_1.2/02-installing/12-upgrading.md b/_ep_1.2/02-installing/12-upgrading.md index cafd62a5..5313e54f 100644 --- a/_ep_1.2/02-installing/12-upgrading.md +++ b/_ep_1.2/02-installing/12-upgrading.md @@ -89,7 +89,9 @@ Before you can upgrade to the latest version, the catalog source for the new ver - If you used the CASE bundle for an online install, [apply the new catalog source](../installing/#adding-specific-versions) to update the `CatalogSource`. - If you used the CASE bundle for an offline install that uses a private registry, follow the instructions in [installing offline](../offline/#download-the-case-bundle) to remirror images and update the `CatalogSource`. - - In both cases, wait for the `status.installedCSV` field in the `Subscription` to update. It should eventually reflect the latest version available in the new `CatalogSource` image for the currently selected channel in the `Subscription`. In the {{site.data.reuse.openshift_short}} web console, the current version of the operator is shown under `Installed Operators`. Using the CLI, when you check the status of the `Subscription` custom resource, the `status.installedCSV` field shows the current operator version. + - In both cases, wait for the `status.installedCSV` field in the `Subscription` to update. It eventually reflects the latest version available in the new `CatalogSource` image for the currently selected channel in the `Subscription`: + - In the {{site.data.reuse.openshift_short}} web console, the current version of the operator is displayed under `Installed Operators`. + - If you are using the CLI, check the status of the `Subscription` custom resource, the `status.installedCSV` field shows the current operator version. @@ -304,9 +306,9 @@ After the upgrade, verify the status of the {{site.data.reuse.ep_name}} and Flin After upgrading your operators to {{site.data.reuse.ep_name}} 1.2.x from version 1.1.x, you must update [the license ID]({{ '/support/licensing/#ibm-event-automation-license-information' | relative_url }}) value in the `spec.license.license` field of your custom resources, depending on the program that you purchased. -You can make this change via the console or the CLI/API, and it is required for both OLM and Helm installations. +You can make this change by using the web console or the CLI, and it is required for both OLM and Helm installations. -The components will be in an error state until you do this, and will not run the new version until the new license ID is entered. After you change the license IDs, check the custom resource status to confirm they are successfully running the new version. +The components will show errors and will not work with the new version until you update the license ID. After you change the license IDs, check the custom resource status to confirm they are successfully running the new version. ### Restart your flows diff --git a/_ep_1.2/03-getting-started/02-canvas.md b/_ep_1.2/03-getting-started/02-canvas.md index be697c38..94a33f79 100644 --- a/_ep_1.2/03-getting-started/02-canvas.md +++ b/_ep_1.2/03-getting-started/02-canvas.md @@ -119,10 +119,17 @@ The clothing company selected **Include historical** to run the filter on the hi ## Flow statuses -A flow status indicates the current state of the flow. A flow can be in one of the following states: +A flow status indicates the current state of the flow. To view the status of a flow, navigate to the **Flows** section on the homepage of the {{site.data.reuse.ep_name}} UI. Each flow tile displays the current status of the flow. + +![Flow tiles displaying various statuses]({{ 'images' | relative_url }}/flowcard-status.png "Image of flow tiles displaying various statuses") + + +A flow can be in one of the following states: - **Draft:** Indicates that the flow includes one or more nodes that need to be configured. The flow cannot be run. - **Valid:** Indicates that all nodes in the flow are configured and valid. The flow is ready to run. - **Invalid:** Indicates that the nodes in the flow are configured but have a validation error, or a required node is missing. The flow cannot be run. - **Running:** Indicates that the flow is configured, validated, running, and generating output. -- **Error:** Indicates that an error occurred during the runtime of a previously running flow. \ No newline at end of file +- **Error:** Indicates that an error occurred during the runtime of a previously running flow. + +**Tip:** You can click the icon next to **Invalid** and **Error** states to find more information about the error. diff --git a/_es_10.0/04-security/06-gdpr-considerations.md b/_es_10.0/04-security/06-gdpr-considerations.md index 96125d64..5055c95d 100644 --- a/_es_10.0/04-security/06-gdpr-considerations.md +++ b/_es_10.0/04-security/06-gdpr-considerations.md @@ -121,7 +121,7 @@ The Kafka core APIs can be used to access message data within the {{site.data.re For more information about controlling access to data stored in {{site.data.reuse.es_name}}, see [managing access](../managing-access). -Cluster-level configuration and resources, including logs that might contain message data, are accessible through the {{site.data.reuse.openshift_short}} [web console](https://docs.openshift.com/container-platform/4.4/web_console/web-console.html) and by using the [`oc` CLI](https://docs.openshift.com/container-platform/4.4/cli_reference/openshift_cli/getting-started-cli.html#cli-logging-in_cli-developer-commands). +Cluster-level configuration and resources, including logs that might contain message data, are accessible through the {{site.data.reuse.openshift_short}} [web console](https://docs.openshift.com/container-platform/4.4/web_console/web-console.html) and by using the `oc` [CLI](https://docs.openshift.com/container-platform/4.4/cli_reference/openshift_cli/getting-started-cli.html#cli-logging-in_cli-developer-commands). [Access and authorization controls](https://kubernetes.io/docs/reference/access-authn-authz/controlling-access/){:target="_blank"} can be used to control which users are able to access this cluster-level information. diff --git a/_es_10.0/05-replication/05-replication-failover.md b/_es_10.0/05-replication/05-replication-failover.md index b4de9f22..ba89a909 100644 --- a/_es_10.0/05-replication/05-replication-failover.md +++ b/_es_10.0/05-replication/05-replication-failover.md @@ -96,7 +96,7 @@ Geo-replication uses the Kafka Mirror Maker 2.0 `MirrorCheckpointConnector` to a **Note**: Consumer offset checkpoint topics are internal topics that are not displayed in the UI and CLI. Run the following CLI command to include internal topics in the topic listing:\\ `cloudctl es topics --internal`. -When processing messages from the destination cluster, you can use the checkpoints to start consuming from an offset that is equivalent to the last committed offset on the origin cluster. If your application is written in Java, Kafka's [`RemoteClusterUtils`](https://kafka.apache.org/25/javadoc/index.html?org/apache/kafka/connect/mirror/RemoteClusterUtils.html){:target="_blank"} class provides the `translateOffsets()` utility method to retrieve the destination cluster offsets for a consumer group from the checkpoints topic. You can then use the `KafkaConsumer.seek()` method to override the offsets that the consumer will use on the next `poll`. +When processing messages from the destination cluster, you can use the checkpoints to start consuming from an offset that is equivalent to the last committed offset on the origin cluster. If your application is written in Java, Kafka's [RemoteClusterUtils](https://kafka.apache.org/25/javadoc/org/apache/kafka/connect/mirror/RemoteClusterUtils.html){:target="_blank"} class provides the `translateOffsets()` utility method to retrieve the destination cluster offsets for a consumer group from the checkpoints topic. You can then use the `KafkaConsumer.seek()` method to override the offsets that the consumer will use on the next `poll`. For example, the following Java code snippet will update the `example-group` consumer group offset from the `origin-cluster` cluster to the destination cluster equivalent: diff --git a/_es_10.0/07-administering/01-deployment-health.md b/_es_10.0/07-administering/01-deployment-health.md index ba9dded1..d66266de 100644 --- a/_es_10.0/07-administering/01-deployment-health.md +++ b/_es_10.0/07-administering/01-deployment-health.md @@ -52,6 +52,6 @@ If any of the components are not ready for an extended period of time, check the 2. To retrieve detailed log data for a pod to help analyze problems, use the following command:\\ `oc -n logs -c ` -For more information about debugging, see the [Kubernetes documentation](https://kubernetes.io/docs/tasks/debug-application-cluster/debug-application-introspection/#using-kubectl-describe-pod-to-fetch-details-about-pod){:target=”\_blank”}. You can use the [`oc` command](https://docs.openshift.com/container-platform/4.4/cli_reference/openshift_cli/usage-oc-kubectl.html){:target=“\_blank”} instead of `kubectl` to perform the debugging steps. +For more information about debugging, see the [Kubernetes documentation](https://kubernetes.io/docs/tasks/debug-application-cluster/debug-application-introspection/#using-kubectl-describe-pod-to-fetch-details-about-pod){:target=”\_blank”}. You can use the `oc` [command](https://docs.openshift.com/container-platform/4.4/cli_reference/openshift_cli/usage-oc-kubectl.html){:target=“\_blank”} instead of `kubectl` to perform the debugging steps. **Note:** After a component restarts, the `oc` command retrieves the logs for the new instance of the container. To retrieve the logs for a previous instance of the container, add the `-–previous` option to the `oc logs` command. diff --git a/_es_11.4/00-home.md b/_es_11.4/00-home.md index 0903abfd..cf0d7360 100644 --- a/_es_11.4/00-home.md +++ b/_es_11.4/00-home.md @@ -1,7 +1,7 @@ --- title: "Home" layout: splash -permalink: /es/ +permalink: /es/:collection/ quickLinks: - link: - title: Introduction diff --git a/_es_11.4/01-about/02-key-concepts.md b/_es_11.4/01-about/02-key-concepts.md index 52ab0823..6678dc18 100644 --- a/_es_11.4/01-about/02-key-concepts.md +++ b/_es_11.4/01-about/02-key-concepts.md @@ -46,4 +46,4 @@ To learn more, see the following information: * [Producing messages](../producing-messages) * [Consuming messages](../consuming-messages) * [Partition leadership](../partition-leadership/) -* [Apache Kafka documentation](https://kafka.apache.org/documentation.html){:target="_blank"} +* [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} diff --git a/_es_11.4/01-about/03-producing-messages.md b/_es_11.4/01-about/03-producing-messages.md index da6d4926..ab589b78 100644 --- a/_es_11.4/01-about/03-producing-messages.md +++ b/_es_11.4/01-about/03-producing-messages.md @@ -34,7 +34,7 @@ max.block.ms | The number of milliseconds that a send or metadata request can b max.in.flight.requests.per.connection | The maximum number of unacknowledged requests that the client sends on a connection before blocking further requests. | 1,… |5 request.timeout.ms | The maximum amount of time the producer waits for a response to a request. If the response is not received before the timeout elapses, the request is retried or fails if the number of retries has been exhausted. | 0,… |30000 (30 seconds) -Many more configuration settings are available, but ensure that you read the [Apache Kafka documentation](https://kafka.apache.org/documentation.html){:target="_blank"} thoroughly before experimenting with them. +Many more configuration settings are available, but ensure that you read the [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} thoroughly before experimenting with them. ## Partitioning @@ -147,4 +147,4 @@ To learn more, see the following information: - [Consuming messages](../consuming-messages) - [Partition leadership](../partition-leadership/) -- [Apache Kafka documentation](https://kafka.apache.org/documentation.html){:target="_blank"} +- [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} diff --git a/_es_11.4/01-about/04-consuming-messages.md b/_es_11.4/01-about/04-consuming-messages.md index 42583eb7..3592365a 100644 --- a/_es_11.4/01-about/04-consuming-messages.md +++ b/_es_11.4/01-about/04-consuming-messages.md @@ -34,7 +34,7 @@ max.poll.records | The maximum number of records returned in a call to poll() session.timeout.ms | The number of milliseconds within which a consumer heartbeat must be received to maintain a consumer’s membership of a consumer group. | 6000-300000 |10000 (10 seconds) max.poll.interval.ms | The maximum time interval between polls before the consumer leaves the group. |1,… |300000 (5 minutes) -Many more configuration settings are available, but ensure you read the [Apache Kafka documentation](https://kafka.apache.org/documentation.html){:target="_blank"} thoroughly before experimenting with them. +Many more configuration settings are available, but ensure you read the [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} thoroughly before experimenting with them. ## Consumer groups @@ -131,4 +131,4 @@ To learn more, see the following information: - [Producing messages](../producing-messages) - [Partition leadership](../partition-leadership/) -- [Apache Kafka documentation](https://kafka.apache.org/documentation.html){:target="_blank"} +- [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} diff --git a/_es_11.4/01-about/05-partition-leadership.md b/_es_11.4/01-about/05-partition-leadership.md index 6fcb990a..7916ed62 100644 --- a/_es_11.4/01-about/05-partition-leadership.md +++ b/_es_11.4/01-about/05-partition-leadership.md @@ -18,4 +18,4 @@ To learn more, see the following information: - [Producing messages](../producing-messages) - [Consuming messages](../consuming-messages) -- [Apache Kafka documentation](https://kafka.apache.org/documentation.html){:target="_blank"} +- [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} diff --git a/_es_11.4/02-installing/02-planning.md b/_es_11.4/02-installing/02-planning.md index a69a34bb..68f3c7f0 100644 --- a/_es_11.4/02-installing/02-planning.md +++ b/_es_11.4/02-installing/02-planning.md @@ -186,7 +186,7 @@ Ensure you have sufficient CPU capacity and physical memory in your environment If you plan to have persistent volumes, [consider the disk space](../capacity-planning/#disk-space-for-persistent-volumes) required for storage. -Both Kafka and ZooKeeper rely on fast write access to disks. Use separate dedicated disks for storing Kafka and ZooKeeper data. For more information, see the disks and filesystems guidance in the [Kafka documentation](https://kafka.apache.org/documentation/#diskandfs){:target="_blank"}, and the deployment guidance in the [ZooKeeper documentation](https://zookeeper.apache.org/doc/r3.5.7/zookeeperAdmin.html#sc_designing){:target="_blank"}. +Both Kafka and ZooKeeper rely on fast write access to disks. Use separate dedicated disks for storing Kafka and ZooKeeper data. For more information, see the disks and filesystems guidance in the [Kafka documentation](https://kafka.apache.org/37/documentation/#diskandfs){:target="_blank"}, and the deployment guidance in the [ZooKeeper documentation](https://zookeeper.apache.org/doc/r3.5.7/zookeeperAdmin.html#sc_designing){:target="_blank"}. If persistence is enabled, each Kafka broker and ZooKeeper server requires one physical volume each. The number of Kafka brokers and ZooKeeper servers depends on your setup (for example, see the provided samples described in [resource requirements](../prerequisites/#resource-requirements)). @@ -329,7 +329,7 @@ You can set up {{site.data.reuse.es_name}} to use the following Cruise Control f {{site.data.reuse.es_name}} follows widely adopted logging method for containerized applications and writes to standard output and standard error streams. -You can install any logging solution that integrates with Kubernetes such as [cluster logging](https://docs.openshift.com/container-platform/4.16/logging/cluster-logging.html){:target="_blank"} provided by the {{site.data.reuse.openshift_short}} to collect, store, and visualize logs. +You can install any logging solution that integrates with Kubernetes such as [cluster logging](https://docs.openshift.com/container-platform/4.16/observability/logging/cluster-logging.html){:target="_blank"} provided by the {{site.data.reuse.openshift_short}} to collect, store, and visualize logs. You can use log data to monitor cluster activity and investigate any problems affecting your [system health](../../administering/deployment-health/). diff --git a/_es_11.4/02-installing/04-capacity-planning.md b/_es_11.4/02-installing/04-capacity-planning.md index fe5a3510..747b4397 100644 --- a/_es_11.4/02-installing/04-capacity-planning.md +++ b/_es_11.4/02-installing/04-capacity-planning.md @@ -46,7 +46,7 @@ For each log segment, there are two index files called the time index and the of Log segments can be deleted or compacted, or both, to manage their size. The topic-level configuration property `cleanup.policy` determines the way the log segments for the topic are managed. -For more information about these settings, see [the Kafka documentation](https://kafka.apache.org/documentation/#configuration){:target="_blank"}. +For more information about these settings, see [the Kafka documentation](https://kafka.apache.org/37/documentation/#configuration){:target="_blank"}. The cluster-level settings are [configured](../configuring/#applying-kafka-broker-configuration-settings) in the `EventStreams` custom resource for an instance when it is first created and can be [modified](../../administering/modifying-installation/#modifying-kafka-broker-configuration-settings) at any time. diff --git a/_es_11.4/02-installing/07-configuring.md b/_es_11.4/02-installing/07-configuring.md index a5093082..7873d28b 100644 --- a/_es_11.4/02-installing/07-configuring.md +++ b/_es_11.4/02-installing/07-configuring.md @@ -19,7 +19,7 @@ You can modify the samples, save them, and apply custom configuration settings a On OpenShift, you can configure and apply them by using the [command line](../installing/#installing-an-instance-by-using-the-cli) or by dragging and dropping them onto the {{site.data.reuse.openshift_short}} [web console](../installing/#installing-by-using-the-yaml-view), and editing them. -**Note:** When applying custom Kafka configuration settings to your {{site.data.reuse.es_name}}, check the [Kafka documentation](https://kafka.apache.org/documentation){:target="_blank"} to ensure the new configuration settings are consistent and do not cause conflicts. +**Note:** When applying custom Kafka configuration settings to your {{site.data.reuse.es_name}}, check the [Kafka documentation](https://kafka.apache.org/37/documentation){:target="_blank"} to ensure the new configuration settings are consistent and do not cause conflicts. ## Checking configuration settings @@ -242,7 +242,7 @@ You can choose to change the authentication type of the UI access from IAM to SC ## Applying Kafka broker configuration settings -Kafka supports a number of [broker configuration settings](http://kafka.apache.org/documentation/#brokerconfigs){:target="_blank"}, typically provided in a properties file. +Kafka supports a number of [broker configuration settings](http://kafka.apache.org/37/documentation/#brokerconfigs){:target="_blank"}, typically provided in a properties file. When creating an instance of {{site.data.reuse.es_name}}, these settings are defined in an `EventStreams` custom resource under a the `spec.strimziOverrides.kafka.config` property. @@ -801,7 +801,7 @@ spec: # ... ``` -**Important:** Enabling the Kafka Proxy to gather producer metrics places an intermediary between your producing clients and your Kafka brokers. This adds latency to any traffic to your Kafka brokers. Consider the performance implications of having the proxy in front of your Kafka brokers. You can also leave the proxy disabled and gather producer metrics from the clients directly by using [JMX](https://kafka.apache.org/documentation/#monitoring){:target="_blank"}. +**Important:** Enabling the Kafka Proxy to gather producer metrics places an intermediary between your producing clients and your Kafka brokers. This adds latency to any traffic to your Kafka brokers. Consider the performance implications of having the proxy in front of your Kafka brokers. You can also leave the proxy disabled and gather producer metrics from the clients directly by using [JMX](https://kafka.apache.org/37/documentation/#monitoring){:target="_blank"}. ## Configuring external monitoring through Prometheus diff --git a/_es_11.4/03-getting-started/02-creating-topics.md b/_es_11.4/03-getting-started/02-creating-topics.md index 8492d73d..5332d48b 100644 --- a/_es_11.4/03-getting-started/02-creating-topics.md +++ b/_es_11.4/03-getting-started/02-creating-topics.md @@ -33,7 +33,7 @@ To use Kafka topics to store events in {{site.data.reuse.es_name}}, create and c **Note:** To view all configuration options you can set for topics, set **Show all available options** to **On**. -**Note:** Kafka supports additional [topic configuration](https://kafka.apache.org/documentation/#topicconfigs){:target="_blank"} settings. Enable **Show all available options** to access more detailed configuration settings if required. +**Note:** Kafka supports additional [topic configuration](https://kafka.apache.org/37/documentation/#topicconfigs){:target="_blank"} settings. Enable **Show all available options** to access more detailed configuration settings if required. ## Using the CLI @@ -62,7 +62,7 @@ To use Kafka topics to store events in {{site.data.reuse.es_name}}, create and c kubectl es topic-create --help ``` -Kafka supports additional [topic configuration](https://kafka.apache.org/documentation/#topicconfigs){:target="_blank"} settings. Extend the topic creation command with one or more `--config =` properties to apply additional configuration settings. The following additional properties are currently supported: +Kafka supports additional [topic configuration](https://kafka.apache.org/37/documentation/#topicconfigs){:target="_blank"} settings. Extend the topic creation command with one or more `--config =` properties to apply additional configuration settings. The following additional properties are currently supported: * cleanup.policy * compression.type diff --git a/_es_11.4/04-security/03-secure-jmx-connections.md b/_es_11.4/04-security/03-secure-jmx-connections.md index 17d1aa24..76f196b9 100644 --- a/_es_11.4/04-security/03-secure-jmx-connections.md +++ b/_es_11.4/04-security/03-secure-jmx-connections.md @@ -10,7 +10,7 @@ toc: true Java Management Extensions (JMX) is a way of retrieving metrics from your specific processes dynamically at runtime. This can be used to get metrics that are specific to Java operations on Kafka -To manage resources, management beans (MBeans) are used. MBeans represent a resource in the JVM. There are specific [MBean attributes](https://kafka.apache.org/documentation/#remote_jmx){:target="_blank"} you can use with Kafka. +To manage resources, management beans (MBeans) are used. MBeans represent a resource in the JVM. There are specific [MBean attributes](https://kafka.apache.org/37/documentation/#remote_jmx){:target="_blank"} you can use with Kafka. Metrics can be retrieved from applications running inside your Kubernetes cluster by connecting to an exposed JMX port. diff --git a/_es_11.4/04-security/07-gdpr-considerations.md b/_es_11.4/04-security/07-gdpr-considerations.md index 60cf851f..380d4456 100644 --- a/_es_11.4/04-security/07-gdpr-considerations.md +++ b/_es_11.4/04-security/07-gdpr-considerations.md @@ -107,16 +107,16 @@ users may also wish to consider when ensuring compliance with GDPR. - Kubernetes activity logs for containers running within the Pods that make up the {{site.data.reuse.es_name}} deployment - Logs captured on the local file system for the Kafka container running in the Kakfa pod for each node -By default, messages published to topics are retained for a week after their initial receipt, but this can be configured by modifying [Kafka broker settings](https://kafka.apache.org/documentation/#brokerconfigs){:target="_blank"}. These settings are [configured](../../installing/configuring/#applying-kafka-broker-configuration-settings) using the `EventStreams` custom resource. +By default, messages published to topics are retained for a week after their initial receipt, but this can be configured by modifying [Kafka broker settings](https://kafka.apache.org/37/documentation/#brokerconfigs){:target="_blank"}. These settings are [configured](../../installing/configuring/#applying-kafka-broker-configuration-settings) using the `EventStreams` custom resource. ## Data Access The Kafka core APIs can be used to access message data within the {{site.data.reuse.es_name}} system: -- [Producer](http://kafka.apache.org/documentation/#producerapi){:target="_blank"} API to allow data to be sent to a topic -- [Consumer](http://kafka.apache.org/documentation/#consumerapi){:target="_blank"} API to allow data to be read from a topic -- [Streams](http://kafka.apache.org/documentation/#streamsapi){:target="_blank"} API to allow transformation of data from an input topic to an output topic -- [Connect](http://kafka.apache.org/documentation/#connectapi){:target="_blank"} API to allow connectors to continually move data in or out of a topic from an external system +- [Producer](http://kafka.apache.org/37/documentation/#producerapi){:target="_blank"} API to allow data to be sent to a topic +- [Consumer](http://kafka.apache.org/37/documentation/#consumerapi){:target="_blank"} API to allow data to be read from a topic +- [Streams](http://kafka.apache.org/37/documentation/#streamsapi){:target="_blank"} API to allow transformation of data from an input topic to an output topic +- [Connect](http://kafka.apache.org/37/documentation/#connectapi){:target="_blank"} API to allow connectors to continually move data in or out of a topic from an external system For more information about controlling access to data stored in {{site.data.reuse.es_name}}, see [managing access](../managing-access). diff --git a/_es_11.4/06-connecting/02-connectors.md b/_es_11.4/06-connecting/02-connectors.md index 747ec29b..b4c656c3 100644 --- a/_es_11.4/06-connecting/02-connectors.md +++ b/_es_11.4/06-connecting/02-connectors.md @@ -13,7 +13,7 @@ You can integrate external systems with {{site.data.reuse.es_name}} by using the ## What is Kafka Connect? -When connecting Apache Kafka and other systems, the technology of choice is the [Kafka Connect framework](https://kafka.apache.org/documentation/#connect){:target="_blank"}. +When connecting Apache Kafka and other systems, the technology of choice is the [Kafka Connect framework](https://kafka.apache.org/37/documentation/#connect){:target="_blank"}. Use Kafka Connect to reliably move large amounts of data between your Kafka cluster and external systems. For example, it can ingest data from sources such as databases and make the data available for stream processing. diff --git a/_es_11.4/06-connecting/04-connecting-mq-source.md b/_es_11.4/06-connecting/04-connecting-mq-source.md index 49d20153..58d5d519 100644 --- a/_es_11.4/06-connecting/04-connecting-mq-source.md +++ b/_es_11.4/06-connecting/04-connecting-mq-source.md @@ -270,8 +270,10 @@ IBM MQ source connector v2 offers exactly-once message delivery semantics. An ad Ensure the following values are set in your environment before you enable the exactly-once behavior: -* When the MQ source connector v2 has been configured to deliver messages to Kafka with exactly-once semantics, ensure that the downstream consumers are only consuming transactionally committed messages. You can do this by setting the [isolation.level](https://kafka.apache.org/documentation/#consumerconfigs_isolation.level){:target="_blank"} configuration property to `read_committed`. -* The IBM MQ source connector must run on Kafka Connect version 3.3.0 or later and the `exactly.once.source.support` property must be set to `enabled` in the Kafka Connect worker configuration. For more information about the `exactly.once.source.support` setting, and the Access Control List (ACL) requirements for the worker nodes, see the [Apache Kafka documentation](https://kafka.apache.org/documentation/#connect_exactlyoncesource){:target="_blank"}. + +* When the MQ source connector has been configured to deliver messages to Kafka with exactly-once semantics, ensure that the downstream consumers are only consuming transactionally committed messages. You can do this by setting the [isolation.level](https://kafka.apache.org/37/documentation/#consumerconfigs_isolation.level){:target="_blank"} configuration property to `read_committed`. +* The IBM MQ source connector must run on Kafka Connect version 3.3.0 or later and the `exactly.once.source.support` property must be set to `enabled` in the Kafka Connect worker configuration. For more information about the `exactly.once.source.support` setting, and the Access Control List (ACL) requirements for the worker nodes, see the [Apache Kafka documentation](https://kafka.apache.org/37/documentation/#connect_exactlyoncesource){:target="_blank"}. + * On the server-connection channel (SVRCONN) used for Kafka Connect, set `HBINT` to 30 seconds to allow IBM MQ transaction rollbacks to occur more quickly in failure scenarios. * On the [state queue](#creating-a-state-queue-in-ibm-mq-by-using-the-runmqsc-tool) (the queue where the messages are stored), set `DEFSOPT` to `EXCL` to ensure the state queue share option is exclusive. * Ensure that the messages that are sent through the MQ source connector v2 do not expire and are persistent. @@ -282,11 +284,13 @@ Ensure you configure the following properties to enable exactly-once delivery: * The IBM MQ source connector must have the state queue name configured in the `mq.exactly.once.state.queue` property. The value of the `mq.exactly.once.state.queue` property is the name of a pre-configured IBM MQ queue on the same queue manager as the source IBM MQ queue. -* If you are configuring the [transaction.boundary](https://kafka.apache.org/documentation/#sourceconnectorconfigs_transaction.boundary){:target="_blank"} property, the only permitted property value for the IBM MQ source connector is `poll` (the default value). The `poll` value in the `transaction.boundary` property ensures that the Kafka producer transactions are started and committed for each batch of records that are provided to Kafka by the IBM MQ source connector v2. + +* If you are configuring the [transaction.boundary](https://kafka.apache.org/37/documentation/#sourceconnectorconfigs_transaction.boundary){:target="_blank"} property, the only permitted property value for the IBM MQ source connector is `poll` (the default value). The `poll` value in the `transaction.boundary` property ensures that the Kafka producer transactions are started and committed for each batch of records that are provided to Kafka by the IBM MQ source connector. + * Only a single connector task can run in the Kafka Connect instance. As a consequence, the `tasks.max` property must be set to `1` to ensure that failure scenarios do not cause duplicated messages to be delivered. -* Ensure that the IBM MQ source connector principal has a specific set of ACLs to be able to write transactionally to Kafka. See the [Kafka documentation](https://kafka.apache.org/documentation/#connect_exactlyoncesource){:target="_blank"} for the ACL requirements. +* Ensure that the IBM MQ source connector principal has a specific set of ACLs to be able to write transactionally to Kafka. See the [Kafka documentation](https://kafka.apache.org/37/documentation/#connect_exactlyoncesource){:target="_blank"} for the ACL requirements. * Ensure that the state queue is empty each time exactly-once delivery is enabled (especially when re-enabling the exactly-once feature). Otherwise, the connector behaves in the same way as when recovering from a failure state, and attempts to get undelivered messages recorded in the out-of-date state message. @@ -322,6 +326,6 @@ Create a state queue as follows. The IBM MQ source connector is designed to fail on start-up in certain cases to ensure that exactly-once delivery is not compromised. In some of these failure scenarios, it will be necessary for an IBM MQ administrator to remove messages from the exactly-once state queue before the IBM MQ source connector can start up and deliver messages from the source queue again. In these cases, the IBM MQ source connector will have the `FAILED` status and the Kafka Connect logs will describe any required administrative action. -## Advanced configuration. +## Advanced configuration -For all available configuration options for IBM MQ source connector, see [connecting to IBM MQ](../connecting-mq/#configuration-options). \ No newline at end of file +For all available configuration options for IBM MQ source connector, see [connecting to IBM MQ](../#configuration-options). \ No newline at end of file diff --git a/_es_11.4/06-connecting/05-connecting-mq-sink.md b/_es_11.4/06-connecting/05-connecting-mq-sink.md index 9c1038be..5a2a8c9f 100644 --- a/_es_11.4/06-connecting/05-connecting-mq-sink.md +++ b/_es_11.4/06-connecting/05-connecting-mq-sink.md @@ -287,7 +287,7 @@ Follow the instructions to enable exactly-once delivery in the IBM MQ sink conne Ensure the following values are set in your environment before you enable the exactly-once behavior: -* Configure the consumer group of the sink connector to ignore records in aborted transactions. You can find detailed instructions in the [Kafka documentation](https://kafka.apache.org/documentation/#connect_exactlyoncesink){:target="_blank"}. Notably, this configuration does not have any additional Access Control List (ACL) requirements. +* Configure the consumer group of the sink connector to ignore records in aborted transactions. You can find detailed instructions in the [Kafka documentation](https://kafka.apache.org/37/documentation/#connect_exactlyoncesink){:target="_blank"}. Notably, this configuration does not have any additional Access Control List (ACL) requirements. * The IBM MQ sink connector v2 is only supported on Kafka Connect version 2.6.0 or later. * On the server-connection channel (SVRCONN) used for Kafka Connect, set `HBINT` to 30 seconds to allow IBM MQ transaction rollbacks to occur more quickly in failure scenarios. * On the [state queue](#creating-a-state-queue-in-ibm-mq-by-using-the-runmqsc-tool) (the queue where the state messages are stored), set `DEFSOPT` to `EXCL` to ensure the state queue share option is exclusive. diff --git a/_es_11.4/06-connecting/06-running-connectors-on-zos.md b/_es_11.4/06-connecting/06-running-connectors-on-zos.md index f0fd67ef..a0a29b4c 100644 --- a/_es_11.4/06-connecting/06-running-connectors-on-zos.md +++ b/_es_11.4/06-connecting/06-running-connectors-on-zos.md @@ -249,7 +249,7 @@ cd kafka ./bin/connect-distributed.sh connect-distributed.properties ``` -To start an individual connector use the [Kafka Connect REST API](https://kafka.apache.org/documentation/#connect_rest){:target="_blank"}. For example, given a configuration file `mq-source.json` with the following contents: +To start an individual connector use the [Kafka Connect REST API](https://kafka.apache.org/37/documentation/#connect_rest){:target="_blank"}. For example, given a configuration file `mq-source.json` with the following contents: ```json { diff --git a/_es_11.4/07-administering/01-deployment-health.md b/_es_11.4/07-administering/01-deployment-health.md index de5ed699..c76ed857 100644 --- a/_es_11.4/07-administering/01-deployment-health.md +++ b/_es_11.4/07-administering/01-deployment-health.md @@ -49,7 +49,7 @@ If any of the components are not ready for an extended period of time, check the ![Example pod overview]({{ 'images' | relative_url }}/pod_overview.png "Example screen capture showing pod details with graphs for memory and CPU usage"){:height="100%" width="100%"} 9. Click on the **Logs** tab to search logs. -**Tip:** You can also use the [cluster logging](https://docs.openshift.com/container-platform/4.16/logging/cluster-logging.html){:target="_blank"} provided by the {{site.data.reuse.openshift_short}} to collect, store, and visualize logs. The cluster logging components are based upon Elasticsearch, Fluentd, and Kibana (EFK). You can download and install the pre-configured {{site.data.reuse.es_name}} Kibana dashboards by following the instructions in [monitoring cluster health](../cluster-health/). +**Tip:** You can also use the [cluster logging](https://docs.openshift.com/container-platform/4.16/observability/logging/cluster-logging.html){:target="_blank"} provided by the {{site.data.reuse.openshift_short}} to collect, store, and visualize logs. The cluster logging components are based upon Elasticsearch, Fluentd, and Kibana (EFK). You can download and install the pre-configured {{site.data.reuse.es_name}} Kibana dashboards by following the instructions in [monitoring cluster health](../cluster-health/). ### Using the CLI diff --git a/_es_11.4/07-administering/02-cluster-health.md b/_es_11.4/07-administering/02-cluster-health.md index e9c0781d..f516fda9 100644 --- a/_es_11.4/07-administering/02-cluster-health.md +++ b/_es_11.4/07-administering/02-cluster-health.md @@ -30,16 +30,16 @@ You can use [Grafana](https://grafana.com/docs/grafana/latest/){:target="_blank" ## Kibana -You can use the Kibana service that is provided by the {{site.data.reuse.openshift_short}} [cluster logging](https://docs.openshift.com/container-platform/4.16/logging/cluster-logging.html){:target="_blank"}, and use the example [Kibana dashboards](https://github.com/IBM/ibm-event-automation/tree/master/event-streams/kibana-dashboards){:target="_blank"} to monitor for specific errors in the logs and set up alerts for when a number of errors occur over a period of time in your {{site.data.reuse.es_name}} instance. +You can use the Kibana service that is provided by the {{site.data.reuse.openshift_short}} [cluster logging](https://docs.openshift.com/container-platform/4.16/observability/logging/cluster-logging.html){:target="_blank"}, and use the example [Kibana dashboards](https://github.com/IBM/ibm-event-automation/tree/master/event-streams/kibana-dashboards){:target="_blank"} to monitor for specific errors in the logs and set up alerts for when a number of errors occur over a period of time in your {{site.data.reuse.es_name}} instance. To install the {{site.data.reuse.es_name}} Kibana dashboards, follow these steps: -1. Ensure you have [cluster logging](https://docs.openshift.com/container-platform/4.16/logging/cluster-logging-deploying.html){:target="_blank"} installed. +1. Ensure you have [cluster logging](https://docs.openshift.com/container-platform/4.16/observability/logging/cluster-logging-deploying.html){:target="_blank"} installed. 2. Download the JSON file that includes the example Kibana dashboards for {{site.data.reuse.es_name}} from [GitHub](https://github.com/IBM/ibm-event-automation/tree/master/event-streams/kibana-dashboards){:target="_blank"}. 3. Navigate to the Kibana homepage on your cluster. - {{site.data.reuse.openshift_ui_login}} Then follow the instructions to navigate to [the cluster logging's Kibana homepage](https://docs.openshift.com/container-platform/4.16/logging/log_visualization/logging-kibana.html#cluster-logging-visualizer-kibana_logging-kibana){:target="_blank"}. + {{site.data.reuse.openshift_ui_login}} Then follow the instructions to navigate to [the cluster logging's Kibana homepage](https://docs.openshift.com/container-platform/4.16/observability/logging/log_visualization/logging-kibana.html#cluster-logging-visualizer-kibana_logging-kibana){:target="_blank"}. 4. Click **Management** in the navigation on the left. 5. Click **Index patterns**. 6. Click **Create index pattern**. diff --git a/_es_11.4/07-administering/05a-audit-logging.md b/_es_11.4/07-administering/05a-audit-logging.md index 9d899421..dfa9cf9e 100644 --- a/_es_11.4/07-administering/05a-audit-logging.md +++ b/_es_11.4/07-administering/05a-audit-logging.md @@ -69,7 +69,7 @@ To configure {{site.data.reuse.es_name}} to provide audit trail for the Kafka cl - Define an output destination by setting up an [appender](https://logging.apache.org/log4j/1.2/manual.html#Appenders_and_Layouts){:target="_blank"} to direct filtered audit records to a central system for aggregation, retention, and visualization. Ensure access to the audit trail is secured by restricting access only to authorized users. - - Define a set of filters based on [Kafka API Keys (protocols)](https://kafka.apache.org/documentation/#operations_resources_and_protocols){:target="_blank"} to include or exclude specific requests in the audit trail. + - Define a set of filters based on [Kafka API Keys (protocols)](https://kafka.apache.org/37/documentation/#operations_resources_and_protocols){:target="_blank"} to include or exclude specific requests in the audit trail. For sample log4j configurations, see the [example snippets](#example-log4j-configurations) later. diff --git a/_es_11.4/07-administering/07-modifying-installation.md b/_es_11.4/07-administering/07-modifying-installation.md index 637ff09e..fe6ffbd0 100644 --- a/_es_11.4/07-administering/07-modifying-installation.md +++ b/_es_11.4/07-administering/07-modifying-installation.md @@ -42,7 +42,7 @@ To modify configuration settings by using the Kubernetes command-line tool (`kub ## Modifying Kafka broker configuration settings -Kafka supports a number of [key/value pair settings](http://kafka.apache.org/documentation/#brokerconfigs){:target="_blank"} for broker configuration, typically provided in a properties file. +Kafka supports a number of [key/value pair settings](http://kafka.apache.org/37/documentation/#brokerconfigs){:target="_blank"} for broker configuration, typically provided in a properties file. In {{site.data.reuse.es_name}}, these settings are defined in an `EventStreams` custom resource under the `spec.strimziOverrides.kafka.config` property. diff --git a/_es_11.4/07-administering/10-quotas.md b/_es_11.4/07-administering/10-quotas.md index 1e107b86..e2086310 100644 --- a/_es_11.4/07-administering/10-quotas.md +++ b/_es_11.4/07-administering/10-quotas.md @@ -18,7 +18,7 @@ After a client that has a quota defined reaches the maximum amount of data it ca By default, clients have unlimited quotas. -For more information about quotas, see the [Kafka documentation](https://kafka.apache.org/documentation/#design_quotas){:target="_blank"}. +For more information about quotas, see the [Kafka documentation](https://kafka.apache.org/37/documentation/#design_quotas){:target="_blank"}. ## Setting user quotas @@ -41,7 +41,7 @@ Decide what you want to limit by defining one or more of the following quota typ |:-----------------|:-----------------|:-----------------| | `producerByteRate` | integer | This quota limits the number of bytes that a producer application is allowed to send per second. | | `consumerByteRate` | integer | This quota limits the number of bytes that a consumer application is allowed to receive per second. | -| `requestPercentage` | integer | This quota limits all clients based on [thread utilization](https://kafka.apache.org/documentation/#design_quotascpu){:target="_blank"}. | +| `requestPercentage` | integer | This quota limits all clients based on [thread utilization](https://kafka.apache.org/37/documentation/#design_quotascpu){:target="_blank"}. | **Note:** Quotas can only be applied to individual `KafkaUser` resources. It is advised to apply a quota to an existing `KafkaUser` custom resource, as described in the following sections. You can [create](../../security/managing-access/#creating-a-kafkauser-in-the-event-streams-ui) a `Kafkauser` custom resource beforehand. diff --git a/_es_11.5/00-home.md b/_es_11.5/00-home.md new file mode 100644 index 00000000..0903abfd --- /dev/null +++ b/_es_11.5/00-home.md @@ -0,0 +1,92 @@ +--- +title: "Home" +layout: splash +permalink: /es/ +quickLinks: + - link: + - title: Introduction + - url: about/overview/ + - link: + - title: What's new? + - url: about/whats-new/ + - link: + - title: Key concepts + - url: about/key-concepts/ +gettingStarted: + - mainTitle: How to get going + - link: + - title: Installing on OpenShift Container Platform + - url: installing/installing/ + - icon: install.svg + - link: + - title: Installing on other Kubernetes platforms + - url: installing/installing-on-kubernetes/ + - icon: install.svg + - link: + - title: Using schemas for your data + - url: schemas/overview/ + - icon: clientApps.svg + - link: + - title: Connecting other systems using Kafka Connect + - url: connecting/connectors/ + - icon: migrate.svg + - link: + - title: Getting started with Event Streams as a hosted service + - url: https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-getting-started + - icon: install.svg + - link: + - title: Upgrading + - url: installing/upgrading/ + - icon: upgrade.svg + - link: + - title: Logging In + - url: getting-started/logging-in/ + - icon: login.svg + - link: + - title: Generating a starter application + - url: getting-started/generating-starter-app/ + - icon: generate.svg + - link: + - title: Testing message loads + - url: getting-started/testing-loads/ + - icon: messageLoads.svg + - link: + - title: Creating client apps + - url: getting-started/client/ + - icon: clientApps.svg + +commonTasks: + - mainTitle: Common tasks + - link: + - title: Managing access + - url: security/managing-access/ + - icon: lock.svg + - link: + - title: Monitoring deployment health + - url: administering/deployment-health/ + - icon: monitorHealth.svg + - link: + - title: Monitoring Kafka cluster health + - url: administering/cluster-health/ + - icon: monitorHealth.svg + - link: + - title: Scaling + - url: administering/scaling/ + - icon: scaling.svg + - link: + - title: Setting quotas + - url: administering/quotas/ + - icon: quota.svg + - link: + - title: Replicating across geographies + - url: georeplication/about/ + - icon: geoRep.svg + - link: + - title: Connecting to IBM MQ + - url: connecting/mq/ + - icon: mqFavicon-01.svg + - link: + - title: Producing messages via API + - url: connecting/rest-api/ + - icon: api.svg +--- diff --git a/_es_11.5/01-about/01-overview.md b/_es_11.5/01-about/01-overview.md new file mode 100644 index 00000000..a5d2ca6d --- /dev/null +++ b/_es_11.5/01-about/01-overview.md @@ -0,0 +1,67 @@ +--- +title: "Introduction" +excerpt: "Event Streams is an event-streaming platform based on the open-source Apache Kafka® project." +categories: about +slug: overview +toc: true +--- + + + +{{site.data.reuse.es_name}} is an event-streaming platform based on the [Apache Kafka®](https://kafka.apache.org/){:target="_blank"} project and incorporates the open-source [Strimzi](https://strimzi.io){:target="_blank"} technology. + +{{site.data.reuse.es_name}} uses Strimzi to deploy Apache Kafka in a resilient and manageable way, and provides a range of additional capabilities to extend the core functionality. + +![Event Streams architecture]({{ 'images' | relative_url }}/architectures/ibm-event-automation-event-streams.svg "Diagram showing the Event Streams architecture as part of IBM Event Automation.") + +{{site.data.reuse.es_name}} features include: + +- Apache Kafka deployed by Strimzi that distributes Kafka brokers across the nodes of a Kubernetes cluster, creating a highly-available and resilient deployment. + +- An administrative user interface (UI) focused on providing essential features for managing production clusters, in addition to enabling developers to get started with Apache Kafka. The {{site.data.reuse.es_name}} UI facilitates topic lifecycle control, message filtering and browsing, client metric analysis, schema and geo-replication management, connection details and credentials generation, and a range of tools, including a sample starter and producer application. + +- A command-line interface (CLI) to enable manual and scripted cluster management. For example, you can use the {{site.data.reuse.es_name}} CLI to inspect brokers, manage topics, deploy schemas, and manage geo-replication. + +- A simplified mechanism for replication of topics between clusters ("geo-replication"). + +- A REST API for producing messages to {{site.data.reuse.es_name}} topics, expanding event source possibilities. + +- A schema registry to support the definition and enforcement of message formats between producers and consumers. {{site.data.reuse.es_name}} includes the open-source [Apicurio Registry](https://www.apicur.io/registry/docs/apicurio-registry/2.6.x/index.html){:target="_blank"} for managing schemas. + +- Health check information to help identify issues with your clusters and brokers. + +- Secure by default production cluster templates with authentication, authorization and encryption included, and optional configurations to simplify proof of concept or lite development environments. + +- Granular security configuration through Kafka access control lists to configure authorization and quotas for connecting applications. + + +## Operators + +Kubernetes [Operators](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/){:target="_blank"} are designed to simplify the deployment and maintenance of complex applications. They do this by packaging up and abstracting away domain-specific knowledge of how to deploy, configure and operate the application and its component parts. An Operator can then extend Kubernetes by providing new Kubernetes constructs to support these abstractions, simplifying the initial deployment and subsequent lifecycle management of the application. + +Strimzi uses Operators in this manner to facilitate the deployment of Kafka clusters. {{site.data.reuse.es_name}} builds on top of Strimzi to deploy not only the Kafka components but also the additional features outlined earlier. A new Kubernetes resource `EventStreams` is provided (in Kubernetes referred to as a `kind`), to allow the definition of a complete deployment that brings together the components provided by Strimzi and {{site.data.reuse.es_name}}. This information is defined in a standard YAML document and deployed in a single operation. + + +## Operators provided by {{site.data.reuse.es_name}} + +The following diagram shows the operators involved in an {{site.data.reuse.es_name}} deployment along with the resources they manage. {{site.data.reuse.es_name}} builds on the Strimzi core, and adds additional components to extend the base capabilities. + +![EventStreams Operator diagram.]({{ 'images' | relative_url }}/operator_structure.png "Diagram that shows the operators involved in an {{site.data.reuse.es_name}} deployment.") + +### The EventStreams Cluster Operator + +The Strimzi Cluster Operator manages the deployment of core components within the Kafka cluster such as Kafka and ZooKeeper nodes. + +The EventStreams Cluster Operator extends the Strimzi Cluster Operator to provision the additional components provided by {{site.data.reuse.es_name}} alongside these core components. A new custom resource Type called `EventStreams` is provided to manage this overall deployment. + +### The Entity Operator + +Kafka provides authentication and authorization through Users and Access Control Lists. These define the operations that users can perform on specific resources within the cluster. + +The User operator provides a new custom resource type called `KafkaUser` to manage these Kafka users and associated Access Control Lists and is a mandatory component of an {{site.data.reuse.es_name}} deployment. + +Kafka topics are a resource within the cluster to which a series of records can be produced. + +The Topic operator provides a custom resource type called `KafkaTopic` to manage these Kafka topics. By default, the Topic operator is part of the {{site.data.reuse.es_name}} cluster. To create and manage Kafka topics, use the {{site.data.reuse.es_name}} UI, CLI, and REST API provided mechanisms. + +Kafka topics can be represented as Kubernetes resources as the Topic operator container is included in the {{site.data.reuse.es_name}} cluster within the Entity operator Pod. \ No newline at end of file diff --git a/_es_11.5/01-about/01-whats-new.md b/_es_11.5/01-about/01-whats-new.md new file mode 100644 index 00000000..cbfbe01a --- /dev/null +++ b/_es_11.5/01-about/01-whats-new.md @@ -0,0 +1,31 @@ +--- +title: "What's new" +excerpt: "Find out what is new in Event Streams." +categories: about +slug: whats-new +toc: true +--- + +Find out what is new in {{site.data.reuse.es_name}} version 11.5.x. + + + +## Release 11.5.0 + +### Simplified Kafka topic management with Unidirectional Topic Operator (UTO) + +{{site.data.reuse.es_name}} version 11.5.0 introduces simplified Kafka topic management by using Kubernetes custom resources (CRs) for creating, updating, and deleting Kafka topics. + +### Apicurio version updated to 2.6.2.Final + +{{site.data.reuse.es_name}} 11.5.0 includes support for Apicurio Registry 2.6.2.Final. Ensure all applications connecting to {{site.data.reuse.es_name}} that use the schema registry are using Apicurio client libraries version 2.6.2.Final or later, then [migrate to the latest Apicurio](../../installing/upgrading/#migrate-to-latest-apicurio-registry). + + +### Kafka version upgraded to 3.7.1 + +{{site.data.reuse.es_name}} version 11.5.0 includes Kafka release 3.7.1, and supports the use of all Kafka interfaces. + +### Security and bug fixes + +{{site.data.reuse.es_name}} release 11.5.0 contains security and bug fixes. + diff --git a/_es_11.5/01-about/02-key-concepts.md b/_es_11.5/01-about/02-key-concepts.md new file mode 100644 index 00000000..6678dc18 --- /dev/null +++ b/_es_11.5/01-about/02-key-concepts.md @@ -0,0 +1,49 @@ +--- +title: "Key concepts" +excerpt: "Read about the key concepts of the Apache Kafka technology." +categories: about +slug: key-concepts +toc: true +--- + +Apache Kafka® forms the reliable messaging core of {{site.data.reuse.es_name}}. It is a publish-subscribe messaging system designed to be fault-tolerant, providing a high-throughput and low-latency platform for handling real-time data feeds. + +![Kafka architecture diagram.]({{ 'images' | relative_url }}/kafka_overview.png "Diagram that shows a Kafka architecture. A producer is feeding into a Kafka topic over 3 partitions and the messages are then being subscribed to by consumers.") + +The following are some key Kafka concepts. + +## Cluster +Kafka runs as a cluster of one or more servers (Kafka brokers). The load is balanced across the cluster by distributing it amongst the servers. + +## Topic +A stream of messages is stored in categories called topics. + +## Partition +Each topic comprises one or more partitions. Each partition is an ordered list of messages. The messages on a partition are each given a monotonically increasing number called the offset. + +If a topic has more than one partition, it allows data to be fed through in parallel to increase throughput by distributing the partitions across the cluster. The number of partitions also influences the balancing of workload among consumers. + +## Message +The unit of data in Kafka. Each message is represented as a record, which comprises two parts: key and value. The key is commonly used for data about the message and the value is the body of the message. Kafka uses the terms record and message interchangeably. + +Many other messaging systems also have a way of carrying other information along with the messages. Kafka 0.11 introduced record headers for this purpose. + +Because many tools in the Kafka ecosystem (such as connectors to other systems) use only the value and ignore the key, it’s best to put all of the message data in the value and just use the key for partitioning or log compaction. You should not rely on everything that reads from Kafka to make use of the key. + +## Producer +A process that publishes streams of messages to Kafka topics. A producer can publish to one or more topics and can optionally choose the partition that stores the data. + +## Consumer +A process that consumes messages from Kafka topics and processes the feed of messages. A consumer can consume from one or more topics or partitions. + +## Consumer group +A named group of one or more consumers that together consume the messages from a set of topics. Each consumer in the group reads messages from specific partitions that it is assigned to. Each partition is assigned to one consumer in the group only. + +* If there are more partitions than consumers in a group, some consumers have multiple partitions. +* If there are more consumers than partitions, some consumers have no partitions. + +To learn more, see the following information: +* [Producing messages](../producing-messages) +* [Consuming messages](../consuming-messages) +* [Partition leadership](../partition-leadership/) +* [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} diff --git a/_es_11.5/01-about/03-producing-messages.md b/_es_11.5/01-about/03-producing-messages.md new file mode 100644 index 00000000..ab589b78 --- /dev/null +++ b/_es_11.5/01-about/03-producing-messages.md @@ -0,0 +1,150 @@ +--- +title: "Producing messages" +excerpt: "A producer is an application that publishes streams of messages to Kafka topics." +categories: about +slug: producing-messages +toc: true +--- + +A producer is an application that publishes streams of messages to Kafka topics. This information focuses on the Java programming interface that is part of the Apache Kafka® project. The concepts apply to other languages too, but the names are sometimes a little different. + +In the programming interfaces, a message is actually called a record. For example, the Java class `org.apache.kafka.clients.producer.ProducerRecord` is used to represent a message from the point of view of the producer API. The terms _record_ and _message_ can be used interchangeably, but essentially a record is used to represent a message. + +When a producer connects to Kafka, it makes an initial bootstrap connection. This connection can be to any of the servers in the cluster. The producer requests the partition and leadership information about the topic that it wants to publish to. Then the producer establishes another connection to the partition leader and can begin to publish messages. These actions happen automatically internally when your producer connects to the Kafka cluster. + +When a message is sent to the partition leader, that message is not immediately available to consumers. The leader appends the record for the message to the partition, assigning it the next offset number for that partition. After all the followers for the in-sync replicas have replicated the record and acknowledged that they have written the record to their replicas, the record is now _committed_ and becomes available for consumers. + +Each message is represented as a record which comprises two parts: key and value. The key is commonly used for data about the message and the value is the body of the message. Because many tools in the Kafka ecosystem (such as connectors to other systems) use only the value and ignore the key, it is best to put all of the message data in the value and just use the key for partitioning or log compaction. You should not rely on everything that reads from Kafka to make use of the key. + +Many other messaging systems also have a way of carrying other information along with the messages. Kafka 0.11 introduces record headers for this purpose. + +You might find it useful to read this information in conjunction with [consuming messages](../consuming-messages) in {{site.data.reuse.es_name}}. + +## Configuration settings + +There are many configuration settings for the producer. You can control aspects of the producer including batching, retries, and message acknowledgment. The following table lists the most important ones: + + Name | Description | Valid values | Default +--|---|---|-- +key.serializer | The class used to serialize keys. | Java class that implements Serializer interface, such as org.apache.kafka.common.serialization.StringSerializer |No default – you must specify a value +value.serializer | The class used to serialize values. | Java class that implements Serializer interface, such as org.apache.kafka.common.serialization.StringSerializer |No default – you must specify a value +acks | The number of servers required to acknowledge each message published. This controls the durability guarantees that the producer requires. | 0, 1, all (or -1) |1 +retries | The number of times that the client resends a message when the send encounters an error. |0,… |0 +max.block.ms | The number of milliseconds that a send or metadata request can block waiting. | 0,… | 60000 (1 minute) | +max.in.flight.requests.per.connection | The maximum number of unacknowledged requests that the client sends on a connection before blocking further requests. | 1,… |5 +request.timeout.ms | The maximum amount of time the producer waits for a response to a request. If the response is not received before the timeout elapses, the request is retried or fails if the number of retries has been exhausted. | 0,… |30000 (30 seconds) + +Many more configuration settings are available, but ensure that you read the [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} thoroughly before experimenting with them. + +## Partitioning + +When the producer publishes a message on a topic, the producer can choose which partition to use. If ordering is important, you must remember that a partition is an ordered sequence of records, but a topic comprises one or more partitions. If you want a set of messages to be delivered in order, ensure that they all go on the same partition. The +most straightforward way to achieve this is to give all of those messages the same key. + +The producer can explicitly specify a partition number when it publishes a message. This gives direct control, but it makes the producer code more complex because it takes on the responsibility for managing the partition selection. For more information, see the method call `Producer.partitionsFor`. For example, the call is described for [Kafka 1.10](https://kafka.apache.org/11/javadoc/org/apache/kafka/clients/producer/KafkaProducer.html){:target="_blank"}. + +If the producer does not specify a partition number, the selection of partition is made by a partitioner. The default partitioner that is built into the Kafka producer works as follows: + +- If the record does not have a key, select the partition in a round-robin fashion. +- If the record does have a key, select the partition by calculating a hash value for the key. This has the effect of selecting the same partition for all messages with the same key. + +You can also write your own custom partitioner. A custom partitioner can choose any scheme to assign records to partitions. For example, use just a subset of the information in the key or an application-specific identifier. + +## Message ordering + +Kafka generally writes messages in the order that they are sent by the producer. However, there are situations where retries can cause messages to be duplicated or reordered. If you want a sequence of messages to be sent in order, it is very important to ensure that they are all written to the same partition. + +The producer is also able to retry sending messages automatically. It is often a good idea to enable this retry feature because the alternative is that your application code has to perform any retries itself. The combination of batching in Kafka and automatic retries can have the effect of duplicating messages and reordering them. + +For example, if you publish a sequence of three messages \ on a topic. The records might all fit within the same batch, so they are actually all sent to the partition leader together. The leader then writes them to the partition and replicates them as separate records. In the case of a failure, it is possible that M1 and M2 are added to the partition, but M3 is not. The producer does not receive an acknowledgment, so it retries sending \. The new leader simply writes M1, M2 and M3 onto the partition, which now contains \, where the duplicated M1 actually follows the original M2. If you restrict the number of requests in flight to each broker to just one, you can prevent this reordering. You might still find a single record is duplicated such as \, but you will never get out of order sequences. You can also use the idempotent producer feature to prevent the duplication of M2. + +It is normal practice with Kafka to write the applications to handle occasional message duplicates because the performance impact of having only a single request in flight is significant. + +## Message acknowledgments + +When you publish a message, you can choose the level of acknowledgments required using the `acks` producer configuration. The choice represents a balance between throughput and reliability. There are three levels as follows: + +**acks=0 (least reliable)** + +The message is considered sent as soon as it has been written to the network. There is no acknowledgment from the partition leader. As a result, messages can be lost if the partition leadership changes. This level of acknowledgment is very fast, but comes with the possibility of message loss in some situations. + +**acks=1 (the default)** + +The message is acknowledged to the producer as soon as the partition leader has successfully written its record to the partition. Because the acknowledgment occurs before the record is known to have reached the in-sync replicas, the message could be lost if the leader fails but the followers do not yet have the message. If partition leadership changes, the old leader informs the producer, which can handle the error and retry sending the message to the new leader. Because messages are acknowledged before their receipt has been confirmed by all replicas, messages that have been acknowledged but not yet fully replicated can be lost if the partition leadership changes. + +**acks=all (most reliable)** + +The message is acknowledged to the producer when the partition leader has successfully written its record and all in-sync replicas have done the same. The message is not lost if the partition leadership changes provided that at least one in-sync replica is available. + +Even if you do not wait for messages to be acknowledged to the producer, messages are still only available to be consumed when committed, and that means replication to the in-sync replicas is complete. In other words, the latency of sending the messages from the point of view of the producer is lower than the end-to-end latency measured from the producer sending a message to a consumer receiving the message. + +If possible, avoid waiting for the acknowledgment of a message before publishing the next message. Waiting prevents the producer from being able to batch together messages and also reduces the rate that messages can be published to below the round-trip latency of the network. + +## Batching, throttling, and compression + +For efficiency purposes, the producer actually collects batches of records together for sending to the servers. If you enable compression, the producer compresses each batch, which can improve performance by requiring less data to be transferred over the network. + +If you try to publish messages faster than they can be sent to a server, the producer automatically buffers them up into batched requests. The producer maintains a buffer of unsent records for each partition. Of course, there comes a point when even batching does not allow the required rate to be achieved. + +In summary, when a message is published, its record is first written into a buffer in the producer. In the background, the producer batches up and sends the records to the server. The server then responds to the producer, possibly applying a throttling delay if the producer is publishing too fast. If the buffer in the producer fills up, the producer\'s send call is delayed but ultimately could fail with an exception. + +## Code snippets + +These code snippets are at a very high level to illustrate the concepts involved. + +To connect to {{site.data.reuse.es_name}}, you first need to build the set of configuration properties. All connections to {{site.data.reuse.es_name}} are secured using TLS and user/password authentication, so you need these properties at a minimum. Replace KAFKA\_BROKERS\_SASL, USER, and PASSWORD with your own credentials: + +```shell +Properties props = new Properties(); +props.put("bootstrap.servers", KAFKA_BROKERS_SASL); +props.put("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"USER\" password=\"PASSWORD\";"); +props.put("security.protocol", "SASL_SSL"); +props.put("sasl.mechanism", "PLAIN"); +props.put("ssl.protocol", "TLSv1.3"); +props.put("ssl.enabled.protocols", "TLSv1.3"); +props.put("ssl.endpoint.identification.algorithm", "HTTPS"); +``` + +To send messages, you will also need to specify serializers for the keys and values, for example: + +```shell +props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); +props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); +``` + +Then use a KafkaProducer to send messages, where each message is represented by a ProducerRecord. Do not forget to close the KafkaProducer when you are finished. This code just sends the message but it does not wait to see whether the send succeeded. + +```shell +Producer producer = new KafkaProducer<>(props); +producer.send(new ProducerRecord("T1", "key", "value")); producer.close(); +``` + +The `send()` method is asynchronous and returns a Future that you can use to check its completion: + +```shell +Future f = producer.send(new ProducerRecord("T1", "key", "value")); + +// Do some other stuff + +// Now wait for the result of the send +RecordMetadata rm = f.get(); +long offset = rm.offset; +``` + +Alternatively, you can supply a callback when sending the message: + +```shell +producer.send(new ProducerRecord("T1","key","value", new Callback() { + public void onCompletion(RecordMetadata metadata, Exception exception) { + // This is called when the send completes, either successfully or with an exception + } +}); +``` + +For more information, see the [Javadoc for the Kafka client](https://kafka.apache.org/11/javadoc/){:target="_blank"}, which is very comprehensive. + +To learn more, see the following information: + +- [Consuming messages](../consuming-messages) +- [Partition leadership](../partition-leadership/) +- [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} diff --git a/_es_11.5/01-about/04-consuming-messages.md b/_es_11.5/01-about/04-consuming-messages.md new file mode 100644 index 00000000..3592365a --- /dev/null +++ b/_es_11.5/01-about/04-consuming-messages.md @@ -0,0 +1,134 @@ +--- +title: "Consuming messages" +excerpt: "A consumer is an application that consumes streams of messages from Kafka topics." +categories: about +slug: consuming-messages +toc: true +--- + +A consumer is an application that consumes streams of messages from Kafka topics. A consumer can subscribe to one or more topics or partitions. This information focuses on the Java programming interface that is part of the Apache Kafka project. The concepts apply to other languages too, but the names are sometimes a little different. + +When a consumer connects to Kafka, it makes an initial bootstrap connection. This connection can be to any of the servers in the cluster. The consumer requests the partition and leadership information about the topic that it wants to consume from. Then the consumer establishes another connection to the partition leader and can begin to consume messages. These actions happen automatically internally when your consumer connects to the Kafka cluster. + +A consumer is normally a long-running application. A consumer requests messages from Kafka by calling `Consumer.poll(...)` regularly. The consumer calls `poll()`, receives a batch of messages, processes them promptly, and then calls `poll()` again. + +When a consumer processes a message, the message is not removed from its topic. Instead, consumers can choose from several ways of letting Kafka know which messages have been processed. This process is known as committing the offset. + +In the programming interfaces, a message is actually called a record. For example, the Java class org.apache.kafka.clients.consumer.ConsumerRecord is used to represent a message for the consumer API. The terms _record_ and _message_ can be used interchangeably, but essentially a record is used to represent a message. + +You might find it useful to read this information in conjunction with [producing messages](../producing-messages) in {{site.data.reuse.es_name}}. + +## Configuring consumer properties + +There are many configuration settings for the consumer, which control aspects of its behavior. The following table lists the most important ones: + +Name |Description |Valid values |Default +--|---|---|-- +key.deserializer | The class used to deserialize keys. | Java class that implements Deserializer interface, such as org.apache.kafka.common.serialization.StringDeserializer | No default – you must specify a value +value.deserializer |The class used to deserialize values. | Java class that implements Deserializer interface, such as org.apache.kafka.common.serialization.StringDeserializer | No default – you must specify a value +group.id | An identifier for the consumer group that the consumer belongs to. | string | No default +auto.offset.reset | The behavior when the consumer has no initial offset or the current offset is no longer available in the cluster. | latest, earliest, none |latest +enable.auto.commit | Determines whether to commit the consumer’s offset automatically in the background. | true, false |true +auto.commit.interval.ms | The number of milliseconds between periodic commits of offsets. | 0,… |5000 (5 seconds) +max.poll.records | The maximum number of records returned in a call to poll() | 1,… |500 +session.timeout.ms | The number of milliseconds within which a consumer heartbeat must be received to maintain a consumer’s membership of a consumer group. | 6000-300000 |10000 (10 seconds) +max.poll.interval.ms | The maximum time interval between polls before the consumer leaves the group. |1,… |300000 (5 minutes) + +Many more configuration settings are available, but ensure you read the [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} thoroughly before experimenting with them. + +## Consumer groups + +A _consumer group_ is a group of consumers cooperating to consume messages from one or more topics. The consumers in a group all use the same value for the `group.id` configuration. If you need more than one consumer to handle your workload, you can run multiple consumers in the same consumer group. Even if you only need one consumer, it is usual to also specify a value for `group.id`. + +Each consumer group has a server in the cluster called the _coordinator_ responsible for assigning partitions to the consumers in the group. This responsibility is spread across the servers in the cluster to even the load. The assignment of partitions to consumers can change at every group rebalance. + +When a consumer joins a consumer group, it discovers the coordinator for the group. The consumer then tells the coordinator that it wants to join the group and the coordinator starts a rebalance of the partitions across the group including the new member. + +When one of the following changes take place in a consumer group, the group rebalances by shifting the assignment of partitions to the group members to accommodate the change: + +- a consumer joins the group +- a consumer leaves the group +- a consumer is considered as no longer live by the coordinator +- new partitions are added to an existing topic + +For each consumer group, Kafka remembers the committed offset for each partition being consumed. + +If you have a consumer group that has rebalanced, be aware that any consumer that has left the group will have its commits rejected until it rejoins the group. In this case, the consumer needs to rejoin the group, where it might be assigned a different partition to the one it was previously consuming from. + +## Consumer liveness + +Kafka automatically detects failed consumers so that it can reassign partitions to working consumers. It uses two mechanisms to achieve this: polling and heartbeating. + +If the batch of messages returned from `Consumer.poll(...)` is large or the processing is time-consuming, the delay before calling `poll()` again can be significant or unpredictable. In some cases, it is necessary to configure a long +maximum polling interval so that consumers do not get removed from their groups just because message processing is taking a while. If this were the only mechanism, it would mean that the time taken to detect a failed consumer would also be long. + +To make consumer liveness easier to handle, background heartbeating was added in Kafka 0.10.1. The group coordinator expects group members to send it regular heartbeats to indicate that they remain active. A background heartbeat thread runs in the consumer sending regular heartbeats to the coordinator. If the coordinator does not receive a heartbeat from a group member within the _session timeout_, the coordinator removes the member from the group and starts a rebalance of the group. The session timeout can be much shorter than the maximum polling interval so that the time taken to detect a failed consumer can be short even if message processing takes a long time. + +You can configure the maximum polling interval using the `max.poll.interval.ms` property and the session timeout using the `session.timeout.ms` property. You will typically not need to use these settings unless it takes more than 5 minutes to process a batch of messages. + +### Managing offsets + +For each consumer group, Kafka maintains the committed offset for each partition being consumed. When a consumer processes a message, it does not remove it from the partition. Instead, it just updates its current offset using a process called committing the offset. + +By default, {{site.data.reuse.es_name}} retains committed offset information for 7 days. + +### What if there is no existing committed offset? + +When a consumer starts and is assigned a partition to consume, it will start at its group\'s committed offset. If there is no existing committed offset, the consumer can choose whether to start with the earliest or latest available message based on the setting of the `auto.offset.reset` property as follows: + +* `latest` (the default): Your consumer receives and consumes only messages that arrive after subscribing. Your consumer has no knowledge of messages that were sent before it subscribed, therefore you should not expect that all messages will be consumed from a topic. +* `earliest`: Your consumer consumes all messages from the beginning because it is aware of all messages that have been sent. + +If a consumer fails after processing a message but before committing its offset, the committed offset information will not reflect the processing of the message. This means that the message will be processed again by the next consumer in that group to be assigned the partition. + +When committed offsets are saved in Kafka and the consumers are restarted, consumers resume from the point they last stopped at. When there is a committed offset, the `auto.offset.reset` property is not used. + +### Committing offsets automatically + +The easiest way to commit offsets is to let the Kafka consumer do it automatically. This is simple but it does give less control than committing manually. By default, a consumer automatically commits offsets every 5 seconds. This default commit happens every 5 seconds, regardless of the progress the consumer is making towards processing the messages. In addition, when the consumer calls `poll()`, this also causes the latest offset returned from the previous call to `poll()` to be committed (because it is probably been processed). + +If the committed offset overtakes the processing of the messages and there is a consumer failure, it is possible that some messages might not be processed. This is because processing restarts at the committed offset, which is later than the last message to be processed before the failure. For this reason, if reliability is more important than simplicity, it is usually best to commit offsets manually. + +### Committing offsets manually + +If `enable.auto.commit` is set to `false`, the consumer commits its offsets manually. It can do this either synchronously or asynchronously. A common pattern is to commit the offset of the latest processed message based on a periodic timer. This pattern means that every message is processed at least once, but the committed offset never overtakes the progress of messages that are actively being processed. The frequency of the periodic timer controls the number of messages that can be reprocessed following a consumer failure. Messages are retrieved again from the last saved committed offset when the application restarts or when the group rebalances. + +The committed offset is the offset of the messages from which processing is resumed. This is usually the offset of the most recently processed message _plus one_. + +### Consumer lag + +The consumer lag for a partition is the difference between the offset of the most recently published message and the consumer\'s committed offset. Although it is usual to have natural variations in the produce and consume rates, the consume rate should not be slower than the produce rate for an extended period. + +If you observe that a consumer is processing messages successfully but occasionally appears to jump over a group of messages, it could be a sign that the consumer is not able to keep up. For topics that are not using log compaction, the amount of log space is managed by periodically deleting old log segments. If a consumer has fallen so far behind that it is consuming messages in a log segment that is deleted, it will suddenly jump forwards to the start of the next log segment. If it is important that the consumer processes all of the messages, this behavior indicates message loss from the point of view of this consumer. + +You can use the `kafka-consumer-groups` tool to see the consumer lag. You can also use the consumer API and the consumer metrics for the same purpose. + +## Controlling the speed of message consumption + +If you have problems with message handling caused by message flooding, you can set a consumer option to control the speed of message consumption. Use `fetch.max.bytes` and `max.poll.records` to control how much data a call to `poll()` can return. + +## Handling consumer rebalancing + +When consumers are added to or removed from a group, a group rebalance takes place and consumers are not able to consume messages. This results in all the consumers in a consumer group being unavailable for a short period. + +You could use a ConsumerRebalanceListener to manually commit offsets (if you are not using auto-commit) when notified with the \"on partitions revoked\" callback, and to pause further processing until notified of the successful rebalance using the \"on partition assigned\" callback. + +## Exception handling + +Any robust application that uses the Kafka client needs to handle exceptions for certain expected situations. In some cases, the exceptions are not thrown directly because some methods are asynchronous and deliver their results using a `Future` or a callback. + +Here is a list of exceptions that you should handle in your code: + +**[org.apache.kafka.common.errors.WakeupException]** Thrown by `Consumer.poll(...)` as a result of `Consumer.wakeup()` being called. This is the standard way to interrupt the consumer\'s polling loop. The polling loop should exit and `Consumer.close()` should be called to disconnect cleanly. + +**[org.apache.kafka.common.errors.NotLeaderForPartitionException]** Thrown as a result of `Producer.send(...)` when the leadership for a partition changes. The client automatically refreshes its metadata to find the up-to-date leader information. Retry the operation, which should succeed with the updated metadata. + +**[org.apache.kafka.common.errors.CommitFailedException]** Thrown as a result of `Consumer.commitSync(...)` when an unrecoverable error occurs. In some cases, it is not possible simply to repeat the operation because the partition assignment might have changed and the consumer might no longer be able to commit its offsets. Because `Consumer.commitSync(...)` can be partially successful when used with multiple partitions in a single call, the error recovery can be simplified by using a separate `Consumer.commitSync(...)` call for each partition. + +**[org.apache.kafka.common.errors.TimeoutException]** Thrown by `Producer.send(...), Consumer.listTopics()` if the metadata cannot be retrieved. The exception is also seen in the send callback (or the returned Future) when the requested acknowledgment does not come back within `request.timeout.ms`. The client can retry the operation, but the effect of a repeated operation depends on the specific operation. For example, if sending a message is retried, the message might be duplicated. + +To learn more, see the following information: + +- [Producing messages](../producing-messages) +- [Partition leadership](../partition-leadership/) +- [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} diff --git a/_es_11.5/01-about/05-partition-leadership.md b/_es_11.5/01-about/05-partition-leadership.md new file mode 100644 index 00000000..7916ed62 --- /dev/null +++ b/_es_11.5/01-about/05-partition-leadership.md @@ -0,0 +1,21 @@ +--- +title: "Partition leadership" +excerpt: "Each partition has one server in the cluster that acts as the partition's leader and other servers that act as the followers." +categories: about +slug: partition-leadership +toc: false +--- + +Each partition has one server in the cluster that acts as the partition's leader and other servers that act as the followers. All produce and consume requests for the partition are handled by the leader. The followers replicate the partition data from the leader with the aim of keeping up with the leader. If a follower is keeping up with the leader of a partition, the follower\'s replica is in-sync. + +When a message is sent to the partition leader, that message is not immediately available to consumers. The leader appends the record for the message to the partition, assigning it the next offset number for that partition. After all the followers for the in-sync replicas have replicated the record and acknowledged that they have written the record to their replicas, the record is now *committed*. The message is available for consumers. + +If the leader for a partition fails, one of the followers with an in-sync replica automatically takes over as the partition\'s leader. In practice, every server is the leader for some partitions and the follower for others. The leadership of partitions is dynamic and changes as servers come and go. + +Applications do not need to take specific actions to handle the change in the leadership of a partition. The Kafka client library automatically reconnects to the new leader, although you will see increased latency while the cluster settles. + +To learn more, see the following information: + +- [Producing messages](../producing-messages) +- [Consuming messages](../consuming-messages) +- [Apache Kafka documentation](https://kafka.apache.org/37/documentation.html){:target="_blank"} diff --git a/_es_11.5/02-installing/00-trying-out.md b/_es_11.5/02-installing/00-trying-out.md new file mode 100644 index 00000000..e4d112dd --- /dev/null +++ b/_es_11.5/02-installing/00-trying-out.md @@ -0,0 +1,20 @@ +--- +title: "Trying out Event Streams" +excerpt: "Install a basic deployment to try out Event Streams." +categories: installing +slug: trying-out +toc: true +--- + +To try out {{site.data.reuse.es_name}}, you have the following options: + +- Create a subscription for a fully managed Kafka service on [IBM Cloud](https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-getting-started){:target="_blank"}. +- [Install {{site.data.reuse.es_name}}]({{ 'installpagedivert' | relative_url }}) on a Kubernetes cluster for development purposes or for production use, and benefit from both the support of IBM and the open-source community. + + {{site.data.reuse.es_name}} comes with a host of useful features such as a user interface (UI) to help you get started with Kafka and help operate a production cluster, geo-replication of topics between clusters, a schema registry to enforce correct and consistent message formats, a connector catalog, and more. + + + +- Use [Strimzi](https://strimzi.io){:target="_blank"} if you want to install your own basic Kafka cluster on Kubernetes for testing and proof-of-concept purposes. + + As {{site.data.reuse.es_name}} is based on Strimzi, you can easily move your deployment to {{site.data.reuse.es_name}} later, and keep your existing configurations and preferences from the Strimzi setup. Moving to {{site.data.reuse.es_name}} adds the benefit of full enterprise-level support from IBM. diff --git a/_es_11.5/02-installing/01-prereqs.md b/_es_11.5/02-installing/01-prereqs.md new file mode 100644 index 00000000..45c28b76 --- /dev/null +++ b/_es_11.5/02-installing/01-prereqs.md @@ -0,0 +1,221 @@ +--- +title: "Prerequisites" +excerpt: "Prerequisites for installing Event Streams." +categories: installing +slug: prerequisites +toc: true +--- + + + +Ensure your environment meets the following prerequisites before installing {{site.data.reuse.es_name}}. + +## Container environment + +{{site.data.reuse.es_name}} 11.5.x is supported on the {{site.data.reuse.openshift}} and other Kubernetes platforms that support the Red Hat Universal Base Images (UBI) containers. + +If you are using {{site.data.reuse.openshift}}, ensure you have the following set up for your environment: + +- A supported version of {{site.data.reuse.openshift_short}} on a supported system [installed](https://docs.openshift.com/container-platform/4.16/welcome/index.html){:target="_blank"}. For supported container platform versions and systems, see the [support matrix]({{ 'support/matrix/#event-streams' | relative_url }}). +- The {{site.data.reuse.openshift_short}} CLI (`oc`) [installed](https://docs.openshift.com/container-platform/4.16/cli_reference/openshift_cli/getting-started-cli.html){:target="_blank"}. +- As an option, you can [install](#prereqs-fs) a supported version of the {{site.data.reuse.icpfs}} to use the components offered by {{site.data.reuse.fs}}. + + +If you are using other Kubernetes platforms, ensure you have the following set up for your environment: + +- A supported version of a Kubernetes platform on a supported system installed. For supported container platform versions and systems, see the [support matrix]({{ 'support/matrix/#event-streams' | relative_url }}). +- The Kubernetes command-line tool (`kubectl`) [installed](https://kubernetes.io/docs/tasks/tools/){:target="_blank"}. + +**Note:** For completing tasks by using the command line, you can use both `kubectl` and `oc` commands if your deployment is on the {{site.data.reuse.openshift_short}}. This documentation set includes instructions that use the `kubectl` command, except for cases where the task is specific to OpenShift. + + +## Hardware requirements + +Ensure your hardware can accommodate the [resource requirements](#resource-requirements) for your planned deployment. + +Kubernetes manages the allocation of containers within your cluster. This allows resources to be available for other {{site.data.reuse.es_name}} components which might be required to reside on the same node. + +For production systems, it is recommended to have {{site.data.reuse.es_name}} configured with at least 3 Kafka brokers, and to have one worker node for each Kafka broker. This requires a minimum of 3 worker nodes available for use by {{site.data.reuse.es_name}}. Ensure each worker node runs on a separate physical server. See the guidance about [Kafka high availability](../planning/#kafka-high-availability) for more information. + +## Resource requirements + +{{site.data.reuse.es_name}} resource requirements depend on several factors. The following sections provide guidance about minimum requirements for a starter deployment, and options for initial production configurations. + +Installing {{site.data.reuse.es_name}} has two phases: + +1. Install the {{site.data.reuse.es_name}} operator. The operator will then be available to install and manage your {{site.data.reuse.es_name}} instances. +2. Install one or more instances of {{site.data.reuse.es_name}} by applying configured custom resources. Sample configurations for development and production use cases are provided to get you started. + +Minimum resource requirements are as follows, and are based on the total of requests set for the deployment. You will require more resources to accommodate the limit settings (see more about "requests" and "limits" later in this section). +Always ensure you have sufficient resources in your environment to deploy the {{site.data.reuse.es_name}} operator together with a development or a production {{site.data.reuse.es_name}} instance. + +| Deployment | CPU (cores) | Memory (GiB) | Chargeable cores (see [licensing]({{ '/support/licensing/#calculating-licenses-required' | relative_url }})) | +| --------------------------------------------------- | ----------- | ----------- | ---- | +| [Operator](#operator-requirements) | 0.2 | 1.0 | N/A | +| [Development](../planning/#sample-deployments) | 2.4 | 5.4 | 1 | +| [Production](../planning/#sample-deployments) | 2.8 | 5.9 | 3 | + +**Note:** {{site.data.reuse.es_name}} provides sample configurations to help you get started with deployments. The resource requirements for these specific samples are detailed in the [planning](../planning/#sample-deployments) section. If you do not have an {{site.data.reuse.es_name}} installation on your system yet, always ensure you include the resource requirements for the operator together with the intended {{site.data.reuse.es_name}} instance requirements (development or production). + +{{site.data.reuse.es_name}} is a [Kubernetes operator-based](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/){:target="_blank"} release and uses custom resources to define your {{site.data.reuse.es_name}} configurations. +The {{site.data.reuse.es_name}} operator uses the declared required state of your {{site.data.reuse.es_name}} in the custom resources to deploy and manage the entire lifecycle of your {{site.data.reuse.es_name}} instances. Custom resources are presented as YAML configuration documents that define instances of the `EventStreams` custom resource type. + +The provided samples define typical configuration settings for your {{site.data.reuse.es_name}} instance, including broker configurations, security settings, and default values for resources such as CPU and memory defined as "request" and "limit" settings. [Requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/){:target="_blank"} are Kubernetes concepts for controlling resource types such as CPU and memory. + +- Requests set the minimum requirements a container requires to be scheduled. If your system does not have the required request value, then the services will not start up. +- Limits set the value beyond which a container cannot consume the resource. It is the upper limit within your system for the service. Containers that exceed a CPU resource limit are throttled, and containers that exceed a memory resource limit are terminated by the system. + +Ensure you have sufficient CPU capacity and physical memory in your environment to service these requirements. Your {{site.data.reuse.es_name}} instance can be dynamically updated later through the configuration options provided in the custom resource. + +### Operator requirements + +The {{site.data.reuse.es_name}} operator requires the following minimum resource requirements. Ensure you always include sufficient CPU capacity and physical memory in your environment to service the operator requirements. + +| CPU request (cores) | CPU limit (cores) | Memory request (GiB) | Memory limit (GiB) | +| ------------------- | ----------------- | ------------------- | ----------------- | +| 0.2 | 1.0 | 1.0 | 1.0 | + + +#### Cluster-scoped permissions required + +The {{site.data.reuse.es_name}} operator requires the following cluster-scoped permissions: + +- **Permission to list nodes in the cluster**: When the {{site.data.reuse.es_name}} operator is deploying a Kafka cluster that spans [multiple availability zones](../planning/#multiple-availability-zones), it needs to label the pods with zone information. The permission to list nodes in the cluster is required to retrieve the information for these labels. +- **Permission to manage admission webhooks**: The {{site.data.reuse.es_name}} operator uses admission webhooks to provide immediate validation and feedback about the creation and modification of {{site.data.reuse.es_name}} instances. The permission to manage webhooks is required for the operator to register these actions. +- **Permission to list specific CustomResourceDefinitions**: This allows {{site.data.reuse.es_name}} to identify whether other optional dependencies have been installed into the cluster. + +In addition to the previous permissions, the {{site.data.reuse.es_name}} operator requires the following cluster-scoped permissions on {{site.data.reuse.openshift}}: + +- **Permission to manage ConsoleYAMLSamples**: ConsoleYAMLSamples are used to provide samples for {{site.data.reuse.es_name}} resources in the {{site.data.reuse.openshift_short}} web console. The permission to manage ConsoleYAMLSamples is required for the operator to register the setting up of samples. +- **Permission to list ClusterRoles and ClusterRoleBindings**: The {{site.data.reuse.es_name}} operator uses ClusterRoles created by the Operator Lifecycle Manager (OLM) as parents for supporting resources that the {{site.data.reuse.es_name}} operator creates. This is needed so that the supporting resources are correctly cleaned up when {{site.data.reuse.es_name}} is uninstalled. The permission to list ClusterRoles is required to allow the operator to identify the appropriate cluster role to use for this purpose. +- **Permission to view ConfigMaps (only on OpenShift with {{site.data.reuse.icpfs}})**: When {{site.data.reuse.es_name}} uses authentication services from {{site.data.reuse.fs}}, the status of these services is maintained in ConfigMaps, so the permission to view the contents of the ConfigMaps allows {{site.data.reuse.es_name}} to monitor the availability of the {{site.data.reuse.fs}} dependencies. + +### Adding {{site.data.reuse.es_name}} geo-replication to a deployment + +The {{site.data.reuse.es_name}} [geo-replicator](../../georeplication/about/) allows messages sent to a topic on one {{site.data.reuse.es_name}} cluster to be automatically replicated to another {{site.data.reuse.es_name}} cluster. + +To use this feature, ensure you have the following additional resources available. The following table shows the prerequisites for each geo-replicator node. + +| CPU request (cores) | CPU limit (cores) | Memory request (GiB) | Memory limit (GiB) | Chargeable cores (see [licensing]({{ '/support/licensing/#calculating-licenses-required' | relative_url }})) | +| ------------------- | ----------------- | ------------------- | ----------------- | ---- | +| 1.0 | 2.0 | 2.0 | 2.0 | 1 | + +For instructions about installing geo-replication, see [configuring](../configuring/#setting-geo-replication-nodes). + + +## Red Hat OpenShift Security Context Constraints + +If used, {{site.data.reuse.es_name}} requires a [Security Context Constraint (SCC)](https://docs.openshift.com/container-platform/4.16/authentication/managing-security-context-constraints.html){:target="_blank"} to be bound to the target namespace prior to installation. + +By default, {{site.data.reuse.es_name}} complies with `restricted` or `restricted-v2` SCC depending on your {{site.data.reuse.openshift_short}} version. + +If you use a custom SCC (for example, one that is more restrictive), or have an operator that updates the default SCC, the changes might interfere with the functioning of your {{site.data.reuse.es_name}} deployment. To resolve any issues, apply the SCC provided by {{site.data.reuse.es_name}} as described in [troubleshooting](../../troubleshooting/default-scc-issues). + +## Kubernetes Pod Security Standards + +If used, {{site.data.reuse.es_name}} requires a Pod Security Standard (PSS) to be bound to the target namespace prior to installation. + +By default, {{site.data.reuse.es_name}} complies with the following: + +- Kubernetes 1.24 and earlier complies with the `ibm-restricted-psp` [Pod Security Policy](https://github.com/IBM/cloud-pak/blob/master/spec/security/psp/ibm-restricted-psp.yaml){:target="_blank"}. +- Kubernetes 1.25 and later complies with the built-in `restricted` [Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/){:target="_blank"}. + +## Network requirements + +{{site.data.reuse.es_name}} is supported for use with IPv4 networks only. + +### Ingress controllers + +When you want to expose {{site.data.reuse.es_name}} services externally outside your cluster, you can use OpenShift routes or ingress on other Kubernetes platforms. + +When using ingress, ensure you install and run an [ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/){:target="_blank"} on your Kubernetes platform. The SSL passthrough must be enabled in the ingress controller for your {{site.data.reuse.es_name}} services to work. Refer to your ingress controller documentation for more information. + +## Data storage requirements + +If you want to set up [persistent storage](../planning/#planning-for-persistent-storage), {{site.data.reuse.es_name}} requires block storage configured to use the XFS or ext4 file system. The use of file storage (for example, NFS) is not recommended. + +For example, you can use one of the following systems: + +- Red Hat OpenShift Data Foundation (previously OpenShift Container Storage) version 4.2 or later (block storage only) +- IBM Cloud Block storage +- IBM Storage Suite for IBM Cloud Paks: block storage from IBM Spectrum Virtualize, FlashSystem, or DS8K +- Portworx Storage version 2.5.5 or later +- Kubernetes local volumes +- Amazon Elastic Block Store (EBS) +- Rook Ceph + +## Schema requirements + +The Apicurio Registry is used in {{site.data.reuse.es_name}} to store message schemas. Each {{site.data.reuse.es_name}} cluster has its own instance of Apicurio Registry providing schema registry functionality. + +To connect your applications to use the [Apicurio Registry](../../schemas/overview#schema-registry), ensure all applications connecting to your instance of {{site.data.reuse.es_name}} that use the schema registry are using Apicurio client libraries version 2.6.2 or later. + +## {{site.data.reuse.es_name}} UI + +The {{site.data.reuse.es_name}} user interface (UI) is supported on the following web browsers: + +- Google Chrome version 65 or later +- Mozilla Firefox version 59 or later +- Safari version 11.1 or later + +## {{site.data.reuse.es_name}} CLI + +The {{site.data.reuse.es_name}} command-line interface (CLI) is supported on the following systems: + +- Windows 10 or later +- Linux® Ubuntu 16.04 or later +- macOS 10.13 (High Sierra) or later + +See the post-installation tasks for information about [installing the CLI](../post-installation/#installing-the-event-streams-command-line-interface). + +## Kafka clients + +The Apache Kafka Java client included with {{site.data.reuse.es_name}} is supported for use with the following Java versions: + +- IBM Java 8 or later +- Oracle Java 8 or later + +You can also use other Kafka version 2.0 or later clients when connecting to {{site.data.reuse.es_name}}. If you encounter client-side issues, IBM can assist you to resolve those issues (see our [support policy]({{ 'support/#support-policy' | relative_url }}){:target="_blank"}). + +{{site.data.reuse.es_name}} is designed for use with clients based on the `librdkafka` [implementation](https://github.com/edenhill/librdkafka) of the Apache Kafka protocol. + +## Optional: {{site.data.reuse.icpfs}} for OpenShift +{: #prereqs-fs} + +If you are installing on the {{site.data.reuse.openshift_short}}, you can access additional features with {{site.data.reuse.icpfs}}. To enable the additional features, install a supported version of {{site.data.reuse.fs}} before installing {{site.data.reuse.es_name}}, as described in the [{{site.data.reuse.fs}} documentation](https://www.ibm.com/docs/en/cloud-paks/foundational-services/4.3?topic=about){:target="_blank"}. {{site.data.reuse.es_name}} supports {{site.data.reuse.fs}} version 3.19.0 or later 3.x releases, and 4.x releases. + +{{site.data.reuse.es_name}} supports both the Continuous Delivery (CD) and the Long Term Service Release (LTSR) version of {{site.data.reuse.fs}} (for more information, see [release types](https://www.ibm.com/docs/en/cloud-paks/foundational-services/4.3?topic=about-release-types){:target="_blank"}). This provides more flexibility for compatibility with other Cloud Pak components (for more information, see [deploying with other Cloud Paks on the same cluster](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=requirements-deploying-multiple-cloud-paks-same-cluster){:target="_blank"}). + + + +By default, the `starterset` profile is requested for new installations. If you are preparing for a production deployment, ensure you set a more suitable profile, for example, the [medium profile](https://www.ibm.com/docs/en/cloud-paks/foundational-services/4.3?topic=services-hardware-requirements-medium-profile){:target="_blank"} as described in [setting the hardware profile](https://www.ibm.com/docs/en/cloud-paks/foundational-services/4.3?topic=online-installing-foundational-services-by-using-console#profile){:target="_blank"}. + +**Note:** If you are installing {{site.data.reuse.es_name}} in an existing {{site.data.reuse.cp4i}} deployment, the required {{site.data.reuse.fs}} might already be installed due to other capabilities, and the dependencies required by {{site.data.reuse.es_name}} might already be satisfied with a profile other than the default `starterset`. + +If you plan to install other {{site.data.reuse.cp4i}} capabilities, ensure you meet the [resource requirements](https://www.ibm.com/docs/en/cloud-paks/foundational-services/4.3?topic=installer-hardware-requirements-recommendations-foundational-services){:target="_blank"} for the whole profile. If you only want to deploy {{site.data.reuse.es_name}} on the cluster, you can calculate more granular sizing requirements based on the {{site.data.reuse.fs}} components that you want to use for your {{site.data.reuse.es_name}} installation. + + +For more information about {{site.data.reuse.fs}}, see the [{{site.data.reuse.icpfs}} documentation](https://www.ibm.com/docs/en/cloud-paks/foundational-services/4.3){:target="_blank"}. + +{{site.data.reuse.iam_note}} + + +## Optional: Authenticate {{site.data.reuse.es_name}} with Keycloak +{: #prereqs-keycloak} + +If you are installing on the {{site.data.reuse.openshift_short}}, you can configure access for your integration capabilities such as {{site.data.reuse.es_name}} by using [Keycloak](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=administering-identity-access-management){:target="_blank"}. + +Keycloak is supported on {{site.data.reuse.es_name}} 11.3.0 and later when an {{site.data.reuse.cp4i}} version 2023.4.1 (operator 7.2.0) or later is available. See the [{{site.data.reuse.cp4i}} documentation](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=installing){:target="_blank"} for information about installing {{site.data.reuse.cp4i}}. + +For more information, see sections about [configuring UI security](../configuring/#configuring-ui-and-cli-security) and [managing access with Keycloak](../../security/managing-access/#managing-access-to-the-ui-with-keycloak). + +### Limitations + +Configuring your access with Keycloak in {{site.data.reuse.es_name}} has the following limitations: + +- You cannot access the dashboards that are included in the {{site.data.reuse.es_name}} UI for monitoring [Kafka health](../../administering/cluster-health/#viewing-the-preconfigured-dashboard) and [topic health](../../administering/topic-health/). +- The {{site.data.reuse.es_name}} [CLI](../post-installation/#installing-the-event-streams-command-line-interface) is not supported. + + **Note:** You can authenticate the CLI by using a different mechanism (for example, SCRAM) even if you have other authentication mechanisms configured for your {{site.data.reuse.es_name}} instance (for example, Keycloak). + +- Authentication with Keycloak is not supported for [REST endpoints](../configuring/#rest-services-access) (REST Producer, Admin API, Apicurio Registry). diff --git a/_es_11.5/02-installing/02-planning.md b/_es_11.5/02-installing/02-planning.md new file mode 100644 index 00000000..cf5aa49d --- /dev/null +++ b/_es_11.5/02-installing/02-planning.md @@ -0,0 +1,368 @@ +--- +title: "Planning your installation" +excerpt: "Planning your installation of Event Streams." +categories: installing +slug: planning +toc: true +--- + +Consider the following when planning your installation of {{site.data.reuse.es_name}}. + +Decide the purpose of your deployment, for example, whether you want to try a starter deployment for testing purposes, or start setting up a production deployment. + +- Use the [sample deployments](#sample-deployments) as a starting point if you need something to base your deployment on. +- Size your planned deployment by considering potential throughput, the number of producers and consumers, Kafka performance tuning, and other aspects. For more details, see the [performance considerations](../capacity-planning) section. +- For production use, and whenever you want your data to be saved in the event of a restart, set up [persistent storage](#planning-for-persistent-storage). +- Consider the options for [securing](#planning-for-security) your deployment. +- Plan for [resilience](#planning-for-resilience) by understanding Kafka high availability and how to support it, set up multiple availability zones for added resilience, and consider topic mirroring to help with your [disaster recovery](../../installing/disaster-recovery) planning. +- Consider setting up [logging](#planning-for-log-management) for your deployment to help troubleshoot any potential issues. + +## Sample deployments + +A number of sample configurations are provided when [installing]({{ 'installpagedivert' | relative_url }}) {{site.data.reuse.es_name}} on which you can base your deployment. These range from smaller deployments for non-production development or general experimentation to large scale clusters ready to handle a production workload. + +- [Lightweight without security](#example-deployment-lightweight-without-security) +- [Development](#example-deployment-development) +- [Minimal production](#example-deployment-minimal-production) +- [Production 3 brokers](#example-deployment-production-3-brokers) +- [Production 6 brokers](#example-deployment-production-6-brokers) +- [Production 9 brokers](#example-deployment-production-9-brokers) + +If you are [installing](../installing/#installing-an-instance-by-using-the-web-console) on the {{site.data.reuse.openshift_short}}, you can view and apply the sample configurations in the web console. The sample configurations are also available in [GitHub](https://ibm.biz/ea-es-samples){:target="_blank"}, where you can select the GitHub tag for your {{site.data.reuse.es_name}} version to access the correct samples, and then go to `/cr-examples/eventstreams/openshift` to access the OpenShift samples. + +For other Kubernetes platforms, the custom resource samples are included in the Helm chart package. The sample configurations are also available in [GitHub](https://ibm.biz/ea-es-samples){:target="_blank"}, where you can select the GitHub tag for your {{site.data.reuse.es_name}} version to access the correct samples, and then go to `/cr-examples/eventstreams/kubernetes` to access the samples for other Kubernetes platforms. + +**Important:** For a production setup, the sample configuration values are for guidance only, and you might need to change them. Ensure you set your resource values as required to cope with the intended usage, and also consider important configuration options for your environment and {{site.data.reuse.es_name}} requirements as described in the rest of this planning section. + + +#### Example deployment: **Lightweight without security** + +Overview: A non-production deployment suitable for basic development and test activities, with access to Kafka brokers only from within the same cluster (only internal listener configured). For environments where minimum resource requirements, persistent storage, access control and encryption are not required. + +This example provides a starter deployment that can be used if you simply want to try {{site.data.reuse.es_name}} with a minimum resource footprint. It installs an {{site.data.reuse.es_name}} instance with the following characteristics: +- A small single broker Kafka cluster and a single node ZooKeeper. +- As there is only 1 broker, no message replication takes place between brokers, and the system topics (message offset and transaction state) are configured accordingly for this. +- There is no encryption internally between containers. +- External connections use TLS encryption (for example, to the {{site.data.reuse.es_name}} UI), but no authentication to keep the configuration to a minimum, making it easy to experiment with the platform. +- No external connections are configured for Kafka brokers. Only internal connections within the cluster can be used for producing and consuming messages. + + +Resource requirements for this deployment: + +| CPU request (cores) | CPU limit (cores) | Memory request (GiB) | Memory limit (GiB) | Chargeable cores (see [licensing]({{ '/support/licensing/#calculating-licenses-required' | relative_url }}))| +| ------------------- | ----------------- | ------------------- | ----------------- | ---- | +| 2.4 | 8.2 | 5.4 | 8.2 | 1 | + +Ensure you have sufficient CPU capacity and physical memory in your environment to service at least the resource **request** values. The resource **limit** values constrain the amount of resource the {{site.data.reuse.es_name}} instance is able to consume. + +{{site.data.reuse.sample_select_note}} + +**Important:** This deployment is not suitable for a production system even if storage configuration is applied. This is due to the number of Kafka and ZooKeeper nodes not being appropriate for data persistence and high availability. For a production system, at least three Kafka brokers and ZooKeeper nodes are required for an instance, see [production](#sample-deployments) sample deployments later for alternatives. + +In addition, this deployment installs a single ZooKeeper node with ephemeral storage. If the ZooKeeper pod is restarted, either during normal operation or as part of an upgrade, all messages and all topics will be lost and both ZooKeeper and Kafka pods will move to an error state. To recover the cluster, restart the Kafka pod by deleting it. + +#### Example deployment: **Development** + +Overview: A non-production deployment for experimenting with {{site.data.reuse.es_name}} configured for high availability, authentication, and no persistent storage. Suitable for basic development and testing activities. + +This example provides a starter deployment that can be used if you want to try {{site.data.reuse.es_name}} with a minimum resource footprint. It installs an {{site.data.reuse.es_name}} instance with the following settings: +- 3 Kafka brokers and 3 ZooKeeper nodes. +- Internally, Transport Layer Security (TLS) encryption is used between containers. +- External connections use TLS encryption and Salted Challenge Response Authentication Mechanism (SCRAM-SHA-512) authentication. + +Resource requirements for this deployment: + +| CPU request (cores) | CPU limit (cores) | Memory request (GiB) | Memory limit (GiB) | Chargeable cores (see [licensing]({{ '/support/licensing/#calculating-licenses-required' | relative_url }}))| +| ------------------- | ----------------- | ------------------- | ----------------- | ---- | +| 2.8 | 12.2 | 5.9 | 14.2 | 3 | + +Ensure you have sufficient CPU capacity and physical memory in your environment to service at least the resource **request** values. The resource **limit** values constrain the amount of resource the {{site.data.reuse.es_name}} instance is able to consume. + +{{site.data.reuse.sample_select_note}} + +**Note:** If you want to authenticate with SCRAM for this **Development** deployment, see the **Development SCRAM** sample. The **Development SCRAM** sample has the same settings and resource requirements as the Development sample, but does not require {{site.data.reuse.icpfs}} to be installed and specifies [SCRAM authentication](../../security/managing-access/#managing-access-to-the-ui-and-cli-with-scram) to access the {{site.data.reuse.es_name}} UI and CLI. + +#### Example deployment: **Minimal production** + +Overview: A minimal production deployment for {{site.data.reuse.es_name}}. + +This example provides the smallest possible production deployment that can be configured for {{site.data.reuse.es_name}}. It installs an {{site.data.reuse.es_name}} instance with the following settings: +- 3 Kafka brokers and 3 ZooKeeper nodes. +- Internally, TLS encryption is used between containers. +- External connections use TLS encryption and SCRAM-SHA-512 authentication. +- Kafka tuning settings consistent with 3 brokers are applied as follows: + + ```shell + num.replica.fetchers: 3 + num.io.threads: 24 + num.network.threads: 9 + log.cleaner.threads: 6 + ``` + +If a storage solution has been configured, the following characteristics make this a production-ready deployment: + +- Messages are replicated between brokers to ensure that no single broker is a point of failure. If a broker restarts, producers and consumers of messages will not experience any loss of service. +- The number of threads made available for replicating messages between brokers, is increased to 3 from the default 1. This helps to prevent bottlenecks when replicating messages between brokers, which might otherwise prevent the Kafka brokers from being fully utilized. +- The number of threads made available for processing requests is increased to 24 from the default 8, and the number of threads made available for managing network traffic is increased to 9 from the default 3. This helps prevent bottlenecks for producers or consumers, which might otherwise prevent the Kafka brokers from being fully utilized. +- The number of threads made available for cleaning the Kafka log is increased to 6 from the default 1. This helps to ensure that records that have exceeded their retention period are removed from the log in a timely manner, and prevents them from accumulating in a heavily loaded system. + +Resource requirements for this deployment: + +| CPU request (cores) | CPU limit (cores) | Memory request (GiB) | Memory limit (GiB) | Chargeable cores (see [licensing]({{ '/support/licensing/#calculating-licenses-required' | relative_url }}))| +| ------------------- | ----------------- | ------------------- | ----------------- | ---- | +| 2.8 | 12.2 | 5.9 | 14.2 | 3 | + +{{site.data.reuse.sample_select_note}} + +{{site.data.reuse.prod_persistence_note}} + +**Note:** If you want to authenticate with SCRAM for this **Minimal production** deployment, see the **Minimal production SCRAM** sample. The **Minimal production SCRAM** sample has the same settings and resource requirements as the Development sample, but does not require {{site.data.reuse.icpfs}} to be installed and specifies [SCRAM authentication](../../security/managing-access/#managing-access-to-the-ui-and-cli-with-scram) to access the {{site.data.reuse.es_name}} UI and CLI. + +#### Example deployment: **Production 3 brokers** + +Overview: A small production deployment for {{site.data.reuse.es_name}}. + +This example installs a production-ready {{site.data.reuse.es_name}} instance similar to the [**Minimal production**](#example-deployment-minimal-production) setup, but with added resource requirements: +- 3 Kafka brokers and 3 ZooKeeper nodes. +- Internally, TLS encryption is used between containers. +- External connections use TLS encryption and SCRAM SHA 512 authentication. +- The memory and CPU requests and limits for the Kafka brokers are increased compared to the **Minimal production** sample described previously to give them the bandwidth to process a larger number of messages. + +Resource requirements for this deployment: + +| CPU request (cores) | CPU limit (cores) | Memory request (GiB) | Memory limit (GiB) | Chargeable cores (see [licensing]({{ '/support/licensing/#calculating-licenses-required' | relative_url }}))| +| ------------------- | ----------------- | ------------------- | ----------------- | ---- | +| 8.5 | 15.2 | 29.3 | 31.9 | 6 | + +Ensure you have sufficient CPU capacity and physical memory in your environment to service at least the resource **request** values. The resource **limit** values constrain the amount of resource the {{site.data.reuse.es_name}} instance is able to consume. + +{{site.data.reuse.sample_select_note}} + +{{site.data.reuse.prod_persistence_note}} + +#### Example deployment: **Production 6 brokers** + +Overview: A medium sized production deployment for {{site.data.reuse.es_name}}. + +This sample configuration is similar to the [**Production 3 brokers**](#example-deployment-production-3-brokers) sample described earlier, but with an increase in the following settings: +- Uses 6 brokers rather than 3 to allow for additional message capacity. +- The resource settings for the individual brokers are the same, but the number of threads made available for replicating messages between brokers is increased to 6 to cater for the additional brokers to manage. + +Resource requirements for this deployment: + +| CPU request (cores) | CPU limit (cores) | Memory request (GiB) | Memory limit (GiB) | Chargeable cores (see [licensing]({{ '/support/licensing/#calculating-licenses-required' | relative_url }}))| +| ------------------- | ----------------- | ------------------- | ----------------- | ---- | +| 14.5 | 21.2 | 53.0 | 55.6 | 12 | + +Ensure you have sufficient CPU capacity and physical memory in your environment to service at least the resource **request** values. The resource **limit** values constrain the amount of resource the {{site.data.reuse.es_name}} instance is able to consume. + +{{site.data.reuse.sample_select_note}} + +{{site.data.reuse.prod_persistence_note}} + + +#### Example deployment: **Production 9 brokers** + +Overview: A large production deployment for {{site.data.reuse.es_name}}. + +This sample configuration is similar to the [**Production 6 brokers**](#example-deployment-production-6-brokers) sample described earlier, but with an increase in the following settings: +- Uses 9 Brokers rather than 6 to allow for additional message capacity. +- The resource settings for the individual brokers are the same, but the number of threads made available for replicating messages between brokers is increased to 9 to cater for the additional brokers to manage. + +Resource requirements for this deployment: + +| CPU request (cores) | CPU limit (cores) | Memory request (GiB) | Memory limit (GiB) | Chargeable cores (see [licensing]({{ '/support/licensing/#calculating-licenses-required' | relative_url }}))| +| ------------------- | ----------------- | ------------------- | ----------------- | ---- | +| 20.5 | 27.2 | 76.7 | 79.3 | 18 | + +Ensure you have sufficient CPU capacity and physical memory in your environment to service at least the resource **request** values. The resource **limit** values constrain the amount of resource the {{site.data.reuse.es_name}} instance is able to consume. + +{{site.data.reuse.sample_select_note}} + +{{site.data.reuse.prod_persistence_note}} + + +## Planning for persistent storage + +If you plan to have persistent volumes, [consider the disk space](../capacity-planning/#disk-space-for-persistent-volumes) required for storage. + +Both Kafka and ZooKeeper rely on fast write access to disks. Use separate dedicated disks for storing Kafka and ZooKeeper data. For more information, see the disks and filesystems guidance in the [Kafka documentation](https://kafka.apache.org/37/documentation/#diskandfs){:target="_blank"}, and the deployment guidance in the [ZooKeeper documentation](https://zookeeper.apache.org/doc/r3.5.7/zookeeperAdmin.html#sc_designing){:target="_blank"}. + +If persistence is enabled, each Kafka broker and ZooKeeper server requires one physical volume each. The number of Kafka brokers and ZooKeeper servers depends on your setup (for example, see the provided samples described in [resource requirements](../prerequisites/#resource-requirements)). + +You either need to create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#static){:target="_blank"} for each physical volume, or specify a storage class that supports [dynamic provisioning](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#dynamic){:target="_blank"}. Each component can use a different storage class to control how physical volumes are allocated. + + For information about creating persistent volumes and creating a storage class that supports dynamic provisioning: + +- For {{site.data.reuse.openshift_short}}, see the [{{site.data.reuse.openshift_short}} documentation](https://docs.openshift.com/container-platform/4.16/storage/understanding-persistent-storage.html){:target="_blank"}. +- For other Kubernetes platforms, see the [Kubernetes documentation](https://kubernetes.io/docs/concepts/storage/persistent-volumes/){:target="_blank"}. + +You must have the Cluster Administrator role for creating persistent volumes or a storage class. + +- If these persistent volumes are to be created manually, this must be done by the system administrator before installing {{site.data.reuse.es_name}}. These will then be claimed from a central pool when the {{site.data.reuse.es_name}} instance is deployed. The installation will then claim the required number of persistent volumes from this pool. +- If these persistent volumes are to be created automatically, ensure a [dynamic provisioner](https://docs.openshift.com/container-platform/4.16/storage/dynamic-provisioning.html){:target="_blank"} is configured for the storage class you want to use. See [data storage requirements](../prerequisites/#data-storage-requirements) for information about storage systems supported by {{site.data.reuse.es_name}}. + +**Important:** When creating persistent volumes for each component, ensure the correct **Access mode** is set for the volumes as described in the following table. + +| Component | Access mode | +| --------------- | -------------------------------- | +| Kafka | `ReadWriteOnce` | +| ZooKeeper | `ReadWriteOnce` | + +To use persistent storage, [configure the storage properties](../configuring/#enabling-persistent-storage) in your `EventStreams` custom resource. + +## Planning for security + +{{site.data.reuse.es_name}} has highly configurable security options that range from the fully secured default configuration to no security for basic development and testing. + +The main security vectors to consider are: + +- Kafka listeners +- Pod-to-Pod communication +- Access to the {{site.data.reuse.es_name}} UI and CLI +- REST endpoints (REST Producer, Admin API, Apicurio Registry) + +Secure instances of {{site.data.reuse.es_name}} will make use of TLS to protect network traffic. Certificates will be generated by default, or you can [use custom certificates](../configuring/#using-your-own-certificates). + +**Note:** If you want to use custom certificates, ensure you configure them before installing {{site.data.reuse.es_name}}. + +### {{site.data.reuse.es_name}} UI and CLI access + +You can [configure secure access](../../installing/configuring/#configuring-ui-and-cli-security) to the {{site.data.reuse.es_name}} UI and CLI. Depending on the authentication type set, you log in by using an {{site.data.reuse.icpfs}} Identity and Access Management (IAM) user, or by using a Kafka user configured with SCRAM-SHA-512 authentication, or by using Keycloak as part of {{site.data.reuse.cp4i}}. For more information about accessing the UI and CLI securely, see [managing access](../../security/managing-access/#accessing-the-event-streams-ui-and-cli). + +#### SCRAM + +If you set SCRAM authentication access for the {{site.data.reuse.es_name}} UI and CLI, Kafka users that have been configured to use SCRAM-SHA-512 authentication can log in to the {{site.data.reuse.es_name}} UI and CLI by using the username and password of that Kafka user. They can then access parts of the UI and run CLI commands based on the ACL permissions of the Kafka user. + +Whilst it is highly recommended to always configure {{site.data.reuse.es_name}} with security enabled, it is also possible to configure the {{site.data.reuse.es_name}} UI to not require a login, which can be useful for proof of concept (PoC) environments. For details, see [configuring {{site.data.reuse.es_name}} UI and CLI access](../configuring/#configuring-ui-and-cli-security). + +#### Keycloak + +By default, in Keycloak, the secure {{site.data.reuse.es_name}} instance will require an `eventstreams-admin` or `admin` [role](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=management-cloud-pak-integration-roles-permissions){:target="_blank"} to authorize access. + +You can add users and groups directly to Keycloak, connect Keycloak to an LDAP user registry to import users and groups, or connect Keycloak to an OpenID Connect (OIDC) or Security Assertion Markup Language (SAML) identity provider to manage users and groups. See how to [add users and groups](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=groups-managing-users-in-keycloak){:target="_blank"} based on your preference. + +**Note:** Authentication through Keycloak is not supported in the {{site.data.reuse.es_name}} CLI. You can authenticate the {{site.data.reuse.es_name}} CLI with the SCRAM authentication and then proceed to use an {{site.data.reuse.es_name}} instance that is configured with Keycloak. + +#### IAM + +{{site.data.reuse.iam_note}} + +By default, in IAM, the secure {{site.data.reuse.es_name}} instance will require an `Administrator` or higher [role](https://www.ibm.com/docs/en/cpfs?topic=guide-role-based-access-control-clusters#iam){:target="_blank"} (`ClusterAdministrator` and `CloudpakAdministrator`) to authorize access. + +You can connect to a Lightweight Directory Access Protocol (LDAP) directory and add users from your LDAP directory into your cluster. For example, if you are using IAM, you can configure IAM to setup LDAP, assign roles to LDAP users, and [create teams](https://www.ibm.com/docs/en/cloud-paks/foundational-services/3.23?topic=apis-team-management#create){:target="_blank"}, as described in [configuring LDAP connections](https://www.ibm.com/docs/en/cloud-paks/foundational-services/3.23?topic=users-configuring-ldap-connection){:target="_blank"}. + +### REST endpoint security + +Review the security and configuration settings of your development and test environments. +The REST endpoints of {{site.data.reuse.es_name}} have a number of configuration capabilities. See [configuring access](../configuring/#rest-services-access) for details. + +### Securing communication between pods + +By default, Pod-to-Pod encryption is enabled. You can [configure encryption between pods](../configuring/#configuring-encryption-between-pods) when configuring your {{site.data.reuse.es_name}} installation. + +### Kafka listeners + +{{site.data.reuse.es_name}} has both internal and external configurable Kafka listeners. Optionally, each Kafka listener can be [secured with TLS or SCRAM](../configuring/#kafka-access). + +## Planning for resilience + +If you are looking for a more resilient setup, or want to plan for [disaster recovery](../../installing/disaster-recovery), consider setting up multiple availability zones and creating mirrored topics in other clusters. Also, set up your environment to support Kafka's inherent high availability design. + +### Kafka high availability + +Kafka is designed for high availability and fault tolerance. + +To reduce the impact of {{site.data.reuse.es_name}} Kafka broker failures, configure your installation with at least three brokers and spread them across several Kubernetes nodes by ensuring you have at least as many nodes as brokers. For example, for 3 Kafka brokers, ensure you have at least 3 nodes running on separate physical servers. + +Kafka ensures that topic-partition replicas are spread across available brokers up to the replication factor specified. Usually, all of the replicas will be in-sync, meaning that they are all fully up-to-date, although some replicas can temporarily be out-of-sync, for example, when a broker has just been restarted. + +The replication factor controls how many replicas there are, and the minimum in-sync configuration controls how many of the replicas need to be in-sync for applications to produce and consume messages with no loss of function. For example, a typical configuration has a replication factor of 3 and minimum in-sync replicas set to 2. This configuration can tolerate 1 out-of-sync replica, or 1 worker node or broker outage with no loss of function, and 2 out-of-sync replicas, or 2 worker node or broker outages with loss of function but no loss of data. + +The combination of brokers spread across nodes together with the replication feature make a single {{site.data.reuse.es_name}} cluster highly available. + +You can also further ensure high availability for your environment by increasing the number of {{site.data.reuse.es_name}} [operator replicas](../installing/#scaling-the-operator-for-high-availability). + +### Multiple availability zones + +To add further resilience to your {{site.data.reuse.es_name}} cluster, you can split your servers across multiple data centers or zones, so that even if one zone experiences a failure, you still have a working system. + +[Multizone support](https://kubernetes.io/docs/setup/best-practices/multiple-zones/){:target="_blank"} provides the option to run a single Kubernetes cluster in multiple availability zones within the same region. Multizone clusters are clusters of either physical or virtual servers that are spread over different locations to achieve greater resiliency. If one location is shut down for any reason, the rest of the cluster is unaffected. + +**Note:** For {{site.data.reuse.es_name}} to work effectively within a multizone cluster, the network latency between zones must not be greater than 20 ms for Kafka to replicate data to the other brokers. + +Typically, high availability requires a minimum of 3 zones (sites or data centers) to ensure a quorum with high availability for components, such as Kafka and ZooKeeper. Without the third zone, you might end up with a third quorum member in a zone that already has a member of the quorum, consequently if that zone goes down, the majority of the quorum is lost and loss of function is inevitable. + +Kubernetes platforms require a minimum of 3 zones for high availability topologies and {{site.data.reuse.es_name}} supports that model. This is different from the traditional primary and backup site configuration, and is a move to support the quorum-based application paradigm. + +With [zone awareness](https://kubernetes.io/docs/setup/best-practices/multiple-zones/#distributing-nodes-across-zones){:target="_blank"}, Kubernetes automatically distributes pods in a replication controller across different zones. For workload-critical components, for example Kafka, ZooKeeper and REST Producer, set the number of replicas of each component to at least match the number of zones. This provides at least one replica of each component in each zone, so in the event of loss of a zone the service will continue using the other working zones. + +For information about how to prepare multiple zones, see [preparing for multizone clusters](../preparing-multizone). + + + +### Topic mirroring + +Consider configuring [geo-replication](../../georeplication/about/) or [MirrorMaker 2.0](../../mirroring/mirrormaker) to aid your [disaster recovery](../../installing/disaster-recovery) and resilience planning, by ensuring a copy of the event data is available in other regions. + +You can deploy multiple instances of {{site.data.reuse.es_name}} and use topic mirroring to synchronize data between your clusters to help maintain service availability. No additional preparation is required on the origin cluster because the mirroring component runs on the destination cluster. + +[MirrorMaker 2.0](https://strimzi.io/blog/2020/03/30/introducing-mirrormaker2/){:target="_blank"} is part of Apache Kafka and is therefore included with {{site.data.reuse.es_name}}. It uses Kafka Connect to mirror topics between separate Kafka clusters. In addition, {{site.data.reuse.es_name}} provides a [geo-replication](../../georeplication/about/) feature, which is built on MirrorMaker 2.0, and offers a simplified mechanism for setting up mirrored topics across multiple {{site.data.reuse.es_name}} environments. + +Use the [geo-replication](../../georeplication/about/) feature to easily perform basic replication of topics between {{site.data.reuse.es_name}} clusters. Use MirrorMaker 2.0 directly to move data between {{site.data.reuse.es_name}} clusters and other Kafka clusters, or when lower-level aspects also need to be replicated, such as offsets and topic configuration. + +### Cruise Control + +Cruise Control is an open-source system for optimizing your Kafka cluster by monitoring cluster workload, rebalancing a cluster based on predefined constraints, and detecting and fixing anomalies. +You can set up {{site.data.reuse.es_name}} to use the following Cruise Control features: + +- Generating optimization proposals from multiple optimization goals. +- Rebalancing a Kafka cluster based on an optimization proposal. + +**Note:** {{site.data.reuse.es_name}} does not support other Cruise Control features. + +[Enable Cruise Control](../configuring/#enabling-and-configuring-cruise-control) for {{site.data.reuse.es_name}} and configure optimization goals for your cluster. + +**Note:** Cruise Control stores data in Kafka topics. It does not have its own persistent storage configuration. Consider using persistent storage for your Kafka topics when using Cruise Control. + +## Planning for log management + +{{site.data.reuse.es_name}} follows widely adopted logging method for containerized applications and writes to standard output and standard error streams. + +You can install any logging solution that integrates with Kubernetes such as [cluster logging](https://docs.openshift.com/container-platform/4.16/observability/logging/cluster-logging.html){:target="_blank"} provided by the {{site.data.reuse.openshift_short}} to collect, store, and visualize logs. + +You can use log data to monitor cluster activity and investigate any problems affecting your [system health](../../administering/deployment-health/). + +## Kafka static configuration properties + +You can set [Kafka broker configuration](https://strimzi.io/docs/operators/0.42.0/configuring.html#type-KafkaClusterSpec-reference){:target="_blank"} settings in your `EventStreams` custom resource under the property `spec.strimziOverrides.kafka`. These settings will override the default Kafka configuration defined by {{site.data.reuse.es_name}}. + +You can also use this configuration property to modify read-only Kafka broker settings for an existing {{site.data.reuse.es_name}} installation. Read-only parameters are defined by Kafka as settings that require a broker restart. Find out more about the [Kafka configuration options and how to modify them](../../administering/modifying-installation/#modifying-kafka-broker-configuration-settings) for an existing installation. + +## Connecting clients + +By default, Kafka client applications connect to cluster using the Kafka bootstrap host address. Find out more about [connecting external clients](../configuring/#configuring-access) to your installation. + +## Monitoring Kafka clusters + +{{site.data.reuse.es_name}} can be configured to export application metrics in Prometheus and JMX formats. These metrics provide useful information about the health and performance of your {{site.data.reuse.es_name}} Kafka clusters. + +You can use any monitoring solution compatible with Prometheus and JMX formats to collect, store, visualize, and set up alerts based on metrics provided by {{site.data.reuse.es_name}}. For example, the {{site.data.reuse.openshift_short}} has a built-in option to use Prometheus. You can also use Prometheus on other Kubernetes platforms, where it might also be included, or you can [install](https://prometheus.io/docs/prometheus/latest/installation/){:target="_blank"} it externally. After installing, [configure Prometheus](../../installing/configuring/#configuring-external-monitoring-through-prometheus) to get the metrics. + +To display the metrics, install and configure a visualization tool if your platform does not provide a built-in solution. For example, you can use a dashboard such as [Grafana](https://grafana.com/docs/grafana/latest/){:target="_blank"} and [configure](../../administering/cluster-health/#grafana) it to work with Prometheus to collect and display metrics. + +Alternatively, you can use an observability tool such as [IBM Instana](https://www.ibm.com/docs/en/instana-observability/current?topic=overview){:target="_blank"} to monitor all aspects of an {{site.data.reuse.es_name}} instance without any extra configuration. Instana also offers [Kafka-centric monitoring](https://www.instana.com/supported-technologies/apache-kafka-observability/){:target="_blank"} that can provide useful insights into the performance and the health of your Kafka cluster. + +For more information about keeping an eye on the health of your Kafka cluster, see the [monitoring Kafka](../../administering/cluster-health/) topic. + +## Licensing + +Licensing is typically based on Virtual Processing Cores (VPC). + +For more information about available licenses, chargeable components, and tracking license usage, see the [licensing reference]({{ 'support/licensing' | relative_url }}). + +## FIPS compliance + +{{site.data.reuse.es_name}} can be configuired in a manner that conforms to Federal Information Processing Standards (FIPS). + +For more information about requirements, configuring, and limitations of setting up {{site.data.reuse.es_name}} in a FIPS-complaint manner, see [enabling FIPS](../../security/fips). \ No newline at end of file diff --git a/_es_11.5/02-installing/03-multizone-considerations.md b/_es_11.5/02-installing/03-multizone-considerations.md new file mode 100644 index 00000000..b905afa9 --- /dev/null +++ b/_es_11.5/02-installing/03-multizone-considerations.md @@ -0,0 +1,49 @@ +--- +title: "Considerations for multizone deployments" +excerpt: "Consider the following when planning for deploying Event Streams to a cluster with multiple availability zones." +categories: installing +slug: multizone-considerations +toc: true +--- + +As part of planning for a [resilient deployment](../planning/#planning-for-resilience), you can deploy {{site.data.reuse.es_name}} on a multizone {{site.data.reuse.openshift_short}} cluster to allow for greater resilience against external disruption. For more information and instructions, see how to [prepare for a multizone deployment](../preparing-multizone/), or see the [tutorial]({{ 'tutorials/multi-zone-tutorial/' | relative_url }}){:target="_blank"}. Before following these instructions and deploying {{site.data.reuse.es_name}} on a multizone cluster, consider the following information. + +## Reasons to consider multiple availability zones + +Kafka itself offers strong resilience against many failures that could occur within a compute environment. However, Kafka cannot deal with a failure that affects the entire cluster it resides on. In preparation for both planned or unplanned outages, Kafka requires an external redundancy solution or backup to avoid data loss. One solution for this is a Kafka cluster with brokers distributed across multiple availability zones. + +An availability zone is an isolated set of computing infrastructure. This includes all aspects of the infrastructure from compute resources to cooling. Data centers are often made up of multiple availability zones. This allows planned outages for maintenance to be done per availability zone, and also helps contain damage from any unplanned outages due to problems within any of the availability zones. + +**Note:** A multizone Kafka cluster offers resilience against failures in a single compute environment, but does not allow for the brokers to be deployed far apart due to latency issues. + +## Multizone Kafka Cluster + +In a multizone Kafka cluster, the brokers are deployed in one virtual compute environment that spans multiple availability zones. This requires the {{site.data.reuse.openshift_short}} environment to have nodes that are distributed across multiple availability zones. The Kafka brokers can then be set to run on specific nodes by using Kubernetes topology labels. The following diagram shows the resulting architecture: + +![3 Kafka brokers on a stretch Openshift Cluster with 3 availability zones]({{ 'images/' | relative_url }}/multizone-kafka-arch.png "Diagram showing 3 Kafka brokers deployed on a stretch OpenShift cluster"){:height="100%" width="100%"} + +### Trade-offs in multizone deployments + +#### Latency + +For Kafka to function reliably in a stretch {{site.data.reuse.openshift_short}} cluster configuration, network latency between the nodes of your Kubernetes cluster must not be greater than 20ms. Low single digit latency values are preferable. The reason is that Kafka brokers and ZooKeeper connections are considered as being deployed on the same hardware. To work around this, you can adjust the following Kafka options in the {{site.data.reuse.es_name}} custom resource: + +- `zookeeper.connection.timeout.ms` sets the maximum time in milliseconds that ZooKeeper waits for a connection to establish when communicating with a Kafka broker. +- `replica.lag.time.max.ms` sets the maximum time available in milliseconds for follower replicas to catch up to the leader. + +By increasing these values, Kafka can avoid timeouts. However, there is a performance cost to this. Increasing `zookeeper.connection.timeout.ms` will result in longer wait times between operations as ZooKeeper might wait longer for a response from Kafka brokers, while increasing `replica.lag.time.max.ms` could result in longer time periods during which your replicas might be out of sync. + +#### Replication + +In a multizone Kafka cluster, topics, partitions, and replicas need to be carefully managed with respect to managing data redundancy. You can manage this by choosing a suitable value for `min.insync.replicas` so that in the event of an outage in one availability zone: +1. No data is lost. +2. The Kafka cluster can still operate, meaning that the number of remaining replicas is greater than or equal to the value of `min.insync.replicas`. + +For example, the following table provides information about what to set as the value for `min.insync.replicas` across 3 availability zones : + +| Number of Kubernetes nodes | Number of Kafka brokers | Value for `min.insync.replicas` | +| -------------------------- | ----------------------- | ---------------------------------| +| 3 | 3 | 2 | +| 3 | 4 | 3 | +| 3 | 5 | 3 | +| 3 | 6 | 4 | diff --git a/_es_11.5/02-installing/04-capacity-planning.md b/_es_11.5/02-installing/04-capacity-planning.md new file mode 100644 index 00000000..747b4397 --- /dev/null +++ b/_es_11.5/02-installing/04-capacity-planning.md @@ -0,0 +1,111 @@ +--- +title: "Performance and capacity planning" +excerpt: "Consider the following when planning for the performance and capacity requirements of your installation." +categories: installing +slug: capacity-planning +toc: true +--- + + + +## Guidance for production environments + +You can use one of the sample configurations provided by {{site.data.reuse.es_name}} to set up a production deployment. For information about the provided samples, see the [sample deployments](../planning/#sample-deployments) section. + +If needed, you can [modify the selected sample configuration](../configuring) when you deploy a new instance, or make changes at a [later time](../../administering/modifying-installation). + +If required by your planned workload, you can further increase the number of Kafka brokers, and the amount of CPU and memory available to them. For changing such values, see the guidance about [scaling](../../administering/scaling/) {{site.data.reuse.es_name}}. + +A [performance report]({{ 'pdfs' | relative_url }}/11.0.4 Performance Report_v3.pdf){:target="_blank"} based on example case studies is available to provide guidance for setting these values. + +**Note:** Although the testing for the report was based on Apache Kafka version 2.3.0, the performance numbers are broadly applicable to current versions of Kafka as well. + +## Tuning {{site.data.reuse.es_name}} Kafka performance + +When preparing for your {{site.data.reuse.es_name}} installation, review your workload requirements and consider the configuration options available for performance tuning your {{site.data.reuse.es_name}} installation. + +Kafka offers a number of configuration settings that can be adjusted as necessary for an {{site.data.reuse.es_name}} deployment. These can be used to address any bottlenecks in the system as well as perform fine tuning of Kafka performance. + +Kafka provides a wide range of configuration properties to set, but consider the following when reviewing performance requirements: + +- The `num.replica.fetchers` property sets the number of threads available on each broker to replicate messages from topic leaders. Increasing this setting increases I/O parallelism in the follower broker, and can help reduce bottlenecks and message latency. You can start by setting this value to match the number of brokers deployed in the system.\\ + **Note:** Increasing this value results in brokers using more CPU resources and network bandwidth. +- The `num.io.threads` property sets the number of threads available to a broker for processing requests. As the load on each broker increases, handling requests can become a bottleneck. Increasing this property value can help mitigate this issue. The value to set depends on the overall system load and the processing power of the worker nodes, which varies for each deployment. There is a correlation between this setting and the `num.network.threads` setting. +- The `num.network.threads` property sets the number of threads available to the broker for receiving and sending requests and responses to the network. The value to set depends on the overall network load, which varies for each deployment. There is a correlation between this setting and the `num.io.threads` setting. +- The `replica.fetch.min.bytes`, `replica.fetch.max.bytes`, and `replica.fetch.response.max.bytes` properties control the minimum and maximum sizes for message payloads when performing inter-broker replication. Set these values to be greater than the `message.max.bytes` property to ensure that all messages sent by a producer can be replicated between brokers. The value to set depends on message throughput and average size, which varies for each deployment. + +These properties are [configured](../configuring/#applying-kafka-broker-configuration-settings) in the `EventStreams` custom resource for an instance when it is first created and can be [modified](../../administering/modifying-installation/#modifying-kafka-broker-configuration-settings) at any time. + +## Disk space for persistent volumes + +You need to ensure you have sufficient disk space in the persistent storage for the Kafka brokers to meet your expected throughput and retention requirements. In Kafka, unlike other messaging systems, the messages on a topic are not immediately removed after they are consumed. Instead, the configuration of each topic determines how much space the topic is permitted and how it is managed. + +Each partition of a topic consists of a sequence of files called log segments. The size of the log segments is determined by the cluster-level configuration property `log.segment.bytes` (default is 1 GB). This can be overridden by using the topic-level configuration `segment.bytes`. + +For each log segment, there are two index files called the time index and the offset index. The size of the index is determined by the cluster-level configuration property `log.index.size.max.bytes` (default is 10 MB). This can be overridden by using the topic-level configuration property `segment.index.bytes`. + +Log segments can be deleted or compacted, or both, to manage their size. The topic-level configuration property `cleanup.policy` determines the way the log segments for the topic are managed. + +For more information about these settings, see [the Kafka documentation](https://kafka.apache.org/37/documentation/#configuration){:target="_blank"}. + +The cluster-level settings are [configured](../configuring/#applying-kafka-broker-configuration-settings) in the `EventStreams` custom resource for an instance when it is first created and can be [modified](../../administering/modifying-installation/#modifying-kafka-broker-configuration-settings) at any time. + +You can specify the cluster and topic-level configurations by using the [{{site.data.reuse.es_name}} CLI](../../administering/modifying-installation/#modifying-kafka-broker-configuration-settings). You can also set topic-level configuration when [setting up the topic](../../getting-started/creating-topics/) in the {{site.data.reuse.es_name}} UI (click **Create a topic**, and set **Show all available options** to **On**). + +**Note:** When using ephemeral storage, ensure you set retention limits for Kafka topics so that you do not run out of disk space. +If [message retention](../../getting-started/creating-topics/) is set to long periods and the message volume is high, the storage requirements for the topics could impact the nodes that host the Kafka pods, and cause the nodes to run out of allocated disk space, which could impact normal operation. + +### Log cleanup by deletion + +If the topic-level configuration property `cleanup.policy` is set to `delete` (the default value), old log segments are discarded when the retention time or size limit is reached, as set by the following properties: + +- Retention time is set by `retention.ms`, and is the maximum time in milliseconds that a log segment is retained before being discarded to free up space. +- Size limit is set by `retention.bytes`, and is the maximum size that a partition can grow to before old log segments are discarded. + +By default, there is no size limit, only a time limit. The default time limit is 7 days (604,800,000 ms). + +You also need to have sufficient disk space for the log segment deletion mechanism to operate. The cluster-level configuration property `log.retention.check.interval.ms` (default is 5 minutes) controls how often the broker checks to see whether log segments should be deleted. The cluster-level configuration property `log.segment.delete.delay.ms` (default is 1 minute) controls how long the broker waits before deleting the log segments. This means that by default you also need to ensure you have enough disk space to store log segments for an additional 6 minutes for each partition. + +#### Worked example 1 + +Consider a cluster that has 3 brokers, and 1 topic with 1 partition with a replication factor of 3. The expected throughput is 3,000 bytes per second. The retention time period is 7 days (604,800 seconds). + +Each broker hosts 1 replica of the topic's single partition. + +The log capacity required for the 7 days retention period can be determined as follows: 3,000 * (604,800 + 6 * 60) = 1,815,480,000 bytes. + +So, each broker requires approximately 2GB of disk space allocated in its persistent volume, plus approximately 20 MB of space for index files. In addition, allow at least 1 log segment of extra space to make room for the actual cleanup process. Altogether, you need a total of just over 3 GB disk space for persistent volumes. + +#### Worked example 2 + +Consider a cluster that has 3 brokers, and 1 topic with 1 partition with a replication factor of 3. The expected throughput is 3,000 bytes per second. The retention size configuration is set to 2.5 GB. + +Each broker hosts 1 replica of the topic's single partition. + +The number of log segments for 2.5 GB is 3, but you should also allow 1 extra log segment after cleanup. + +So, each broker needs approximately 4 GB of disk space allocated in its persistent volume, plus approximately 40 MB of space for index files. + +The retention period achieved at this rate is approximately 2,684,354,560 / 3,000 = 894,784 seconds, or 10.36 days. + +### Log cleanup by compaction + +If the topic-level configuration property `cleanup.policy` is set to `compact`, the log for the topic is compacted periodically in the background by the log cleaner. In a compacted topic, each message has a key. The log only needs to contain the most recent message for each key, while earlier messages can be discarded. The log cleaner calculates the offset of the most recent message for each key, and then copies the log from start to finish, discarding all but the last of each duplicate key in the log. As each copied segment is created, they are swapped into the log right away to keep the amount of additional space required to a minimum. + +Estimating the amount of space that a compacted topic will require is complex, and depends on factors such as the number of unique keys in the messages, the frequency with which each key appears in the uncompacted log, and the size of the messages. + +### Log cleanup by using both + +You can specify both `delete` and `compact` values for the `cleanup.policy` configuration property at the same time. In this case, the log is compacted, but the cleanup process also follows the retention time or size limit settings. + +When both methods are enabled, capacity planning is simpler than when you only have compaction set for a topic. However, some use cases for log compaction depend on messages not being deleted by log cleanup, so consider whether using both is right for your scenario. + + diff --git a/_es_11.5/02-installing/05-preping-multizone.md b/_es_11.5/02-installing/05-preping-multizone.md new file mode 100644 index 00000000..548c6943 --- /dev/null +++ b/_es_11.5/02-installing/05-preping-multizone.md @@ -0,0 +1,110 @@ +--- +title: "Preparing for multizone clusters" +excerpt: "Prepare for installing your Event Streams on multizone clusters." +categories: installing +slug: preparing-multizone +toc: true +--- + +{{site.data.reuse.es_name}} supports [multiple availability zones](../planning/#multiple-availability-zones) for your clusters. Multizone clusters add resilience to your {{site.data.reuse.es_name}} installation. + +For guidance about handling outages in a multizone setup, see [managing a multizone setup](../../administering/managing-multizone/). + +## Zone awareness + +Kubernetes uses zone-aware information to determine the zone location of each of its nodes in the cluster to enable scheduling of pod replicas in different zones. + +Some clusters, typically AWS, will already be zone aware. For clusters that are not zone aware, each Kubernetes node will need to be set up with a zone label. + +To determine if your cluster is zone aware: + +1. {{site.data.reuse.cncf_cli_login}} +2. Run the following command: + + ```shell + kubectl get nodes --show-labels + ``` + +If your Kubernetes cluster is zone aware, the [label](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/){:target="_blank"} `topology.kubernetes.io/zone` is displayed against each node. + +The value of the label is the zone the node is in, for example, `es-zone-1`. + +If your Kubernetes cluster is not zone aware, all cluster nodes will need to be labeled using a value that identifies the zone that each node is in. For example, run the following command to label and allocate a node to `es-zone-1`: + +```shell +kubectl label node topology.kubernetes.io/zone=es-zone-1 +``` + +The zone label is needed to set up rack awareness when [installing for multizone](../configuring/#applying-kafka-rack-awareness). + + +## Kafka rack awareness + +In addition to zone awareness, Kafka rack awareness helps to spread the Kafka broker pods and Kafka topic replicas across different availability zones, and also sets the brokers' `broker.rack` configuration property for each Kafka broker. + +To set up Kafka rack awareness, Kafka brokers require a cluster role to provide permission to view which Kubernetes node they are running on. + +Before applying [Kafka rack awareness](../configuring/#applying-kafka-rack-awareness) to an {{site.data.reuse.es_name}} installation, apply a cluster role and a cluster role binding: + +1. Create a file called `eventstreams-kafka-broker.yaml` and copy the following YAML content to create the cluster role: + + ```yaml + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: eventstreams-kafka-broker + labels: + app: eventstreams + rules: + - verbs: + - get + - create + - watch + - update + - delete + - list + apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - verbs: + - get + - create + - watch + - update + - delete + - list + apiGroups: + - "" + resources: + - nodes + ``` + +2. Apply the cluster role by using the following command: + + ```shell + kubectl apply -f eventstreams-kafka-broker.yaml + ``` + +3. Create a file called `eventstreams-kafka-broker-crb.yaml` and copy the following YAML content to create the cluster role binding: + + ```yaml + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: eventstreams-kafka-broker + subjects: + - kind: ServiceAccount + name: eventstreams-cluster-operator + namespace: + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: eventstreams-kafka-broker + ``` + +4. Apply the cluster role binding by using the following command: + + ```shell + kubectl apply -f eventstreams-kafka-broker-crb.yaml + ``` diff --git a/_es_11.5/02-installing/06-installing.md b/_es_11.5/02-installing/06-installing.md new file mode 100644 index 00000000..58159dc5 --- /dev/null +++ b/_es_11.5/02-installing/06-installing.md @@ -0,0 +1,500 @@ +--- +title: "Installing on OpenShift Container Platform" +excerpt: "Find out how to install Event Streams on the OpenShift Container Platform." +categories: installing +slug: installing +toc: true +--- + +The following sections provide instructions about installing {{site.data.reuse.es_name}} on the {{site.data.reuse.openshift}}. The instructions are based on using the {{site.data.reuse.openshift_short}} web console and `oc` command-line utility. + + +{{site.data.reuse.es_name}} can also be installed as part of [{{site.data.reuse.cp4i}}](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=instances-event-streams-deployment){:target="_blank"}. + +## Overview + +{{site.data.reuse.es_name}} is an [operator-based](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/){:target="_blank"} release and uses custom resources to define your {{site.data.reuse.es_name}} configurations. The {{site.data.reuse.es_name}} operator uses the custom resources to deploy and manage the entire lifecycle of your {{site.data.reuse.es_name}} instances. Custom resources are presented as YAML configuration documents that define instances of the `EventStreams` custom resource type. + +Installing {{site.data.reuse.es_name}} has two phases: + +1. Install the {{site.data.reuse.es_name}} operator: this will deploy the operator that will install and manage your {{site.data.reuse.es_name}} instances. +2. Install one or more instances of {{site.data.reuse.es_name}} by using the operator. + +## Before you begin + +- Ensure you have set up your environment [according to the prerequisites](../prerequisites), including setting up your {{site.data.reuse.openshift_short}}. +- If you want to authenticate with Keycloak, ensure you have {{site.data.reuse.cp4i}} 2023.4.1 (operator version 7.2.0) or later [installed](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=installing){:target="_blank"}, including the required dependencies. +- Ensure you have [planned for your installation](../planning), such as preparing for persistent storage, considering security options, and considering adding resilience through multiple availability zones. +- Obtain the connection details for your {{site.data.reuse.openshift_short}} cluster from your administrator. +- The {{site.data.reuse.es_name}} UI includes dashboards for monitoring [Kafka health](../../administering/cluster-health/#viewing-the-preconfigured-dashboard) and [topic health](../../administering/topic-health/). To provide metrics for these dashboards, ensure you enable the {{site.data.reuse.openshift_short}} monitoring stack as described in the {{site.data.reuse.cp4i}} [documentation](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=administering-enabling-openshift-container-platform-monitoring){:target="_blank"}. + + In addition, to provide metrics about topic health, [enable the Kafka Proxy](../../installing/configuring/#enabling-collection-of-producer-metrics). +- In addition, for services offered by {{site.data.reuse.icpfs}}, ensure you have a supported version of the {{site.data.reuse.fs}} [installed](../prerequisites/#optional-ibm-cloud-pak-foundational-services-for-openshift). + +## Create a project (namespace) + +Create a namespace into which the {{site.data.reuse.es_name}} instance will be installed by creating a [project](https://docs.openshift.com/container-platform/4.16/applications/projects/working-with-projects.html){:target="_blank"}. +When you create a project, a namespace with the same name is also created. + +Ensure you use a namespace that is dedicated to a single instance of {{site.data.reuse.es_name}}. This is required because {{site.data.reuse.es_name}} uses network security policies to restrict network connections between its internal components. A single namespace per instance also allows for finer control of user accesses. + +**Important:** Do not use any of the default or system namespaces to install an instance of {{site.data.reuse.es_name}} (some examples of these are: `default`, `kube-system`, `kube-public`, and `openshift-operators`). + +### Creating a project by using the web console + +1. {{site.data.reuse.openshift_ui_login}} +2. Expand the **Home** dropdown and select **Projects** to open the **Projects** panel. +3. Click **Create Project**. +4. Enter a new project name in the **Name** field, and optionally, a display name in the **Display Name** field, and a description in the **Description** field. +5. Click **Create**. + +### Creating a project by using the CLI + +1. {{site.data.reuse.openshift_cli_login}} +2. Run the following command to create a new project: + + ```shell + oc new-project --description="" --display-name="" + ``` + + Where `description` and `display-name` are optional flags to set a description and custom descriptive name for your project. +3. Ensure you are using the project you created by selecting it as follows: + + ```shell + oc project + ``` + + The following message is displayed if successful: + + ```shell + Now using project "" on server "https://:6443". + ``` + +## Choose the operator installation mode + +Before installing the {{site.data.reuse.es_name}} operator, decide if you want the operator to: + +- Manage instances of {{site.data.reuse.es_name}} in **any namespace**. + + To use this option, select `All namespaces on the cluster (default)` later. The operator will be deployed into the system namespace `openshift-operators`, and will be able to manage instances of {{site.data.reuse.es_name}} in any namespace. + +- Only manage instances of {{site.data.reuse.es_name}} in a **single namespace**. + + To use this option, select `A specific namespace on the cluster` later. The operator will be deployed into the specified namespace, and will not be able to manage instances of {{site.data.reuse.es_name}} in any other namespace. + + **Important:** When multiple {{site.data.reuse.es_name}} operators are installed on the same cluster, all the operators share the same custom resource definitions (CRDs). Do not install a previous version of the {{site.data.reuse.es_name}} operator if a later version is already installed on the same cluster. + +## Creating an image pull secret + +Before installing an {{site.data.reuse.es_name}} instance, create an image pull secret called `ibm-entitlement-key` in the namespace where you want to create an instance of {{site.data.reuse.es_name}}. The secret enables container images to be pulled from the registry. + +1. Obtain an entitlement key from the [IBM Container software library](https://myibm.ibm.com/products-services/containerlibrary){:target="_blank"}. +2. Create the secret in the namespace that will be used to deploy an instance of {{site.data.reuse.es_name}} as follows. + + Name the secret `ibm-entitlement-key`, use `cp` as the username, your entitlement key as the password, and `cp.icr.io` as the docker server: + + ```shell + oc create secret docker-registry ibm-entitlement-key --docker-username=cp --docker-password="" --docker-server="cp.icr.io" -n + ``` + + +**Note:** If you do not create the required secret, pods will fail to start with `ImagePullBackOff` errors. In this case, ensure the secret is created and allow the pod to restart. + + +## Decide version control and catalog source + +Before you can install the required IBM operators, make them available for installation by adding the catalog sources to your cluster. Selecting how the catalog source is added will determine the versions you receive. + +Consider how you want to control your deployments, whether you want to install specific versions, and how you want to receive updates. + +- Latest versions: You can install the latest versions of all operators from the IBM Operator Catalog as described in [adding latest versions](#adding-latest-versions). This means that every deployment will always have the latest versions made available, and you cannot specify which version is installed. In addition, upgrades to the latest versions are automatic and provided when they become available. This path is more suitable for development or proof of concept deployments. + +- Specific versions: You can control the version of the operator and instances that are installed by downloading specific Container Application Software for Enterprises (CASE) files as described in [adding specific versions](#adding-specific-versions). This means you can specify the version you deploy, and only receive updates when you take action manually to do so. This is often required in production environments where the deployment of any version might require it to go through a process of validation and verification before it can be pushed to production use. + +### Adding latest versions + +**Important:** Use this method of installation only if you want your deployments to always have the latest version and if you want upgrades to always be automatic. + +Before you can install the {{site.data.reuse.es_name}} operator and use it to create instances of {{site.data.reuse.es_name}}, you must have the IBM Operator Catalog available in your cluster. + +If you already have the IBM Operator Catalog available, you can continue to [installing](#installing-by-using-the-web-console) the {{site.data.reuse.es_name}} operator. + +If you are installing {{site.data.reuse.es_name}} as the first IBM product in your cluster, complete the following steps: + +To make the {{site.data.reuse.es_name}} operator and optional {{site.data.reuse.fs}} dependencies available in the OpenShift OperatorHub catalog, create the following YAML files and apply them as follows. + +To add the IBM Operator Catalog: + +1. Create a file for the IBM Operator Catalog source with the following content, and save as `ibm_catalogsource.yaml`: + + ```yaml + apiVersion: operators.coreos.com/v1alpha1 + kind: CatalogSource + metadata: + name: ibm-operator-catalog + namespace: openshift-marketplace + labels: + backup.eventstreams.ibm.com/component: catalogsource + spec: + displayName: "IBM Operator Catalog" + publisher: IBM + sourceType: grpc + image: icr.io/cpopen/ibm-operator-catalog + updateStrategy: + registryPoll: + interval: 45m + ``` + + Automatic updates of your IBM Operator Catalog can be disabled by removing the polling attribute, `spec.updateStrategy.registryPoll`. + To disable automatic updates, remove the following parameters in the IBM Operator Catalog source YAML under the `spec` field: + + ```yaml + updateStrategy: + registryPoll: + interval: 45m + ``` + + **Important:** Other factors such as Subscription might enable the automatic updates of your deployments. For tight version control of your operators or to install a fixed version, [add specific versions](#adding-specific-versions) of the CASE bundle, and install the operators by using the [CLI](#installing-by-using-the-command-line). + +2. In the OpenShift web console, select the Import YAML option (+) on the upper right. +3. Paste the IBM Operator Catalog source YAML in the YAML editor. You can also drag the YAML files into the editor. +4. Select **Create**. + +Alternatively, you can add the catalog source through the CLI by running the following commands: + +1. {{site.data.reuse.openshift_cli_login}} +2. Apply the source by using the following command: + + ```shell + oc apply -f ibm_catalogsource.yaml + ``` + +The IBM Operator Catalog source is added to the OperatorHub catalog, making the {{site.data.reuse.es_name}} operator available to install. + +### Adding specific versions + +**Important:** Use this method if you want to install specific versions and do not want to automatically receive upgrades or have the latest versions made available immediately. + +Before you can install the required operator versions and use them to create instances of {{site.data.reuse.es_name}}, make their catalog source available in your cluster as described in the following sections. + +**Note:** This procedure must be performed by using the CLI. + +1. Before you begin, ensure that you have the following set up for your environment: + + - The {{site.data.reuse.openshift_short}} CLI (`oc`) [installed](https://docs.openshift.com/container-platform/4.16/cli_reference/openshift_cli/getting-started-cli.html){:target="_blank"}. + - The IBM Catalog Management Plug-in for IBM Cloud Paks (`ibm-pak`) [installed](https://github.com/IBM/ibm-pak#readme){:target="_blank"}. After installing the plug-in, you can run `oc ibm-pak` commands against the cluster. Run the following command to confirm that `ibm-pak` is installed: + + ```shell + oc ibm-pak --help + ``` + +2. Run the following command to download, validate, and extract the {{site.data.reuse.es_name}} CASE. + + ```shell + oc ibm-pak get ibm-eventstreams --version + ``` + +3. Generate mirror manifests by running the following command: + + ```shell + oc ibm-pak generate mirror-manifests ibm-eventstreams icr.io + ``` + + **Note**: To filter for a specific image group, add the parameter `--filter ` to the previous command. + + The previous command generates the following files based on the target (internal) registry provided: + + - catalog-sources.yaml + - catalog-sources-linux-``.yaml (if there are architecture specific catalog sources) + - image-content-source-policy.yaml + - images-mapping.txt + +4. Apply the catalog sources for the operator to the cluster by running the following command: + + ```shell + oc apply -f ~/.ibm-pak/data/mirror/ibm-eventstreams//catalog-sources.yaml + ``` + + Where `` is the case version. + +This adds the catalog source for the {{site.data.reuse.es_name}} making the operator available to install. +You can install the operator by using the [OpenShift web console](#installing-by-using-the-web-console) or the [CLI](#installing-by-using-the-command-line). + +## Install the {{site.data.reuse.es_name}} operator + +Ensure you have considered the {{site.data.reuse.es_name}} operator [requirements](../prerequisites/#operator-requirements), including resource requirements and the required cluster-scoped permissions. + +### Installing by using the web console + +**Important:** To install the operators by using the OpenShift web console, you must add the operators to the [OperatorHub catalog](#adding-latest-versions). OperatorHub updates your operators automatically when a latest version is available. This might not be suitable for some production environments. For production environments that require manual updates and version control, [add specific version](#adding-specific-versions), and then install the {{site.data.reuse.es_name}} operator by using the [CLI](#installing-by-using-the-command-line). + +To install the operator by using the {{site.data.reuse.openshift_short}} web console, complete the following steps: + +1. {{site.data.reuse.openshift_ui_login}} +2. Expand the **Operators** dropdown and select **OperatorHub** to open the **OperatorHub** dashboard. +3. Select the project you want to deploy the {{site.data.reuse.es_name}} instance in. +4. In the **All Items** search box enter `Event Streams` to locate the operator title. +5. Click the **Event Streams** tile to open the install side panel. +6. Click the **Install** button to open the **Create Operator Subscription** dashboard. +7. Select the chosen [installation mode](#choose-the-operator-installation-mode) that suits your requirements. + If the installation mode is **A specific namespace on the cluster**, select the target namespace you created previously. +8. Click **Install** to begin the installation. + +The installation can take a few minutes to complete. + +**Important:** Only install one {{site.data.reuse.es_name}} operator on a cluster. + +### Installing by using the command line + +To install the operator by using the {{site.data.reuse.openshift_short}} command line, complete the following steps: + +1. Change to the namespace (project) where you want to install the operator. For command line installations, this sets the chosen [installation mode](#choose-the-operator-installation-mode) for the operator: + + - Change to the system namespace `openshift-operators` if you are installing the operator to be able to manage instances in all namespaces. + - Change to the custom namespace if you are installing the operator for use in a specific namespace only. + + ```shell + oc project + ``` + +2. Check whether there is an existing `OperatorGroup` in your target namespace: + + ```shell + oc get OperatorGroup + ``` + + If there is an existing `OperatorGroup`, continue to the next step to create a `Subscription`. + + If there is no `OperatorGroup`, create one as follows: + + a. Create a YAML file with the following content, replacing `` with your namespace: + + ```yaml + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: ibm-eventautomation-operatorgroup + namespace: + spec: + targetNamespaces: + - + ``` + + b. Save the file as `operator-group.yaml`. + + c. Run the following command: + + ```shell + oc apply -f operator-group.yaml + ``` + +3. Create a `Subscription` for the {{site.data.reuse.es_name}} operator as follows: + + a. Create a YAML file similar to the following example: + + ```yaml + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: ibm-eventstreams + namespace: + labels: + backup.eventstreams.ibm.com/component: subscription + spec: + channel: + name: ibm-eventstreams + source: + sourceNamespace: openshift-marketplace + ``` + + Where: + + - `` is the namespace where you want to install {{site.data.reuse.es_name}} (`openshift-operators` if you are installing in all namespaces, or a custom name if you are installing in a specific namespace). + - `` is the operator channel for the release you want to install (see the [support matrix]({{ 'support/matrix/#event-streams' | relative_url }})). + - `` is the name of the catalog source that was created for this operator. This is `ibm-eventstreams` when installing a specific version by using a CASE bundle, or `ibm-operator-catalog` if the source is the IBM Operator Catalog. + + b. Save the file as `subscription.yaml`. + + c. Run the following command: + + ```shell + oc apply -f subscription.yaml + ``` + +### Checking the operator status + +You can view the status of the installed operator as follows. + +- To see the installed operator and check its status by using the web console, complete the following steps: + + 1. {{site.data.reuse.openshift_ui_login}} + 2. {{site.data.reuse.task_openshift_navigate_installed_operators}} + 3. {{site.data.reuse.task_openshift_select_operator}} + 4. Scroll down to the **ClusterServiceVersion details** section of the page. + 5. Check the **Status** field. After the operator is successfully installed, this will change to `Succeeded`. + + In addition to the status, information about key events that occur can be viewed under the **Conditions** section of the same page. After a successful installation, a condition with the following message is displayed: `install strategy completed with no errors`. + +- To check the status of the installed operator by using the command line: + + ```shell + oc get csv + ``` + + The command returns a list of installed operators. The installation is successful if the value in the `PHASE` column for your {{site.data.reuse.es_name}} operator is `Succeeded`. + + +**Note:** If the operator is installed into a specific namespace, then it will only appear under the associated project. If the operator is installed for all namespaces, then it will appear under any selected project. If the operator is installed for all namespaces and you select **all projects** from the **Project** drop down, the operator will be shown multiple times in the resulting list, once for each project. + +When the {{site.data.reuse.es_name}} operator is installed, the following additional operators will appear in the installed operator list: + +- Operand Deployment Lifecycle Manager. +- IBM Common Service Operator. + +### Scaling the operator for high availability + +High availability (HA) is the elimination of single points of failure in an environment. In addition to setting up your [Kafka brokers](../planning/#kafka-high-availability) for high availability, you can also set the number of the {{site.data.reuse.es_name}} operator replicas to enable more resilience. + +By increasing the number of replicas to a value greater than 1, you can ensure that the {{site.data.reuse.es_name}} operator continues to function in a wider range of outage scenarios. To ensure uptime in failure situations, the management of your {{site.data.reuse.es_name}} is delegated to the other available operator pods. + +To increase the number replicas, edit the replicas in the `ClusterServiceVersion` object manually or by running the following command: + +```shell +oc patch csv -n ibm-eventstreams.v -p '[{"op":"replace","path":"/spec/install/spec/deployments/0/spec/replicas","value":3}]' --type json +``` + +### Technology Preview feature: KRaft + +Technology Preview features are available to evaluate potential upcoming features. Such features are intended for testing purposes only and not for production use. IBM does not support these features, but might help with any issues raised against them. IBM welcomes feedback on Technology Preview features to improve them. As the features are still under development, functions and interfaces can change, and it might not be possible to upgrade when updated versions become available. + +IBM offers no guarantee that Technology Preview features will be part of upcoming releases and as such become fully supported. + +{{site.data.reuse.es_name}} version 11.1.5 and later includes [Apache Kafka Raft (KRaft)](https://cwiki.apache.org/confluence/display/KAFKA/KIP-500%3A+Replace+ZooKeeper+with+a+Self-Managed+Metadata+Quorum){:target="_blank"} as a Technology Preview feature. +KRaft replaces ZooKeeper for managing metadata, moving the overall handling of metadata into Kafka itself. + + +#### Limitations + +The KRaft mode in {{site.data.reuse.es_name}} has the following limitations: +- Moving existing Kafka clusters deployed with ZooKeeper to use KRaft, or the other way around, is not supported. +- Upgrading your Apache Kafka or {{site.data.reuse.es_name}} operator version, or reverting either one to an earlier version is not supported. To do so, you delete the cluster, upgrade the operator, and deploy a new Kafka cluster. +- The Topic Operator is not supported. The `spec.entityOperator.topicOperator` property must be removed from the Kafka custom resource. +- JBOD storage is not supported. You can use `type: jbod` for storage, but the JBOD array can contain only one disk. +- Geo-replication is not supported. + +#### Enabling KRaft + +To enable KRaft, add the following annotations to the `EventStreams` custom resource: +- `eventstreams.ibm.com/kraft: enabled` +- `eventstreams.ibm.com/node-pools: enabled` + +For example: + +```yaml +- metadata: + name: light-insecure + namespace: openshift-operators + annotations: + eventstreams.ibm.com/node-pools: enabled + eventstreams.ibm.com/kraft: enabled +``` + + +## Install an {{site.data.reuse.es_name}} instance + +Instances of {{site.data.reuse.es_name}} can be created after the {{site.data.reuse.es_name}} operator is installed. If the operator was installed into **a specific namespace**, then it can only be used to manage instances of {{site.data.reuse.es_name}} in that namespace. If the operator was installed for **all namespaces**, then it can be used to manage instances of {{site.data.reuse.es_name}} in any namespace, including those created after the operator was deployed. + +When installing an instance of {{site.data.reuse.es_name}}, ensure you are using a namespace that an operator is managing. + + + +### Installing an instance by using the web console + +To install an {{site.data.reuse.es_name}} instance through the {{site.data.reuse.openshift_short}} web console, do the following: + +1. {{site.data.reuse.openshift_ui_login}} +2. {{site.data.reuse.task_openshift_navigate_installed_operators}} +3. {{site.data.reuse.task_openshift_select_operator}} + + **Note:** If the operator is not shown, it is either not installed or not available for the selected namespace. + +4. In the **Operator Details** dashboard, click the **{{site.data.reuse.es_name}}** tab. +5. Click the **Create EventStreams** button to open the **Create EventStreams** panel. You can use this panel to define an `EventStreams` custom resource. + +From here you can install by using the [form view](#installing-by-using-the-form-view). For more advanced configurations or to install one of the samples, see [installing by using the YAML view](#installing-by-using-the-yaml-view). + +#### Installing by using the form view + +To configure an `EventStreams` custom resource, do the following: + +1. Enter a name for the instance in the **Name** field. +2. Click the license accept toggle to set it to **True**. + ![Accepting license toggle]({{ 'images' | relative_url }}/license_accept_form.png "Screen capture showing how to toggle the license accept field to true"){:height="100%" width="100%"} +3. Ensure that the correct values for **Product License** and **Product Use** are selected from the drop-down lists. These values are used for metering purposes and could result in inaccurate charging and auditing if set incorrectly. For more information about the available options, see the [licensing reference]({{ 'support/licensing' | relative_url }}). +4. Optional: You can configure other components such as **Kafka**, **ZooKeeper**, and **Security** to suit your [requirements](../configuring). +5. Scroll down and click the **Create** button at the bottom of the page to deploy the {{site.data.reuse.es_name}} instance. +6. Wait for the installation to complete. +7. You can now verify your installation and consider other [post-installation tasks](../post-installation/). + +#### Installing by using the YAML view + +Alternatively, you can configure the `EventStreams` custom resource by editing YAML documents. To do this, click the **Edit YAML** tab. + +A number of sample configurations are provided on which you can base your deployment. These range from smaller deployments for non-production development or general experimentation to large scale clusters ready to handle a production workload. Alternatively, a pre-configured YAML file containing the custom resource sample can be dragged and dropped onto this screen to apply the configuration. + +To view the samples, do the following: + +1. Select the **Samples** tab to show the available sample configurations. +2. Click the **Try it** link under any of the samples to open the configuration in the **Create EventStreams** panel. + +More information about these samples is available in the [planning](../planning/#sample-deployments) section. You can base your deployment on the sample that most closely reflects your requirements and apply [customizations](../configuring) on top as required. + +When modifying the sample configuration, the updated document can be exported from the **Create EventStreams** panel by clicking the **Download** button and re-imported by dragging the resulting file back into the window. + +**Important:** Ensure that the `spec.license.accept` field in the custom resource YAML is set to `true`, and that the correct values are selected for the `spec.license.license` and `spec.license.use` fields before deploying the {{site.data.reuse.es_name}} instance. These values are used for metering purposes and could result in inaccurate charging and auditing if set incorrectly. For more information about the available options, see the [licensing reference]({{ 'support/licensing' | relative_url }}). + +**Note:** If experimenting with {{site.data.reuse.es_name}} for the first time, the **Lightweight without security** sample is the smallest and simplest example that can be used to create an experimental deployment. For the smallest production setup, use the **Minimal production** sample configuration. + +To deploy an {{site.data.reuse.es_name}} instance, use the following steps: + +1. Complete any changes to the sample configuration in the **Create EventStreams** panel. +2. Click **Create** to begin the installation process. +3. Wait for the installation to complete. +4. You can now verify your installation and consider other [post-installation tasks](../post-installation/). + +### Installing an instance by using the CLI + +To install an instance of {{site.data.reuse.es_name}} from the command-line, you must first prepare an `EventStreams` custom resource configuration in a YAML file. + +A number of sample configuration files are available in [GitHub](https://ibm.biz/ea-es-samples){:target="_blank"}, where you can select the GitHub tag for your {{site.data.reuse.es_name}} version to access the correct samples, and then go to `/cr-examples/eventstreams/openshift` to access the OpenShift samples. + +The sample configurations range from smaller deployments for non-production development or general experimentation to large scale clusters ready to handle a production workload. + +More information about these samples is available in the [planning](../planning/#sample-deployments) section. You can base your deployment on the sample that most closely reflects your requirements and apply [customizations](../configuring) on top as required. + +**Important:** Ensure that the `spec.license.accept` field in the custom resource YAML is set to `true`, and that the correct values are selected for the `spec.license.license` and `spec.license.use` fields before deploying the {{site.data.reuse.es_name}} instance. These values are used for metering purposes and could result in inaccurate charging and auditing if set incorrectly. For more information about the available options, see the [licensing reference]({{ 'support/licensing' | relative_url }}). + +**Note:** If experimenting with {{site.data.reuse.es_name}} for the first time, the **Lightweight without security** sample is the smallest and simplest example that can be used to create an experimental deployment. For the smallest production setup, use the **Minimal production** sample configuration. + +To deploy an {{site.data.reuse.es_name}} instance, run the following commands: + +1. Set the project where your `EventStreams` custom resource will be deployed in: + + ```shell + oc project + ``` + +2. Apply the configured `EventStreams` custom resource: + + ```shell + oc apply -f + ``` + + For example: + + ```shell + oc apply -f development.yaml + ``` + +3. Wait for the installation to complete. +4. You can now verify your installation and consider other [post-installation tasks](../post-installation/). diff --git a/_es_11.5/02-installing/06a-installing-offline.md b/_es_11.5/02-installing/06a-installing-offline.md new file mode 100644 index 00000000..f825eea8 --- /dev/null +++ b/_es_11.5/02-installing/06a-installing-offline.md @@ -0,0 +1,330 @@ +--- +title: "Installing in an offline environment" +excerpt: "Find out how to install in an offline (also referred to as air-gapped or disconnected) environment." +categories: installing +slug: offline +toc: true +--- + +If you are working in an environment where your cluster is not connected to the internet, you can install {{site.data.reuse.es_name}} by using the container-based software that is provided as a Container Application Software for Enterprises (CASE) bundle. + +CASE is a specification that defines metadata and structure for packaging, managing, and unpacking containerized applications. When deploying in an offline (also referred to as air-gapped or disconnected) environment, you mimic a typical online installation by using images in your own registry. You can use the CASE content to mirror images to an internal registry within your restricted environment, and to install the images from that registry. + +Follow the instructions to download the {{site.data.reuse.es_name}} CASE bundle, mirror the image, apply the catalog source, and install the operator on OpenShift and other Kubernetes platforms. + +**Note:** For completing tasks by using the command line, you can use both `kubectl` and `oc` commands if your deployment is on the {{site.data.reuse.openshift_short}}. This documentation set includes instructions that use the `kubectl` command, except for cases where the task is specific to OpenShift. + + +## Prerequisites + +Ensure you have the following set up for your environment: + +- A computer with access to both the public internet and the network-restricted environment on which you can run the required commands. This computer must also have access to a local registry and to the {{site.data.reuse.openshift_short}} clusters, and is referred to as a *bastion host*. +- [Docker](https://docs.docker.com/engine/install/){:target="_blank"} or [Podman CLI](https://podman.io/getting-started/installation.html){:target="_blank"} installed. +- A private container registry that can be accessed by the cluster and the bastion host, and which will be used to store all images in your restricted network. +- The IBM Catalog Management Plug-in for IBM Cloud Paks (`ibm-pak`) [installed](https://github.com/IBM/ibm-pak#readme){:target="_blank"}. This plug-in allows you to run `kubectl ibm-pak` commands against the cluster. Run the following command to confirm that `ibm-pak` is installed: + + ```shell + kubectl ibm-pak --help + ``` + +If you are using {{site.data.reuse.openshift}}, ensure you have the following set up for your environment: + +- A supported version of {{site.data.reuse.openshift_short}} [installed](https://docs.openshift.com/container-platform/4.16/welcome/index.html){:target="_blank"}. For supported versions, see the [support matrix]({{ 'support/matrix/#event-streams' | relative_url }}). +- The {{site.data.reuse.openshift_short}} CLI (`oc`) [installed](https://docs.openshift.com/container-platform/4.16/cli_reference/openshift_cli/getting-started-cli.html){:target="_blank"}. + +If you are using other Kubernetes platforms, ensure you have the following set up for your environment: + +- A supported version of a Kubernetes platform installed. For supported versions, see the [support matrix]({{ 'support/matrix/#event-streams' | relative_url }}). +- The Kubernetes command-line tool (`kubectl`) [installed](https://kubernetes.io/docs/tasks/tools/){:target="_blank"}. +- The Helm command-line tool (`helm`) [installed](https://helm.sh/docs/intro/install/). +- Skopeo [installed](https://github.com/containers/skopeo/blob/main/install.md) to move images from one repository to another. + + +## Prepare your host + +You must be able to connect your bastion host to the internet and to the restricted network environment (with access to the {{site.data.reuse.openshift_short}} cluster and the local registry) at the same time. + +Ensure that the prerequisites are set up and that the bastion host can access: + +- The public internet to download the CASE and images. +- The target (internal) image registry where all the images will be mirrored to. +- The OpenShift or other Kubernetes cluster to install the operator on. + +**Note:** In the absence of a bastion host, prepare a portable device with public internet access to download the CASE and images and a target registry where the images will be mirrored. + +**Important:** Ensure you have access to the Kubernetes cluster by running the `kubectl get namespaces` command which lists all the available namespaces. + +## Download the CASE bundle + +Before mirroring your images, set the environment variables for the CASE images on your host, and then download the CASE by following these instructions: + +1. Configure the GitHub repository to download the CASE: + + ```shell + kubectl ibm-pak config repo 'default' -r "https://github.com/IBM/cloud-pak/raw/master/repo/case/" --enable + ``` + +2. Run the following command to download, validate, and extract the {{site.data.reuse.es_name}} CASE. + + ```shell + kubectl ibm-pak get ibm-eventstreams --version + ``` + + Where `` is the version of the CASE file to be downloaded. + + The CASE is downloaded in `~/.ibm-pak` and the following output is displayed: + + ```shell + Downloading and extracting the CASE ... + - Success + Retrieving CASE version ... + - Success + Validating the CASE ... + Validating the signature for the ibm-eventstreams CASE... + - Success + Creating inventory ... + - Success + Finding inventory items + - Success + Resolving inventory items ... + Parsing inventory items + - Success + Download of CASE: ibm-eventstreams, version: 3.5.0 is complete + ``` + + **Note:** To download the latest version of CASE, do not specify the CASE version. For example: + + ```shell + oc ibm-pak get ibm-eventstreams + ``` + +3. Verify that the CASE and images (`.csv`) files have been generated for {{site.data.reuse.es_name}}. + + For example, ensure that the following files have been generated for {{site.data.reuse.es_name}}. + + ```shell + tree ~/.ibm-pak + + ├── config + │ └── config.yaml + ├── data + │ ├── cases + │ │ └── ibm-eventstreams + │ │ └── 3.5.0 + │ │ ├── caseDependencyMapping.csv + │ │ ├── charts + │ │ ├── ibm-eventstreams-3.5.0-airgap-metadata.yaml + │ │ ├── ibm-eventstreams-3.5.0-charts.csv + │ │ ├── ibm-eventstreams-3.5.0-images.csv + │ │ ├── ibm-eventstreams-3.5.0.tgz + │ │ └── resourceIndexes + │ │ └── ibm-eventstreams-resourcesIndex.yaml + │ └── mirror + └── logs + └── oc-ibm_pak.log + + 9 directories, 8 files + ``` + + +## Configure registry authentication + + + +To mirror images across both the source registry and the target (internal) registry where all images are available publicly, you must create an authentication secret for each. You need a Docker CLI login (`docker login`) or Podman CLI login (`podman login`) for configuring the registry. + +A Skopeo CLI login (`skopeo login`) is also required for Kubernetes platforms other than OpenShift. + +For {{site.data.reuse.es_name}}, all images are either present in the IBM Entitled Registry (`cp.icr.io`), which requires authentication, or in the IBM Container Registry (`icr.io/cpopen`), which does not. + +### Creating an authentication secret for the source registry + +Run the following command to create an authentication secret for the source registry: + +```shell +docker login --username --password +``` + +Where: + +- `` is the Entitled Registry (`cp.icr.io`). +- `` is your username. +- `` is your entitlement key. + +Additionally, if you are installing on Kubernetes platforms other than OpenShift, run the following command: + + +```shell +skopeo login -u -p +``` + +### Creating an authentication secret for the target registry + +Run the following command to create an authentication secret for the target registry: + +```shell +docker login --username --password +``` + +Additionally, if you are running on Kubernetes platforms other than OpenShift, run the following command: + +```shell +skopeo login -u -p +``` + +Where: + +- `target-registry` is the internal container image registry. +- `target-registry-user` is the username for the internal container image registry. +- `target-registry-pass` is the password for the internal container image registry. + + +## Mirror the images + + + +The process of mirroring images pulls the image from the internet and pushes it to your local registry. After mirroring your images, you can configure your cluster and complete the offline installation. + +Complete the following steps to mirror the images from your host to your offline environment: + +1. Run the following command to generate mirror manifests: + + ```shell + kubectl ibm-pak generate mirror-manifests ibm-eventstreams + ``` + + Where `target-registry` is the internal container image registry. + + **Note**: If you need to filter for a specific image group, add the parameter `--filter ` to this command. + + This generates the following files based on the target (internal) registry provided: + + - catalog-sources.yaml + - catalog-sources-linux-``.yaml (if there are architecture specific catalog sources) + - image-content-source-policy.yaml + - images-mapping.txt + +2. Run the following command to copy the images to the local registry. Your device must be connected to both the internet and the restricted network environment that contains the local registry. + + If you are installing on the {{site.data.reuse.openshift_short}}, run the following command: + + ```shell + oc image mirror -f ~/.ibm-pak/data/mirror/ibm-eventstreams//images-mapping.txt --filter-by-os '.*' --insecure --skip-multiple-scopes --max-per-registry=1 + ``` + + If you are installing on Kubernetes platforms other than OpenShift, run the following command: + + ```shell + cat ~/.ibm-pak/data/mirror/ibm-eventstreams//images-mapping.txt | awk -F'=' '{ print "skopeo copy --all docker://"$1" docker://"$2 }' | xargs -I {} sh -c 'echo {}; {}' + ``` + + **Note:** If you are using a macOS system and encounter the `xargs: command line cannot be assembled, too long` error, add `-S1024` to `xargs`, and run the command as follows: + + ```shell + cat ~/.ibm-pak/data/mirror/ibm-eventstreams//images-mapping.txt | awk -F'=' '{ print "skopeo copy --all docker://"$1" docker://"$2 }' | xargs -S1024 -I {} sh -c 'echo {}; {}' + ``` + + Where: + + + - `` is the version of the CASE file to be copied. + - `target-registry` is the internal container image registry. + +Ensure that all the images have been mirrored to the target registry by checking the registry. + +## Create `ImageContentSourcePolicy` on OpenShift platform + + +**Note:** Only applicable when installing {{site.data.reuse.es_name}} on the {{site.data.reuse.openshift_short}}. + + + +1. {{site.data.reuse.openshift_cli_login}} +2. Update the global image pull secret for your OpenShift cluster by following the steps in [OpenShift documentation](https://docs.openshift.com/container-platform/4.16/openshift_images/managing_images/using-image-pull-secrets.html#images-update-global-pull-secret_using-image-pull-secrets){:target="_blank"}. This enables your cluster to have proper authentication credentials to pull images from your `target-registry`, as specified in the `image-content-source-policy.yaml`. +3. Apply `ImageContentSourcePolicy` YAML by running the following command: + + ```shell + oc apply -f ~/.ibm-pak/data/mirror/ibm-eventstreams//image-content-source-policy.yaml + ``` + + Where `` is the version of the CASE file. + +4. Additionally, a global image pull secret must be added so that images can be pulled from the target registry. Follow the instructions in the [OpenShift documentation](https://github.com/openshift/openshift-docs/blob/main/modules/images-update-global-pull-secret.adoc#updating-the-global-cluster-pull-secret){:target="_blank"} to add credentials for the target registry. + + **Important:** + Cluster resources must adjust to the new pull secret, which can temporarily limit the access to the cluster. Applying the `ImageSourceContentPolicy` causes cluster nodes to recycle, which results in limited access to the cluster until all the nodes are ready. + +5. Verify that the `ImageContentSourcePolicy` resource is created: + + ```shell + oc get imageContentSourcePolicy + ``` + + **Important**: After the `ImageContentsourcePolicy` and global image pull secret are applied, you might see the node status as `Ready`, `Scheduling`, or `Disabled`. Wait until all the nodes show a `Ready` status. + +6. Verify your cluster node status and wait for all nodes to be updated before proceeding: + + ```shell + oc get MachineConfigPool -w + ``` + +## Apply catalog sources to your cluster on OpenShift platform + +**Note:** Only applicable when installing {{site.data.reuse.es_name}} on the {{site.data.reuse.openshift_short}}. + +Apply the catalog sources for the operator to the cluster by running the following command: + +```shell +oc apply -f ~/.ibm-pak/data/mirror/ibm-eventstreams//catalog-sources.yaml +``` + +Where `` is the case version. + +## Install the operator + +If you are installing the operator for the first time, complete the instructions in the following sections to install it on the platform that you are using. + +If you are upgrading an existing offline installation, follow the [upgrading](../upgrading) instructions to upgrade your operator to the version that you [downloaded](#download-the-case-bundle) and [mirrored](#mirror-the-images) earlier. + +### Installing on OpenShift + +After considering the operator requirements, resource requirements, and cluster-scoped permissions, you can install the operator by using the {{site.data.reuse.openshift_short}} web console or command line. For more information, see the instructions for installing the [{{site.data.reuse.es_name}} operator](../installing/#install-the-event-streams-operator). + + +### Installing on other Kubernetes platforms by using the `kubectl` + +Complete the following steps to install the operator: + +1. Create a namespace where you want to install the operator: + + ```shell + kubectl create namespace + ``` + +2. Create an image pull secret called `ibm-entitlement-key` in the namespace where you want to install the {{site.data.reuse.es_name}} operator. The secret enables container images to be pulled from the target registry: + + ```shell + kubectl create secret docker-registry ibm-entitlement-key --docker-username="" --docker-password="" --docker-server="" -n + ``` + +3. Install the operator by using the Helm CLI: + + ```shell + helm install ~/.ibm-pak/data/cases/ibm-eventstreams//charts/ibm-eventstreams-operator-.tgz -n --set imagePullPolicy="Always" --set public.repo= --set public.path="cpopen/" --set private.repo= --set private.path="cp/" --set watchAnyNamespace= + ``` + + Where: + - `` is the name you provide to identify your operator. + - `` is the namespace where we want to install IBM Event Streams. + - `` is the CASE version. + - `` is the internal container image registry. + - `` is the username for the internal container image registry. + - `` is the password for the internal container image registry. + +4. Wait for the installation to complete. +5. You can now verify your installation and consider other [post-installation tasks](../post-installation/). + +## Install an instance + +{{site.data.reuse.es_name}} instances can be created after the operators are installed. You can install the instances by using the {{site.data.reuse.openshift_short}} web console. For more information, see the instructions for installing the {{site.data.reuse.es_name}} instances on [OpenShift](../installing/#install-an-event-streams-instance), or on other [Kubernetes platforms](../instaling-on-kubernetes/#install-an-event-streams-instance). diff --git a/_es_11.5/02-installing/06b-installing-on-kubernetes.md b/_es_11.5/02-installing/06b-installing-on-kubernetes.md new file mode 100644 index 00000000..a0b4e7d9 --- /dev/null +++ b/_es_11.5/02-installing/06b-installing-on-kubernetes.md @@ -0,0 +1,205 @@ +--- +title: "Installing on other Kubernetes platforms" +excerpt: "Find out how to install Event Streams on other Kubernetes platforms." +categories: installing +slug: installing-on-kubernetes +toc: true +--- + +The following sections provide instructions about installing {{site.data.reuse.es_name}} on Kubernetes platforms that support the Red Hat Universal Base Images (UBI) containers. + +## Overview + +{{site.data.reuse.es_name}} is an [operator-based](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/){:target="_blank"} release and uses custom resources to define your {{site.data.reuse.es_name}} configurations. The {{site.data.reuse.es_name}} operator uses the custom resources to deploy and manage the entire lifecycle of your {{site.data.reuse.es_name}} instances. Custom resources are presented as YAML configuration documents that define instances of the `EventStreams` custom resource type. + +When deploying in an air-gapped (also referred to as offline or disconnected) environment, follow the instructions in the [offline installation](../offline) to install {{site.data.reuse.es_name}} on OpenShift and other Kubernetes platforms by using the CASE package. + + +Installing {{site.data.reuse.es_name}} has two phases: + +1. Use Helm to install the {{site.data.reuse.es_name}} operator: this will deploy the operator that will install and manage your {{site.data.reuse.es_name}} instances. +2. Install one or more instances of {{site.data.reuse.es_name}} by using the operator. + +## Before you begin + +- Ensure you have set up your environment [according to the prerequisites](../prerequisites). +- Ensure you have [planned for your installation](../planning), such as preparing for persistent storage, considering security options, and considering adding resilience through multiple availability zones. +- Obtain the connection details for your Kubernetes cluster from your administrator. + +## Create a namespace + +Create a namespace into which the {{site.data.reuse.es_name}} instance will be installed. For more information about namespaces, see the [Kubernetes documentation](https://kubernetes.io/docs/tasks/administer-cluster/namespaces/){:target="_blank"}. + +Ensure you use a namespace that is dedicated to a single instance of {{site.data.reuse.es_name}}. This is required because {{site.data.reuse.es_name}} uses network security policies to restrict network connections between its internal components. A single namespace per instance also allows for finer control of user accesses. + +**Important:** Do not use any of the initial or system [namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/#initial-namespaces){:target="_blank"} to install an instance of {{site.data.reuse.es_name}} (`default`, `kube-node-lease`, `kube-public`, and `kube-system`). + + +## Add the Helm repository + +Before you can install the {{site.data.reuse.es_name}} operator and use it to create instances of {{site.data.reuse.es_name}}, add the IBM Helm repository to your local repository list. This will provide access to the {{site.data.reuse.es_name}} Helm chart package that will install the operator on your cluster. + +To add the [IBM Helm repository](https://github.com/IBM/charts/tree/master/repo/ibm-helm){:target="_blank"} to the local repository list, run the following command: + +```shell +helm repo add ibm-helm https://raw.githubusercontent.com/IBM/charts/master/repo/ibm-helm +``` + +## Install the {{site.data.reuse.es_name}} operator + +Ensure you have considered the {{site.data.reuse.es_name}} operator [requirements](../prerequisites/#operator-requirements), including resource requirements and the required cluster-scoped permissions. + +### Choosing operator installation mode + +Before installing the {{site.data.reuse.es_name}} operator, decide if you want the operator to: + +- Manage instances of {{site.data.reuse.es_name}} in **any namespace**. + + To use this option, set `watchAnyNamespace: true` when installing the operator. The operator will be deployed into the specified namespace, and will be able to manage instances of {{site.data.reuse.es_name}} in any namespace. + +- Only manage instances of {{site.data.reuse.es_name}} in a **single namespace**. + + This is the default option: if `watchAnyNamespace` is not set, then it defaults `false`. The operator will be deployed into the specified namespace, and will only be able to manage instances of {{site.data.reuse.es_name}} in that namespace. + + **Important:** When multiple {{site.data.reuse.es_name}} operators are installed on the same cluster, all the operators share the same custom resource definitions (CRDs). Do not install a previous version of the {{site.data.reuse.es_name}} operator if a later version is already installed on the same cluster. + +### Installing the operator + +To install the operator, run the following command: + +```shell +helm install \ + ibm-helm/ibm-eventstreams-operator \ + -n \ + --set watchAnyNamespace= +``` + +Where: +- `` is the name you provide to identify your operator. +- `` is the name of the namespace where you want to install the operator. +- `watchAnyNamespace=` determines whether the operator manages instances of {{site.data.reuse.es_name}} in any namespace or only a single namespace (default is `false` if not specified). + + Set to `true` for the operator to manage instances in any namespace, or do not specify if you want the operator to only manage instances in a single namespace. + +For example, to install the operator on a cluster where it will manage all instances of {{site.data.reuse.es_name}}, run the command as follows: + +```shell +helm install eventstreams ibm-helm/ibm-eventstreams-operator -n "my-namespace" --set watchAnyNamespace=true +``` + +For example, to install the operator that will manage {{site.data.reuse.es_name}} instances in only the `my-eventstreams` namespace, run the command as follows: + +```shell +helm install eventstreams ibm-helm/ibm-eventstreams-operator -n "my-eventstreams" +``` + +**Note:** If you are installing any subsequent operators in the same cluster, ensure you run the `helm install` command with the `--set createGlobalResources=false` option (as these resources have already been installed). + +#### Checking the operator status + +To check the status of the installed operator, run the following command: + +```shell +kubectl get deploy eventstreams-cluster-operator -n +``` +Where: +- `` is the name of the namespace where the operator is installed. + +A successful installation will return a result similar to the following with `1/1` in the `READY` column: + +```shell +NAME READY UP-TO-DATE AVAILABLE AGE +eventstreams-cluster-operator 1/1 1 1 7d4h +``` + +### Technology Preview feature: KRaft + +Technology Preview features are available to evaluate potential upcoming features. Such features are intended for testing purposes only and not for production use. IBM does not support these features, but might help with any issues raised against them. IBM welcomes feedback on Technology Preview features to improve them. As the features are still under development, functions and interfaces can change, and it might not be possible to upgrade when updated versions become available. + +IBM offers no guarantee that Technology Preview features will be part of upcoming releases and as such become fully supported. + +{{site.data.reuse.es_name}} version 11.1.5 and later includes [Apache Kafka Raft (KRaft)](https://cwiki.apache.org/confluence/display/KAFKA/KIP-500%3A+Replace+ZooKeeper+with+a+Self-Managed+Metadata+Quorum){:target="_blank"} as a Technology Preview feature. +KRaft replaces ZooKeeper for managing metadata, moving the overall handling of metadata into Kafka itself. + + + +#### Limitations + +The KRaft mode in {{site.data.reuse.es_name}} has the following limitations: + +- Moving existing Kafka clusters deployed with ZooKeeper to use KRaft, or the other way around, is not supported. +- Upgrading your Apache Kafka or {{site.data.reuse.es_name}} operator version, or reverting either one to an earlier version is not supported. To do so, you delete the cluster, upgrade the operator, and deploy a new Kafka cluster. +- The Topic Operator is not supported. The `spec.entityOperator.topicOperator` property must be removed from the Kafka custom resource. +- JBOD storage is not supported. You can use `type: jbod` for storage, but the JBOD array can contain only one disk. +- Geo-replication is not supported. + +#### Enabling KRaft + +To enable KRaft, add the following annotations to the `EventStreams` custom resource: +- `eventstreams.ibm.com/kraft: enabled` +- `eventstreams.ibm.com/node-pools: enabled` + +For example: + +```yaml +- metadata: + name: light-insecure + namespace: openshift-operators + annotations: + eventstreams.ibm.com/node-pools: enabled + eventstreams.ibm.com/kraft: enabled +``` + +## Install an {{site.data.reuse.es_name}} instance + +Instances of {{site.data.reuse.es_name}} can be created after the {{site.data.reuse.es_name}} operator is installed. If the operator was installed to manage **a specific namespace**, then it can only be used to manage instances of {{site.data.reuse.es_name}} in that namespace. If the operator was installed to manage **all namespaces**, then it can be used to manage instances of {{site.data.reuse.es_name}} in any namespace, including those created after the operator was deployed. + +When installing an instance of {{site.data.reuse.es_name}}, ensure you are using a namespace that an operator is managing. + +### Creating an image pull secret + +Before installing an {{site.data.reuse.es_name}} instance, create an image pull secret called `ibm-entitlement-key` in the namespace where you want to create an instance of {{site.data.reuse.es_name}}. The secret enables container images to be pulled from the registry. + +1. Obtain an entitlement key from the [IBM Container software library](https://myibm.ibm.com/products-services/containerlibrary){:target="_blank"}. +2. Click **Entitlement keys** in the navigation on the left, and click **Add new key**, or if you have an existing active key available, click **Copy** to copy the entitlement key to the clipboard. +3. Create the secret in the namespace that will be used to deploy an instance of {{site.data.reuse.es_name}} as follows. + + Name the secret `ibm-entitlement-key`, use `cp` as the username, your entitlement key as the password, and `cp.icr.io` as the docker server: + + ```shell + kubectl create secret docker-registry ibm-entitlement-key --docker-username=cp --docker-password="" --docker-server="cp.icr.io" -n "" + ``` + + +**Note:** If you do not create the required secret, pods will fail to start with `ImagePullBackOff` errors. In this case, ensure the secret is created and allow the pod to restart. + +### Installing an instance by using the CLI + +To install an instance of {{site.data.reuse.es_name}} from the command line, you must first prepare an `EventStreams` custom resource configuration in a YAML file. + +A number of sample configuration files are included in the Helm chart package to base your deployment on. The sample configurations range from smaller deployments for non-production development or general experimentation to large scale clusters ready to handle a production workload. + +The sample configurations are also available in [GitHub](https://ibm.biz/ea-es-samples){:target="_blank"}, where you can select the GitHub tag for your {{site.data.reuse.es_name}} version to access the correct samples, and then go to `/cr-examples/eventstreams/kubernetes` to access the samples for the Kubernetes platforms that support the Red Hat Universal Base Images (UBI) containers. + +More information about these samples is available in the [planning](../planning/#sample-deployments) section. You can base your deployment on the sample that most closely reflects your requirements and apply [customizations](../configuring) on top as required. + +**Important:** Ensure that the `spec.license.accept` field in the custom resource YAML is set to `true`, and that the correct values are selected for the `spec.license.license` and `spec.license.use` fields before deploying the {{site.data.reuse.es_name}} instance. These values are used for metering purposes and could result in inaccurate charging and auditing if set incorrectly. For more information about the available options, see the [licensing reference]({{ 'support/licensing' | relative_url }}). + +**Note:** If experimenting with {{site.data.reuse.es_name}} for the first time, the **Lightweight without security** sample is the smallest and simplest example that can be used to create an experimental deployment. For the smallest production setup, use the **Minimal production** sample configuration. + +To deploy an {{site.data.reuse.es_name}} instance, run the following commands: + +1. Apply the configured `EventStreams` custom resource in the selected namespace: + + ```shell + kubectl apply -f -n "" + ``` + + For example: + + ```shell + kubectl apply -f development.yaml -n "my-eventstreams" + ``` + +2. Wait for the installation to complete. +3. Verify your installation and consider other [post-installation tasks](../post-installation/). diff --git a/_es_11.5/02-installing/07-configuring.md b/_es_11.5/02-installing/07-configuring.md new file mode 100644 index 00000000..8966780b --- /dev/null +++ b/_es_11.5/02-installing/07-configuring.md @@ -0,0 +1,1467 @@ +--- +title: "Configuring" +excerpt: "Configure your Event Streams installation." +categories: installing +slug: configuring +toc: true +--- + +{{site.data.reuse.es_name}} provides samples to help you get started with deployments, as described in the [planning](../planning/#sample-deployments) section. Select one of the samples suited to your requirements to get started: + +- Lightweight without security +- Development +- Minimal production +- Production 3 brokers +- Production 6 brokers +- Production 9 brokers + +You can modify the samples, save them, and apply custom configuration settings as well. See the following sections for guidance about configuring your instance of {{site.data.reuse.es_name}}. + +On OpenShift, you can configure and apply them by using the [command line](../installing/#installing-an-instance-by-using-the-cli) or by dragging and dropping them onto the {{site.data.reuse.openshift_short}} [web console](../installing/#installing-by-using-the-yaml-view), and editing them. + +**Note:** When applying custom Kafka configuration settings to your {{site.data.reuse.es_name}}, check the [Kafka documentation](https://kafka.apache.org/37/documentation){:target="_blank"} to ensure the new configuration settings are consistent and do not cause conflicts. + +## Checking configuration settings + +This page gives information about many configuration options. To see further information about specific configuration options, or to see what options are available, you can use the `kubectl explain` command. To see information about a specific field, run the following: + +```shell +kubectl explain eventstreams. +``` + +Where `path-of-field` is the JSON path of the field of interest. + +For example, if you want to see more information about configuring external listeners for Kafka you can run the following command: + +```shell +kubectl explain eventstreams.spec.strimziOverrides.kafka.listeners.external +``` + +## Enabling persistent storage + +If you want your data to be preserved in the event of a restart, configure persistent storage for Kafka and ZooKeeper in your {{site.data.reuse.es_name}} installation. + +**Note:** Ensure you have sufficient [disk space](../capacity-planning/#disk-space-for-persistent-volumes) for persistent storage. + +These settings are specified in the YAML configuration document that defines an instance of the `EventStreams` custom resource and can be applied when defining a new {{site.data.reuse.es_name}} instance under the "Event Streams" operator in the {{site.data.reuse.openshift_short}} web console. + +- To enable persistent storage for Kafka, add the `storage` property under `spec.strimziOverrides.kafka` +- To enable persistent storage for ZooKeeper, add the `storage` property under `spec.strimziOverrides.zookeeper` + +Complete the configuration by adding additional fields to these storage properties as follows: + +1. Specify the storage type in `storage.type` (for example, `"ephemeral"` or `"persistent-claim"`). + + **Note:** When using ephemeral storage, ensure you set retention limits for Kafka topics so that you do not run out of disk space. + If [message retention](../../getting-started/creating-topics/) is set to long periods and the message volume is high, the storage requirements for the topics could impact the OpenShift nodes that host the Kafka pods, and cause the nodes to run out of allocated disk space, which could impact normal operation. + +2. Specify the storage size in `storage.size` (for example, `"100Gi"`). +3. Optionally, specify the storage class in `storage.class` (for example, `"rook-ceph-block-internal"`). +4. Optionally, specify the retention setting for the storage if the cluster is deleted in `storage.deleteClaim` (for example, `"true"`). + +An example of these configuration options: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + strimziOverrides: + kafka: + # ... + storage: + type: "persistent-claim" + size: "100Gi" + class: "ceph-block" + zookeeper: + # ... + storage: + type: "persistent-claim" + size: "100Gi" + class: "ceph-block" +# ... +``` + +If present, existing persistent volumes with the specified storage class are used after installation, or if a [dynamic provisioner](https://docs.openshift.com/container-platform/4.16/storage/dynamic-provisioning.html){:target="_blank"} is configured for the specified storage class, new persistent volumes are created. + +Where optional values are not specified: + +- If no storage class is specified and a default storage class has been defined in the {{site.data.reuse.openshift_short}} settings, the default storage class will be used. +- If no storage class is specified and no default storage class has been defined in the {{site.data.reuse.openshift_short}} settings, the deployment will use any [persistent volume claims](https://kubernetes.io/docs/concepts/storage/persistent-volumes/){:target="_blank"} that have at least the set size value. + + **Note:** An empty string is not the same as not specifying a value for a field. If you include the `class` field, the field value must be a valid storage class, it cannot be an empty string. An empty string will not be accepted by the operator. + +- If no retention setting is provided, the storage will be retained when the cluster is deleted. + + +The following example YAML document shows an example `EventStreams` custom resource with dynamically allocated storage provided using CephFS for Kafka and ZooKeeper. To try this deployment, set the required `namespace` and accept the license by changing the `spec.license.accept` value to `"true"`. + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +metadata: + name: example-storage + namespace: myproject + labels: + backup.eventstreams.ibm.com/component: eventstreams +spec: + license: + accept: false + version: 11.1.0 + adminApi: {} + adminUI: {} + apicurioRegistry: {} + collector: {} + restProducer: {} + strimziOverrides: + kafka: + replicas: 1 + config: + offsets.topic.replication.factor: 1 + transaction.state.log.min.isr: 1 + transaction.state.log.replication.factor: 1 + listeners: + - name: external + type: route + port: 9092 + authentication: + type: scram-sha-512 + tls: true + - name: plain + port: 9093 + type: internal + tls: false + - name: tls + port: 9094 + type: internal + tls: true + authentication: + type: tls + storage: + type: persistent-claim + size: 100Gi + class: rook-ceph-block-internal + deleteClaim: true + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + key: kafka-metrics-config.yaml + name: metrics-config + zookeeper: + replicas: 1 + storage: + type: persistent-claim + size: 100Gi + class: rook-ceph-block-internal + deleteClaim: true + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + key: zookeeper-metrics-config.yaml + name: metrics-config +``` + +## Configuring encryption between pods + +Pod-to-Pod encryption is enabled by default for all {{site.data.reuse.es_name}} pods. Unless explicitly overridden in an `EventStreams` custom resource, the configuration option `spec.security.internalTls` will be set to `TLSv1.2`. This value can be set to `NONE` which will disable Pod-to-Pod encryption. + +For example, the following YAML snippet disables encryption between pods: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + security: + # ... + internalTls: NONE +# ... +``` + +## Configuring UI and CLI security + +By default, accessing the {{site.data.reuse.es_name}} UI and CLI requires a user that has been assigned access to {{site.data.reuse.es_name}} (see [managing access](../../security/managing-access/#accessing-the-event-streams-ui-and-cli) for details). + + + +For accessing both the UI and CLI, {{site.data.reuse.es_name}} supports the Salted Challenge Response Authentication Mechanism (SCRAM) and the {{site.data.reuse.icpfs}} Identity and Access Management (IAM). Additionally, {{site.data.reuse.es_name}} also supports Keycloak for accessing the UI. + +{{site.data.reuse.iam_note}} + +You can choose to change the authentication type of the UI access from IAM to SCRAM-SHA-512 by setting the authentication type in the `EventStreams` custom resource as follows: + +- If no authentication type is provided, IAM is the default authentication type. You can also specifically configure IAM by setting the `adminUI` authentication type to `iam` in the `EventStreams` custom resource as follows: + + ```yaml + # ... + spec: + # ... + adminUI: + authentication: + - type: iam + ``` + +- You can change the authentication type from the default IAM to SCRAM by setting the `adminUI` authentication type to `scram-sha-512` in the `EventStreams` custom resource as follows: + + ```yaml + # ... + spec: + # ... + adminUI: + authentication: + - type: scram-sha-512 + ``` + + This allows a Kafka user that has been configured for SCRAM authentication to log in to the {{site.data.reuse.es_name}} UI and CLI by using the username and password of that Kafka user. For more information about configuring Kafka users, see [managing access to Kafka resources](../../security/managing-access/#managing-access-to-kafka-resources). + + When using SCRAM, the Access Control List (ACL) that is configured for the user will determine which parts of the UI are available to the user to access and what CLI commands the user can run. For more information, see [permission mappings](../../security/managing-access/#managing-access-to-the-ui-and-cli-with-scram). + +- To change the authentication type to Keycloak, set the `adminUI` authentication type to `integrationKeycloak` in the `EventStreams` custom resource as follows: + + ```yaml + # ... + spec: + # ... + adminUI: + authentication: + - type: integrationKeycloak + ``` + + **Note:** Authentication through Keycloak is not supported in the {{site.data.reuse.es_name}} CLI. You can authenticate the {{site.data.reuse.es_name}} CLI with the SCRAM authentication and then proceed to use an {{site.data.reuse.es_name}} instance that is configured with Keycloak. For Keycloak requirements and limitations, see the [prerequisites](../../installing/prerequisites/#prereqs-keycloak). + +- The login requirement for the UI is disabled when all Kafka authentication and authorization is disabled. This is demonstrated by the proof-of-concept [**lightweight without security**](../planning/#example-deployment-lightweight-without-security) sample. + + **Important:** When security is not configured, the **[Producers](../../administering/topic-health/)** and the **[Monitoring](../../administering/cluster-health/#viewing-the-preconfigured-dashboard)** dashboards are not available in the UI. + + +**Note:** The {{site.data.reuse.es_name}} UI and CLI only support the use of one authentication type for a single {{site.data.reuse.es_name}} instance. This means that you can only set one authentication type at the same time. The operator will issue an error message if more than one type is provided. + + +## Applying Kafka broker configuration settings + +Kafka supports a number of [broker configuration settings](http://kafka.apache.org/37/documentation/#brokerconfigs){:target="_blank"}, typically provided in a properties file. + +When creating an instance of {{site.data.reuse.es_name}}, these settings are defined in an `EventStreams` custom resource under a the `spec.strimziOverrides.kafka.config` property. + +{{site.data.reuse.es_name}} uses `KafkaTopic` custom resources to manage topics in Kafka. Set `auto.create.topics.enable: false` to ensure all Kafka topics are represented as a `KafkaTopic` resource. + +The following example uses Kafka broker settings to configure replication for system topics: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + # ... + config: + offsets.topic.replication.factor: 1 + transaction.state.log.min.isr: 1 + transaction.state.log.replication.factor: 1 + auto.create.topics.enable: 'false' +``` + +This custom resource can be created using the `kubectl` command or the {{site.data.reuse.openshift_short}} web console under the **Event Streams** operator page. + +You can specify all the broker configuration options supported by Kafka except those managed directly by {{site.data.reuse.es_name}}. For further information, see the list of [supported configuration options](https://strimzi.io/docs/operators/0.42.0/configuring.html#type-KafkaClusterSpec-reference){:target="_blank"}. + +After deployment, these settings can be [modified](../../administering/modifying-installation/#modifying-kafka-broker-configuration-settings) by updating the `EventStreams` custom resource. + +## Applying Kafka rack awareness + +Kafka rack awareness is configured by setting the `rack` property in the `EventStreams` custom resource using the zone label as the topology key in the `spec.strimziOverrides.kafka.rack` field. This key needs to match the [zone label](../preparing-multizone/#zone-awareness) name applied to the nodes. + +**Note:** Before this is applied, ensure the [Kafka cluster role](../preparing-multizone/#kafka-rack-awareness) for rack awareness has been applied. + +The following example sets the `rack` topologyKey to `topology.kubernetes.io/zone`: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + # ... + rack: + topologyKey: topology.kubernetes.io/zone + # ... +``` + +## Setting geo-replication nodes + +You can install geo-replication in a cluster to enable messages to be automatically synchronized between local and remote topics. A cluster can be a geo-replication origin or destination. Origin clusters send messages to a remote system, while destination clusters receive messages from a remote system. A cluster can be both an origin and a destination cluster at the same time. + +To enable geo-replication, create an `EventStreamsGeoReplicator` custom resource alongside the `EventStreams` custom resource. This can be defined in a YAML configuration document under the **Event Streams** operator in the {{site.data.reuse.openshift_short}} web console. + +When setting up geo-replication, consider the [number of geo-replication worker nodes (replicas)](../../georeplication/planning/#preparing-a-destination-cluster) to deploy and configure this in the `spec.replicas` property. + +Ensure that the following properties match the name of the {{site.data.reuse.es_name}} instance: + +- `metadata.name` +- `metadata.labels["eventstreams.ibm.com/cluster"]` + +For example, to configure geo-replication with `2` replicas for an {{site.data.reuse.es_name}} instance called `sample-three` in the namespace `myproject`, create the following `EventStreamsGeoReplicator` configuration: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta1 +kind: EventStreamsGeoReplicator +metadata: + labels: + eventstreams.ibm.com/cluster: sample-three + name: sample-three + namespace: myproject +spec: + # ... + replicas: 2 +``` + +**Note:** Geo-replication can be deployed or reconfigured at any time. For more information, see [Setting up geo-replication](../../georeplication/setting-up/). + +## Configuring access + +You can configure external access to the following services representing {{site.data.reuse.es_name}} components: + +- **The {{site.data.reuse.es_name}} UI:** A graphical user interface you can log in to from a web browser to access and administer your {{site.data.reuse.es_name}} instance. The UI requires the Admin API service to be enabled. +- **The Admin API:** A RESTful administration API that can be used for managing functions of your {{site.data.reuse.es_name}} instance (for example, creating topics). +- **The Apicurio Registry:** The schema registry in {{site.data.reuse.es_name}} to store message schemas. Disable if you do not require schemas for your messages. +- **The REST Producer:** The REST producer API is a scalable REST interface for producing messages to {{site.data.reuse.es_name}} over a secure HTTP endpoint. + +On the {{site.data.reuse.openshift_short}}, external access to these services are automatically configured by using routes (unless they are not included in the {{site.data.reuse.es_name}} installation). + +On other Kubernetes platforms, all services have internal endpoints configured, creating access to them from within the cluster. To expose any service outside your cluster, configure external access by setting up a Kubernetes ingress controller and configuring the `EventStreams` custom resource to use ingress. + +**Note:** When using ingress, ensure you install and run an [ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/){:target="_blank"} on your Kubernetes platform. The SSL passthrough must be enabled in the ingress controller and ingresses for your {{site.data.reuse.es_name}} services to work. Refer to your ingress controller documentation for more information. + +The following sections provide more information and examples about setting up external access to these {{site.data.reuse.es_name}} services and Kafka. + +### REST services access + +The REST services for {{site.data.reuse.es_name}} are configured with defaults for the container port, type, TLS version, certificates, and authentication mechanisms. If the [Kafka listeners](#kafka-access) have been configured without authentication requirements, then the authentication mechanisms are automatically removed from the REST endpoints. + +The schema for REST endpoint configuration is described in the following table, followed by an example of an endpoint configuration for the Admin API that uses routes, and then an ingress example. In these examples, the potential values for `` in `spec..endpoints` are: + +- `adminApi` for the Admin API +- `restProducer` for the REST Producer +- `apicurioRegistry` for the Apicurio Registry + +**Note:** If you are overriding the certificates for any of the REST endpoints (`certOverrides`) with your own certificates, you can [provide the external CA certificate](#providing-external-ca-certificates) that signed them in a secret. If you provide the external CA certificates, they will be included in the truststore you can download from the [{{site.data.reuse.es_name}} UI](../../getting-started/connecting/#obtaining-the-server-side-public-certificate-from-the-event-streams-ui) and [CLI](../../getting-started/connecting/#obtaining-the-server-side-public-certificate-from-the-event-streams-cli). + +| Key | Type | Description | +| :-------------------------- | :--------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | String | Name to uniquely identify the endpoint among other endpoints in the list for a component. | +| `containerPort` | Integer | A unique port to open on the container that this endpoint will serve requests on. Restricted ranges are 0-1000 and 7000-7999. | +| `type` | String [`internal`, `route`, `ingress`] | {{site.data.reuse.es_name}} REST components support internal, route, or ingress type endpoints. | +| `tlsVersion` | String [`TLSv1.2`,`TLSv1.3`,`NONE`] | Specifies the TLS version where `NONE` disables HTTPS. The default is `TLSv1.2`. | +| `authenticationMechanisms` | List of Strings | List of authentication mechanisms to be supported at this endpoint. By default, all authentication mechanisms that are defined in the [Kafka listeners](#kafka-access) are enabled. Optionally, you can configure a subset, or none (`[]`).
**Note:** `iam-bearer` is only supported when you have {{site.data.reuse.icpfs}} installed on the {{site.data.reuse.openshift_short}}. | +| `certOverrides.certificate` | String | The name of the key in the provided `certOverrides.secretName` secret that contains the Base64-encoded certificate. | +| `certOverrides.key` | String | The name of the key in the provided `certOverrides.secretName` secret that contains the Base64-encoded key. | +| `certOverrides.secretName` | String | The name of the secret in the instance namespace that contains the encoded certificate and key to secure the endpoint with. | +| `host` | String (DNS rules apply) | The hostname for accessing the service. This is required for ingress. On the {{site.data.reuse.openshift_short}}, this is the default hostname that an OpenShift route generates, and you can override the default hostname by providing a different one here. | +| `class` | String | Identifies the ingress controller when you have more than one controller installed. | +| `annotations` | String | Optionally, you can use annotations to configure different options for an ingress resource. | + +**Note:** Each authentication mechanism you want to use for the {{site.data.reuse.es_name}} REST endpoints must have a corresponding [Kafka listener](../configuring/#kafka-access) configured with the same authentication type set. + +For example, the following configuration means that only SCRAM can be used as the authentication mechanism: + +```yaml + # ... + strimziOverrides: + # ... + kafka: + listeners: + - name: intscram + type: internal + port: 9092 + tls: false + authentication: + type: scram-sha-512 +``` + +To enable other authentication mechanisms, for example, TLS, add an additional Kafka listener configuration with `tls` set as follows: + +```yaml +# ... +spec: + # ... + adminApi: + endpoints: + - name: routes-example + containterPort: 9080 + type: route + + # ... + strimziOverrides: + # ... + kafka: + listeners: + - name: intscram + type: internal + port: 9092 + tls: false + authentication: + type: scram-sha-512 + - name: inttls + type: internal + port: 9093 + tls: true + authentication: + type: tls +``` + + +#### Routes example + +Example of an endpoint configuration for the Admin API that uses routes: + +```yaml +# ... +spec: + externalCACertificates: + secretName: + # ... + adminApi: + # ... + endpoints: + - name: routes-example + containerPort: 9080 + type: route + tlsVersion: TLSv1.3 + authenticationMechanisms: + - iam-bearer + - tls + - scram-sha-512 + certOverrides: + certificate: mycert.crt + key: mykey.key + secretName: custom-endpoint-cert + host: example-host.apps.example-domain.com +``` + +Where `` is the name of the secret that contains the external CA certificates. + +**Note:** Changing an endpoint in isolation might have adverse effects if Kafka is configured to require authentication and the configured endpoint has no authentication mechanisms specified. In such cases, a warning message might be displayed in the status conditions for the instance. + +#### Ingress example + +Example of an endpoint configuration for the Admin API that uses ingress and NGINX as the ingress controller: + +```yaml +# ... +spec: + # ... + adminApi: + # ... + endpoints: + - name: ingress-example + type: ingress + host: + class: nginx + containerPort: 8001 +``` + +#### Cipher suite override + +You can override the default set of cipher suites for the {{site.data.reuse.es_name}} REST components. Overriding the default set and enabling alternative cipher suites should only be done for specific scenarios, for example, to facilitate connectivity of legacy systems. You can set the override through the `CIPHER_SUITES` environment variable as shown in the following example: + +```yaml +# ... +spec: + # ... + restProducer: + # ... + env: + - name: CIPHER_SUITES + value: >- + TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 +``` + +### Kafka access + +Sample configurations provided for {{site.data.reuse.es_name}} typically include an external listener for Kafka and varying internal listener types by default. The supported external listener can be of type `route` or `ingress`, and it can have one of the following authentication mechanism types: + +- Mutual TLS (`tls`) +- SCRAM-SHA-512 (`scram-sha-512`) +- OAuth (`oauth`) + +Any number of external listeners can be configured, each with any of the supported authentication mechanisms. + +**Note:** If you are overriding the certificates for any of the Kafka listeners with your own certificates, ensure you [provide the external CA certificates](#providing-external-ca-certificates) that signed them in a secret. + +#### Internal access + +Internal listeners for Kafka can also be configured by setting the listener `type:` to `internal`. Each of these can be configured to have any of the authentication mechanism types (Mutual TLS, SCRAM-SHA-512, or OAuth). The following example shows 2 internal listeners configured: the first is set to use SCRAM authentication, while the second listener is set to use mutual TLS. + + +```yaml +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + listeners: + - name: intplain + type: internal + port: 9092 + tls: false + authentication: + type: scram-sha-512 + - name: inttls + type: internal + port: 9093 + tls: true + authentication: + type: tls +``` + +When secure listeners are configured, {{site.data.reuse.es_name}} will automatically generate cluster and client CA certificates, and a valid certificate for the listener endpoint. The generated CA certificates and the certificate for the endpoint can be replaced by provided certificates as described in [providing certificate overrides](#using-your-own-certificates). + +#### Routes example + +The following example snippet defines 2 external listeners that expose the Kafka brokers with 2 {{site.data.reuse.openshift_short}} routes, one with SCRAM-SHA-512 authentication and one with Mutual TLS enabled. + +```yaml +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + listeners: + - name: extscram + type: route + port: 9092 + tls: true + authentication: + type: scram-sha-512 + - name: exttls + type: route + port: 9093 + tls: true + authentication: + type: tls +``` + +#### Ingress example +{: #kafka-ingress-example} + +To use ingress, you define ingress as the external listener and specify the hostname or IP address for your Kafka bootstrap servers. You also define the ingress class for your ingress controller. + +The following example snippet defines an external listener type for ingress that uses NGINX as the ingress controller (`class: nginx`), providing external access to 3 Kafka brokers, with SCRAM-SHA-512 authentication enabled. + +```yaml +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + listeners: + - name: external + type: ingress + port: 9094 + tls: true + authentication: + type: scram-sha-512 + configuration: + bootstrap: + host: + brokers: + - broker: 0 + host: + - broker: 1 + host: + - broker: 2 + host: + class: nginx + - name: tls + type: internal + port: 9093 + tls: true + authentication: + type: tls +``` + +When secure listeners are configured, {{site.data.reuse.es_name}} will automatically generate cluster and client CA certificates, and a valid certificate for the listener endpoint. The generated CA certificates and the certificate for the endpoint can be replaced by provided certificates as described in [providing certificate overrides](#using-your-own-certificates). + +## Enabling OAuth + +Open Authorization (OAuth) is an open standard for authentication and authorization that allows client applications secure delegated access to specified resources. OAuth works over HTTPS and uses access tokens for authorization rather than credentials. + +### Enable OAuth authentication + +To configure OAuth authentication, configure a Kafka listener with type `oauth`, and set the listener to use one of the following token validation methods: + +- **Fast local JSON Web Token (JWT) validation:** a signed token is verified against the OAuth authentication server's public certificate, and a check ensures that the token has not expired on the Kafka cluster. This means that the OAuth authorization server does not need to be contacted, which speeds up the validation. + + **Important:** Local JWT validation does not check the validity of the token with the OAuth authorization server each time the client attempts to authenticate. This means tokens that have not expired are valid even if the client that used the token has been revoked. As the token expiry time is used to identify the token's validity, consider setting short token expiry times when configuring JWT validation, keeping the `grace period` as short as possible. + +- **Token validation by using an introspection endpoint:** the validation of the token is performed on the OAuth authorization server. This is a slower mechanism than the JWT validation method, but ensures that there is no grace period for revoked clients. As soon as a client is revoked, the token will become invalid, regardless of the expiration of the token. + +{{site.data.reuse.es_name}} supports 2 types of SASL mechanisms: `OAUTHBEARER` or `PLAIN`. By default, OAuth authentication uses `OAUTHBEARER` SASL mechanism, which is the most secure mechanism. + +**Important:** For clients that do not support the `OAUTHBEARER` authentication mechanism, you can configure the cluster to use the `PLAIN` mechanism by setting the `enableOauthBearer` property to `false` (default setting is `true` for `OAUTHBEARER`). For more information, see [OAuth 2.0 authentication mechanisms](https://strimzi.io/docs/operators/0.42.0/deploying.html#con-oauth-authentication-broker-str){:target="_blank"}. + +#### Configuring OAuth to use fast local JWT validation + +To configure an OAuth listener to use fast local JWT validation authentication, add the following snippet to your `EventStreams` custom resource, and edit the settings as follows: +- Add the respective URIs of the OAuth authentication server to the `jwksEndpointUri` and `validIssuerUri` properties. +- Create a secret that contains the public CA certificate of the OAuth authentication Server, and reference this secret in the `tlsTrustedCertificates` property of the listener configuration. The `certificate` element in the `tlsTrustedCertificates` references the secret key that contains the CA certificate. + +```yaml +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + listeners: + - name: extoauth + port: 9094 + tls: true + authentication: + type: oauth + jwksEndpointUri: + maxSecondsWithoutReauthentication: 3600 + tlsTrustedCertificates: + - certificate: + secretName: + userNameClaim: preferred_username + validIssuerUri: + type: route +``` + +The snippet provided shows a configuration containing the most commonly used properties. For information about further OAuth properties, see [Using OAuth 2.0 token-based authentication](https://strimzi.io/docs/operators/0.42.0/deploying.html#assembly-oauth-authentication_str){:target="_blank"}. + +#### Configuring OAuth to use token validation by using an introspection endpoint + +To configure an OAuth listener to use introspection endpoint token validation, add the following snippet to your `EventStreams` custom resource, and edit the settings as follows: +- Add the respective URIs of the OAuth authentication server to the `validIssuerUri` and `introspectionEndpointUri` properties. +- Create a secret that contains the public CA certificate of the OAuth authentication Server, and reference this secret in the `tlsTrustedCertificates` property of the listener configuration. The `certificate` element in the `tlsTrustedCertificates` references the secret key that contains the CA certificate. +- Create another secret that contains the secret value of the `userid` as defined in the `clientId` property of the configuration, and reference this secret in the `clientSecret` property of the configuration. In the `key` property, add the key from the Kubernetes secret that contains the secret value for the `userid`. + +```yaml +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + listeners: + - name: extoauth + port: 9094 + tls: true + authentication: + type: oauth + clientId: + clientSecret: + secretName: + key: + validIssuerUri: + introspectionEndpointUri: + userNameClaim: preferred_username + maxSecondsWithoutReauthentication: 3600 + tlsTrustedCertificates: + - certificate: + secretName: + type: route + +``` + +The snippet provided shows a configuration containing the most commonly used properties. For information about further OAuth properties, see [Using OAuth 2.0 token-based authentication](https://strimzi.io/docs/operators/0.42.0/deploying.html#assembly-oauth-authentication_str){:target="_blank"}. + + +### Enable OAuth authorization + +To use OAuth for authorizing access to Kafka resources in {{site.data.reuse.es_name}}, enable OAuth in the {{site.data.reuse.es_name}} custom resource, and then configure your Access Control List (ACL) rules in your selected OAuth server. + +To enable OAuth authorization for {{site.data.reuse.es_name}}, add the following snippet to your `EventStreams` custom resource, and edit the settings as follows: +- Ensure you set the `delegateToKafkaAcls` property to `true`. If this property is set to `false`, some {{site.data.reuse.es_name}} components will not work as expected. +- If you configure OAuth authorization, include in the `superUsers` property the user IDs of the Identity and Access Management (IAM) [admin users](../../security/managing-access/#accessing-the-event-streams-ui-and-cli) and Kubernetes Cluster admin users that administer {{site.data.reuse.es_name}} through the UI or the CLI. If you are not using OAuth authorization, you do not need to specify any `superUsers`. + + + +```yaml +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + authorization: + clientId: + delegateToKafkaAcls: true + tlsTrustedCertificates: + - certificate: ca.crt + secretName: keycloak-ca-cert + tokenEndpointUri: + type: keycloak + superUsers: + - "admin" + - "kubeadmin" +``` + +The snippet provided shows a configuration containing the most commonly used properties. For information about further OAuth properties, see [configuring an OAuth 2.0 authorization server](https://strimzi.io/docs/operators/0.42.0/deploying.html#proc-oauth-server-config-str){:target="_blank"}. + +## Configuring node affinity for components + +You can configure {{site.data.reuse.es_name}} components to run on nodes with specific labels by using node affinity. Node affinity is configured as part of the component's pod template in the `EventStreams` custom resource. + +For REST services, you can configure affinity as follows: + +```yaml +# ... +spec: + # ... + : + # ... + template: + pod: + affinity: + # ... +``` + +Where `` is one of the following values: `adminApi`, `adminUI`, `restProducer`, or `apicurioRegistry`. + +For Kafka and ZooKeeper, you can configure affinity as follows: + +```yaml +# ... +spec: + # ... + strimziOverrides: + # ... + : + # ... + template: + pod: + affinity: + # ... +``` + +Where `` is either `kafka` or `zookeeper`. + +The format of the `affinity` property matches the [Kubernetes specification](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity){:target="_blank"}. For example, if a node is labeled with `mykey=myvalue`, the `affinity` would contain the following settings: + +```yaml +# ... +template: + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: mykey + operator: In + values: + - myvalue +``` + +You can also configure architecture-based node affinity. For example, to configure a component to deploy on Linux on IBM z13 (`s390x`), Linux on IBM Power Systems (`ppc64le`), and Linux 64-bit (`amd64`) systems, you can use the following settings: + +Node affinities for Linux on IBM Power Systems (`ppc64le`), Linux 64-bit (`amd64`) systems , and Linux on IBM z13 (`s390x`) architectures are added by default. However, you can specify node affinities in the {{site.data.reuse.es_name}} custom resource to deploy pods only on the specific architecture. + +```yaml +# ... +template: + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - s390x + - ppc64le +``` + +## Enabling collection of producer metrics + +Producer metrics provide information about the health of your Kafka topics through metrics gathered from producing applications. You can view the information in the [**Producers** dashboard](../../administering/topic-health/). + +Gathering producer metrics is done through a Kafka Proxy, and is not enabled by default. To enable metrics gathering and have the information displayed in the dashboard, enable the Kafka Proxy by adding the `spec.kafkaProxy` property to the `EventStreams` custom resource as follows: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + kafkaProxy: {} +# ... +``` + +**Important:** Enabling the Kafka Proxy to gather producer metrics places an intermediary between your producing clients and your Kafka brokers. This adds latency to any traffic to your Kafka brokers. Consider the performance implications of having the proxy in front of your Kafka brokers. You can also leave the proxy disabled and gather producer metrics from the clients directly by using [JMX](https://kafka.apache.org/37/documentation/#monitoring){:target="_blank"}. + + +## Configuring external monitoring through Prometheus + +Metrics provide information about the health and operation of the {{site.data.reuse.es_name}} instance. + +Metrics can be enabled for Kafka, ZooKeeper, geo-replicator, and Kafka Connect pods. + +**Note:** Kafka metrics can also be exposed externally through JMX by [configuring external monitoring tools](#configuring-external-monitoring-through-jmx). + +Kafka metrics can be enabled by setting `spec.strimziOverrides.kafka.metricsConfig` in the `EventStreams` custom resource to point to the `metrics-config` ConfigMap. For example: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + strimziOverrides: + kafka: + # ... + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + key: kafka-metrics-config.yaml + name: metrics-config +# ... +``` + +ZooKeeper metrics can be enabled by setting `spec.strimziOverrides.zookeeper.metricsConfig` in the `EventStreams` custom resource to point to the `metrics-config` ConfigMap. For example: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + strimziOverrides: + zookeeper: + # ... + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + key: zookeeper-metrics-config.yaml + name: metrics-config +# ... +``` + +The following is the default `metrics-config` ConfigMap in YAML format: + +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: metrics-config +data: + kafka-metrics-config.yaml: | + lowercaseOutputName: true + rules: + - attrNameSnakeCase: false + name: kafka_controller_$1_$2_$3 + pattern: kafka.controller<>(Count|Value|Mean) + - attrNameSnakeCase: false + name: kafka_server_BrokerTopicMetrics_$1_$2 + pattern: kafka.server<>(Count) + - attrNameSnakeCase: false + name: kafka_server_BrokerTopicMetrics_$1__alltopics_$2 + pattern: kafka.server<>(OneMinuteRate) + - attrNameSnakeCase: false + name: kafka_server_ReplicaManager_$1_$2 + pattern: kafka.server<>(Value) + zookeeper-metrics-config.yaml: | + lowercaseOutputName: true + rules: [] + +``` + +Geo-replicator metrics can be enabled by setting `spec.metrics` to `{}` in the `KafkaMirrorMaker2` custom resource. For example: + +```yaml +apiVersion: eventstreams.ibm.com/v1alpha1 +kind: KafkaMirrorMaker2 +# ... +spec: + # ... + metrics: {} +# ... +``` + +**Note:** The {{site.data.reuse.es_name}} operator automatically applies a `KafkaMirrorMaker2` custom resource when a `EventStreamsGeoReplicator` custom resource is created. Metrics can then be enabled by editing the generated `KafkaMirrorMaker2` custom resource. + +Kafka Connect metrics can be enabled by setting `spec.metrics` to `{}` in the `KafkaConnect` custom resource. For example: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: KafkaConnect +# ... +spec: + # ... + metrics: {} +# ... +``` + +To complement the default Kafka metrics, you can configure {{site.data.reuse.es_name}} to publish additional information about your {{site.data.reuse.es_name}} instance by adding the `spec.kafkaProxy` property to the `EventStreams` custom resource as follows: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + kafkaProxy: {} +# ... +``` + +**Note:** For details about viewing metrics information, see the [cluster health](../../administering/cluster-health) and [topic health](../../administering/topic-health) sections. + +## Configuring external monitoring through JMX + +You can use third-party monitoring tools to monitor the deployed {{site.data.reuse.es_name}} Kafka cluster by collecting Kafka metrics. To set this up, you need to: + +- Have a third-party monitoring tool set up to be used within your {{site.data.reuse.openshift_short}} cluster. +- Enable access to the broker JMX port by setting `spec.strimizOverrides.kafka.jmxOptions`. + + ```yaml + apiVersion: eventstreams.ibm.com/v1beta2 + kind: EventStreams + # ... + spec: + # ... + strimziOverrides: + # ... + kafka: + jmxOptions: {} + ``` + +- Include any configuration settings for {{site.data.reuse.es_name}} as required by your monitoring tool. For example, Datadog's autodiscovery requires you to annotate Kafka broker pods (`strimziOverrides.kafka.template.pod.metadata.annotations`) +- Configure your monitoring applications to [consume JMX metrics](../../security/secure-jmx-connections/). + +## Configuring the Kafka Exporter + +You can configure the Kafka Exporter to expose additional metrics to Prometheus on top of the default ones. For example, you can obtain the consumer group lag information for each topic. + +Kafka Exporter can be enabled by setting `spec.kafkaExporter` to `{}` in the `EventStreams` custom resource. For example: + +```yaml + apiVersion: eventstreams.ibm.com/v1beta2 + kind: EventStreams + # ... + spec: + # ... + strimziOverrides: + # ... + kafkaExporter: {} +``` + + +You can also configure Kafka Exporter using a `regex` to expose metrics for a collection of topics and consumer groups that match the expression. For example, to enable JMX metrics collection for the topic `orders` and the group `buyers`, configure the `EventStreams` custom resource as follows: + +```yaml + apiVersion: eventstreams.ibm.com/v1beta2 + kind: EventStreams + # ... + spec: + # ... + strimziOverrides: + # ... + kafkaExporter: + groupRegex: buyers + topicRegex: orders +``` + +For more information about configuration options, see [configuring the Kafka Exporter](https://strimzi.io/docs/operators/0.42.0/deploying.html#proc-metrics-kafka-deploy-options-str){:target="_blank"}. + +## Configuring the JMX Exporter + +You can configure the JMX Exporter to expose JMX metrics from Kafka brokers, ZooKeeper nodes, and Kafka Connect nodes to Prometheus. + +To enable the collection of all JMX metrics available on the Kafka brokers and ZooKeeper nodes, configure the `EventStreams` custom resource as follows: + +```yaml + apiVersion: eventstreams.ibm.com/v1beta2 + kind: EventStreams + # ... + spec: + # ... + strimziOverrides: + kafka: + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + key: kafka-metrics-config.yaml + name: metrics-config + # ... + zookeepers: + # + # ... +``` + +For more information about configuration options, see the following documentation: + +- [Kafka and ZooKeeper JMX metrics configuration](https://strimzi.io/docs/operators/0.42.0/deploying.html#assembly-metrics-str){:target="_blank"} +- [Kafka JMX metrics configuration](https://strimzi.io/docs/operators/0.42.0/configuring.html#con-common-configuration-prometheus-reference){:target="_blank"} + +## Enabling and configuring Kafka Bridge + +With [Kafka Bridge](https://strimzi.io/docs/bridge/latest/){:target="_blank"}, you can connect client applications to your {{site.data.reuse.es_name}} Kafka cluster over HTTP, providing a standard web API connection to {{site.data.reuse.es_name}} rather than the custom Kafka protocol. + +To enable Kafka Bridge for {{site.data.reuse.es_name}}, create a `KafkaBridge` custom resource alongside the `EventStreams` custom resource. This can be defined in a YAML configuration document under the **Event Streams** operator in the {{site.data.reuse.openshift_short}} web console. + +For example, to enable Kafka Bridge for {{site.data.reuse.es_name}} in the namespace `es-kafka-bridge`, create the following `KafkaBridge` configuration, where `spec.bootstrapServers` is the address of your {{site.data.reuse.es_name}} Kafka cluster, and `spec.http.port` is the port number for Kafka Bridge to access your cluster (default is `8080`): + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: KafkaBridge +metadata: + name: my-bridge + namespace: es-kafka-bridge + labels: + eventstreams.ibm.com/cluster: + backup.eventstreams.ibm.com/component: kafkabridge +spec: + replicas: 1 + bootstrapServers: 'test.kafka-bootstrap.es-kafka-bridge:9093' + http: + port: 8080 + template: + bridgeContainer: + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - s390x + - ppc64le +``` + +Depending on your setup and purpose of deployment, you can add more `replicas` which sets the number of Kafka Bridge instances to run. For production environments, for example, consider deploying more than one replica for resilience. + +After enabling Kafka Bridge, you can expose the service outside your cluster for external clients to connect to it as described in [exposing services for external access](../../connecting/expose-service/). + +For more information about Kafka Bridge, including further configuration options and usage, see [connecting with Kafka Bridge](../../connecting/kafka-bridge/). + +## Enabling and configuring Cruise Control + +To enable Cruise Control, set the `spec.strimizOverrides.cruiseControl` property to `{}` in the `EventStreams` custom resource: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + strimziOverrides: + # ... + cruiseControl: {} +``` + +**Note:** Ensure you have more than 1 Kafka broker configured to take advantage of Cruise Control. All [sample configurations](../planning/#sample-deployments) provided have more than 1 broker except the **Lightweight without security** sample. + +When enabled, you can use the [default](#cruise-control-defaults) Cruise Control configuration to optimize your Kafka cluster. You can also specify your required configuration as described in the following sections. + +When configuring Cruise Control, you can define the following settings in the `EventStreams` custom resource: +- Main optimization goals in `spec.strimziOverrides.cruiseControl.config.goals` +- Default optimization goals in `spec.strimziOverrides.cruiseControl.config["default.goals"]` + + **Note:** If you do not set main optimization goals and default goals, then the [Cruise Control defaults](#cruise-control-defaults) are used. + +- Hard goals in `spec.strimziOverrides.cruiseControl.config["hard.goals"]` +- The capacity limits for broker resources, which Cruise Control uses to determine if resource-based optimization goals are being broken. The `spec.strimziOverrides.cruiseControl.brokerCapacity` property defines the Kafka broker resource capacities that Cruise Control will optimize around. + +Cruise Control includes a number of [configuration options](https://github.com/linkedin/cruise-control/wiki/Configurations#cruise-control-configurations){:target="_blank"}. You can modify these configuration options for {{site.data.reuse.es_name}}, except the options managed directly by [Strimzi](https://strimzi.io/docs/operators/0.42.0/configuring.html#property-cruise-control-config-reference){:target="_blank"}. + +When enabled, you can use Cruise Control and the `KafkaRebalance` custom resources to [optimize](../../administering/cruise-control/) your deployed {{site.data.reuse.es_name}} Kafka cluster. + +### Cruise Control defaults + +{{site.data.reuse.es_name}} supports a subset of the Cruise Control goals. If the main optimization goals and default goals (`spec.strimziOverrides.cruiseControl.config.goals` and `spec.strimziOverrides.cruiseControl.config["default.goals"]`, respectively) are not set, then the Cruise Control configuration defaults to the following goals (in descending order of priority): + +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderReplicaDistributionGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.PreferredLeaderElectionGoal` + +For more information about the optimization goals, see the Cruise Control [documentation](https://github.com/linkedin/cruise-control/wiki/Pluggable-Components#goals){:target="_blank"}. + +### Main optimization goals + +The main optimization goals define the goals available to be used in Cruise Control operations. Goals not listed cannot be used. + +The `spec.strimziOverrides.cruiseControl.config.goals` property defines the list of goals Cruise Control can use. + +The main optimization goals have [defaults](#cruise-control-defaults) if not configured. + +For example, if you want Cruise Control to only consider using `com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal` and `com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal`, set values for `spec.strimziOverrides.cruiseControl.config.goals` property as follows: + +```yaml +# ... +spec: + # ... + strimziOverrides: + # ... + cruiseControl: + # ... + config: + # ... + goals: > + com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal, + com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal +``` + +### Default goals + +The default goals define the set of goals that you want your cluster to meet most often. They are set in the `spec.strimziOverrides.cruiseControl.config["default.goals"]` property. By default, every 15 minutes, Cruise Control will use the current state of your Kafka cluster to generate a cached optimization proposal by using the configured `default.goals` list. + +If no default goals are set, the main optimization goals are used as the [default](#cruise-control-defaults) optimization goals. + +For example, if you want Cruise Control to always consider meeting `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal`, `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal`, and `com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal`, set values for the `spec.strimziOverrides.cruiseControl.config["default.goals"]` property as follows: + +```yaml +# ... +spec: + # ... + strimziOverrides: + # ... + cruiseControl: + # ... + config: + # ... + default.goals: > + com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal, + com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal, + com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal +``` + +### Hard goals + +Hard goals define the list of goals that must be met by an optimization proposal and cannot be violated in any of the optimization functions of Cruise Control. + +Hard goals can be set by the `spec.strimziOverrides.cruiseControl.config["hard.goals"]` property. + +In Cruise Control, the following [main optimization goals](#cruise-control-defaults) are preset as hard goals: +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal` +- `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal` + +For example, to configure Cruise Control to always consider `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal` and `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal` in an optimization proposal, provide these goals as values in the `spec.strimziOverrides.cruiseControl.config["hard.goals"]` property as follows: + +```yaml +# ... +spec: + # ... + strimziOverrides: + # ... + cruiseControl: + # ... + config: + # ... + hard.goals: > + com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal, + com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal +``` + +**Note:** The longer the list of hard goals, the less likely it is that Cruise Control will be able to find a viable optimization proposal. Consider configuring fewer hard goals and more goals for the [optimization proposals](../../administering/cruise-control/#setting-up-optimization) in the `KafkaRebalance` custom resource. + + +### Cruise Control BrokerCapacity + +Specifies capacity limits for broker resources. + +Use the `spec.strimziOverrides.cruiseControl.brokerCapacity` property to define capacity limits for Kafka broker resources. Cruise Control will use the set limits to determine if resource-based optimization goals are being broken. The following table provides information about the resource capacity settings, the goals they affect, and the units they use: + +| brokerCapacity | Goal | unit | +| :--------------- | :---------------------------------------------------------------------------- | :--- | +| inboundNetwork | `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal` | KB/s | +| outboundNetwork | `com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal` | KB/s | +| disk | `com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal` | Bytes | +| cpuUtilization | `com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal` | Percent (0-100) | + +**Note:** Ensure you add the unit when configuring a `brokerCapacity` key, except for `cpuUtilization` where the percentage is not required. + +For example, to configure Cruise Control to optimize around having an inbound network byte rate of `1000` kilobytes per second and a cpu utilization of 80 percent, configure the `spec.strimziOverrides.cruiseControl.brokerCapacity` property as follows: + +```yaml +# ... +spec: + # ... + strimziOverrides: + # ... + cruiseControl: + # ... + brokerCapacity: + # ... + inboundNetwork: 1000KB/s + # Optimize for CPU utilization of 80% + cpuUtilization: 80 +``` + + +## Using your own certificates + +{{site.data.reuse.es_name}} offers the capability to provide your own CA certificates and private keys instead of using the ones generated by the operator. If a CA certificate and private key are provided, the listener certificate is generated automatically and signed using the CA certificate. + +{{site.data.reuse.es_name}} also offers the capability to provide your own certificates. + +**Note:** You must complete the process of providing your own certificates before installing an instance of {{site.data.reuse.es_name}}. + +You must provide your own X.509 certificates and keys in PEM format with the addition of a PKCS12-formatted certificate and the CA password. If you want to use a CA which is not a Root CA, you have to include the whole chain in the certificate file. Include the chain in the following order: + +1. The cluster or client CA +2. One or more intermediate CAs +3. The root CA + +Ensure you configure each CA in the chain as a CA with the X509v3 Basic Constraints. + +### Providing a CA certificate and key + +**Note:** In the following instructions, the CA public certificate file is denoted `CA.crt` and the CA private key is denoted `CA.key`. + +As {{site.data.reuse.es_name}} also serves the `truststore` in PKCS12 format, generate a `.p12` file containing the relevant CA Certificates. When generating your PKCS12 truststore, ensure that the truststore does not contain the CA private key. This is important because the `.p12` file will be available to download from the {{site.data.reuse.es_name}} UI and distributed to clients. + +The following is an example showing how to use the Java `keytool` utility to generate a PKCS12 truststore that does not contain a private key: + +```shell +keytool -import -file -keystore ca.jks +keytool -importkeystore -srckeystore ca.jks -srcstoretype JKS -deststoretype PKCS12 -destkeystore ca.p12 +``` + +**Note:** Using OpenSSL PKCS12 commands to generate a truststore without private keys can break the cluster, because the resulting truststore is not compatible with Java runtimes. + +One way to test that the truststore is compatible and contains the correct certificates is to use the following java `keytool` utility command: + +```shell +keytool -list -keystore ca.p12 -storepass +``` + +The cluster and client certificates, and keys must be added to secrets in the namespace that the {{site.data.reuse.es_name}} instance is intended to be created in. The naming of the secrets and required labels must follow the conventions detailed in the following command templates. + +The following commands can be used to create and label the secrets for custom certificates and keys. The templates demonstrate providing cluster certificates but the same commands can be re-used substituting `cluster` with `clients` in each secret name. + +For each command, provide the intended name and namespace for the {{site.data.reuse.es_name}} instance. + +```shell +kubectl create --namespace secret generic -cluster-ca --from-file=ca.key=CA.key +``` + +```shell +kubectl label --namespace secret -cluster-ca eventstreams.ibm.com/kind=Kafka eventstreams.ibm.com/cluster= +``` + +```shell +kubectl annotate --namespace secret -cluster-ca eventstreams.ibm.com/ca-key-generation=0 +``` + +```shell +kubectl create --namespace secret generic -cluster-ca-cert --from-file=ca.crt=CA.crt --from-file=ca.p12=CA.p12 --from-literal=ca.password='' +``` + +```shell +kubectl label --namespace secret -cluster-ca-cert eventstreams.ibm.com/kind=Kafka eventstreams.ibm.com/cluster= +``` + +```shell +kubectl annotate --namespace secret -cluster-ca-cert eventstreams.ibm.com/ca-cert-generation=0 +``` + +**Note:** The `eventstreams.ibm.com/ca-cert-generation` and `eventstreams.ibm.com/ca-key-generation` values identify whether certificates are being renewed or not. Only set 0 for these values if you have not installed an instance of {{site.data.reuse.es_name}} yet. For more information about when to amend these annotations, see [renewing certificates](../../security/renewing-certificates/). + +To make use of the provided secrets, {{site.data.reuse.es_name}} will require the following overrides to be added to the custom resource. + +```yaml +spec: + # ... + strimziOverrides: + clusterCa: + generateCertificateAuthority: false + + # And/Or + + clientsCa: + generateCertificateAuthority: false +``` + +For information about configuring the renewal settings for certificates, see [renewing certificates](../../security/renewing-certificates/). + +### Providing listener certificates + +To use TLS hostname verification with your own Kafka listener certificates, ensure you use the correct Subject Alternative Names (SANs) for each listener. The certificate SANs must specify hostnames for: + + - All of the Kafka brokers in your cluster + + - The Kafka cluster bootstrap service + +You can use wildcard certificates if they are supported by your CA. + +For internal listeners, the hostnames will be service names. For external listeners, the hostnames will be present in the ingress or route definitions. + + +Create a secret containing the private key and server certificate: + +```shell +kubectl create secret generic my-secret --from-file=my-listener-key.key --from-file=my-listener-certificate.crt +``` + +To make use of the secret, {{site.data.reuse.es_name}} will require the following overrides to be added to the custom resource. + +```yaml +spec: + # ... + externalCACertificates: + secretName: + strimziOverrides: + kafka: + listeners: + - name: external + # ... + configuration: + brokerCertChainAndKey: + certificate: my-listener-certificate.crt + key: my-listener-key.key + secretName: my-secret +``` + +Where `` is the name of the secret that contains the external CA certificates. + +## Providing external CA certificates + +If you provide your own certificates for [Kafka listeners](#kafka-access) instead of using the ones generated by {{site.data.reuse.es_name}}, you must create a secret containing the external public certificate authority (CA) certificates that were used to sign those certificates. + +You can also provide your own certificates for any of the [REST component endpoints](#rest-services-access), and include the external CA certificates in the secret for those as well if you want to make them available to download in {{site.data.reuse.es_name}}. + +After you provide the secrets, both the {{site.data.reuse.es_name}} cluster CA certificates and the external CA certificates will be included in the PEM or PKCS12 file that you can [download by using the {{site.data.reuse.es_name}} UI or CLI](../../getting-started/connecting/#securing-the-connection). This will ensure that clients can connect to any of the endpoints in the cluster regardless of whether they have overridden certificates or not. + +To provide the external CA certificates to the {{site.data.reuse.es_name}} cluster, you must create a secret that contains a `ca.crt` element that has all the required external public CA certificates in PEM format. If a CA chain was used to sign the overriding certificates, ensure that the full CA chain is provided in the secret. Include the chain in the following order: + +1. The CA that signed the Kafka listener (`brokerCertChainAndKey`) or REST endpoint certificates +2. Any intermediate CAs +3. The root CA + +The following is the format of an external CA file with a Single CA Certificate to upload to the secret: + +``` +-----BEGIN CERTIFICATE----- + +-----END CERTIFICATE----- + +``` + +The following is the format of an external CA file with a CA Certificate chain to upload to the secret: + +``` +-----BEGIN CERTIFICATE----- + +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- + +-----END CERTIFICATE----- +... +-----BEGIN CERTIFICATE----- + +-----END CERTIFICATE----- + +``` + +1. To create the secret that contains the external CA certificate, run the following command: + + ```shell + kubectl create secret generic --from-file=ca.crt=.crt + ``` + +## Setting up the connection + +To set up a connection between your {{site.data.reuse.es_name}} and {{site.data.reuse.eem_name}} instances, complete the following steps: + +1. {{site.data.reuse.cncf_cli_login}} +1. Create a file called `es-eem-configmap.yaml` and copy the following YAML content into the file to create the ConfigMap that has the details for creating a server connection to your {{site.data.reuse.eem_name}} instance: + + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: -ibm-es-discovery-endpoints + namespace: + data: + endpoints.json: |- + { + "instances": [ + { + "name": "", + "type": "event-endpoint-manager", + "server": { + "host": "", + "port": , + "ssl": true, + "certificates": ["-----BEGIN CERTIFICATE-----\n\n-----END CERTIFICATE-----\n"] + }, + "additional-info": { + "eem-ui-url": "" + } + } + ] + } + ``` + + Where: + + - `` is the name of the {{site.data.reuse.es_name}} instance. + - `` is the namespace where your {{site.data.reuse.es_name}} instance is installed. + - `` is the name of the {{site.data.reuse.eem_name}} instance. + - `` is the hostname of the [API endpoint URI]({{ 'eem/getting-started/logging-in/#retrieving-the-urls' | relative_url}}) of the {{site.data.reuse.eem_manager}}. + - `` is the port number of the [API endpoint URI]({{ 'eem/getting-started/logging-in/#retrieving-the-urls' | relative_url}}) of the {{site.data.reuse.eem_manager}}. + - `` is the CA certificate of the {{site.data.reuse.eem_name}} instance that you downloaded earlier. + - `` is the hostname of the [UI endpoint URI]({{ 'eem/getting-started/logging-in/#retrieving-the-urls' | relative_url}}) of the {{site.data.reuse.eem_name}} UI. + +2. Apply the ConfigMap by running the following command: + + ```shell + kubectl apply -f es-eem-configmap.yaml + ``` + +After the ConfigMap is created, you can [share your topics](../../getting-started/sharing-topic/) with {{site.data.reuse.eem_name}} from the {{site.data.reuse.es_name}} UI. diff --git a/_es_11.5/02-installing/08-postinstallation.md b/_es_11.5/02-installing/08-postinstallation.md new file mode 100644 index 00000000..c4425ee6 --- /dev/null +++ b/_es_11.5/02-installing/08-postinstallation.md @@ -0,0 +1,158 @@ +--- +title: "Post-installation tasks" +excerpt: "Post-installation tasks after successfully installing Event Streams." +categories: installing +slug: post-installation +toc: true +--- + +Consider the following tasks after installing {{site.data.reuse.es_name}}. + +## Verifying an installation + +To verify that your {{site.data.reuse.es_name}} installation deployed successfully, you can check the status of your instance through the {{site.data.reuse.openshift_short}} web console or command line. + +### Check the status of the EventStreams instance through the {{site.data.reuse.openshift_short}} web console + +1. {{site.data.reuse.openshift_ui_login}} +2. {{site.data.reuse.task_openshift_navigate_installed_operators}} +3. {{site.data.reuse.task_openshift_select_operator}} +4. {{site.data.reuse.task_openshift_select_instance}} +5. The **Phase** field will display the current state of the EventStreams custom resource. When the {{site.data.reuse.es_name}} instance is ready, the phase will display `Ready`, meaning the deployment has completed. + +### Check the status of the {{site.data.reuse.es_name}} instance through the command line + +After all the components of an {{site.data.reuse.es_name}} instance are active and ready, the `EventStreams` custom resource will have a `Ready` phase in the status. +To verify the status: + +1. {{site.data.reuse.cncf_cli_login}} +2. Run the `kubectl get` command as follows: + + ```shell + kubectl get eventstreams + ``` + +For example, the installation of the instance called `development` is complete when the `STATUS` returned by the `kubectl get` command displays `Ready`. + +An example output: + +```shell +$ kubectl get eventstreams +> +NAME STATUS +development Ready +``` + +**Note:** It might take several minutes for all the resources to be created and the `EventStreams` instance to become ready. + +## Installing the {{site.data.reuse.es_name}} command-line interface + +The {{site.data.reuse.es_name}} CLI is a plugin for the `kubectl` and `cloudctl` CLI. Use the {{site.data.reuse.es_name}} CLI to manage your {{site.data.reuse.es_name}} instance from the command line. + +**Note:** For completing tasks by using the {{site.data.reuse.es_name}} CLI, you can use `cloudctl es` commands if your deployment is on the {{site.data.reuse.openshift_short}} with {{site.data.reuse.fs}}. This documentation set includes instructions that use the `kubectl` command, except for cases where the task is specific to OpenShift with {{site.data.reuse.fs}}. + +Examples of management activities include: + +- Creating, deleting, and updating Kafka topics. +- Creating, deleting, and updating Kafka users. +- Creating, deleting, and updating Kafka message schemas. +- Managing geo-replication. +- Displaying the cluster configuration and credentials. + +{{site.data.reuse.es_cli_kafkauser_note}} +You can also use the {{site.data.reuse.es_name}} UI to generate the Kafka users. + +### Kubernetes plugin (`kubectl es`) + +For OpenShift and other Kubernetes platforms running without {{site.data.reuse.fs}}, install the {{site.data.reuse.es_name}} CLI with the Kubernetes command-line tool (`kubectl`) as follows: + +1. Ensure you have the Kubernetes command-line tool (`kubectl`) [installed](https://kubernetes.io/docs/tasks/tools/){:target="_blank"}. +2. [Log in](../../getting-started/logging-in/) to your {{site.data.reuse.es_name}} instance as an administrator. +3. Click **Toolbox** in the primary navigation. +4. Go to the **{{site.data.reuse.es_name}} command-line interface** section and click **Find out more**. +5. Download the {{site.data.reuse.es_name}} CLI plug-in for your system by using the appropriate link. +6. Rename the plugin file to `kubectl-es` and move it into a directory on the user's PATH. For example, on Linux and MacOS, move and rename the plugin file by running the following command: + + ```shell + sudo mv ./kubectl-es-plugin.bin /usr/local/bin/kubectl-es + ``` + +For more information about `kubectl` plugins, see the [Kubernetes documentation](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/){:target="_blank"}. + +To start the {{site.data.reuse.es_name}} CLI and check all available command options in the CLI, use the `kubectl es` command. +For an exhaustive list of commands, you can run: +```shell +kubectl es --help +``` + +To get help for a specific command, run: +```shell +kubectl es --help +``` + +To run commands after installing, log in and initialize the CLI as described in [logging in](../../getting-started/logging-in/). + +### IBM Cloud Pak CLI plugin (`cloudctl es`) + +For {{site.data.reuse.openshift_short}} with {{site.data.reuse.icpfs}}, install the {{site.data.reuse.es_name}} CLI with the IBM Cloud Pak CLI (`cloudctl`) as follows: + +1. Ensure you have the IBM Cloud Pak CLI (`cloudctl`) installed either by [retrieving the binary from your cluster](https://www.ibm.com/docs/en/cloud-paks/foundational-services/3.23?topic=323-installing-foundational-services-by-using-cli){:target="_blank"} or [downloading the binary from a release on the GitHub project](https://github.com/IBM/cloud-pak-cli/releases){:target="_blank"}. + + **Note:** Ensure you download the correct binary for your architecture and operating system. +2. [Log in](../../getting-started/logging-in/) to your {{site.data.reuse.es_name}} instance as an administrator. +3. Click **Toolbox** in the primary navigation. +4. Go to the **{{site.data.reuse.es_name}} command-line interface** section and click **Find out more**. +5. Download the {{site.data.reuse.es_name}} CLI plug-in for your system by using the appropriate link. +6. Install the plugin using the following command: + + ```shell + cloudctl plugin install + ``` + +To start the {{site.data.reuse.es_name}} CLI and check all available command options in the CLI, use the `cloudctl es` command. +For an exhaustive list of commands, you can run: + +```shell +cloudctl es --help +``` + +To get help for a specific command, run: + +```shell +cloudctl es --help +``` + +## Firewall and load balancer settings + +In your firewall settings, ensure you enable communication for the endpoints that {{site.data.reuse.es_name}} services use. + +If you have load balancing set up to manage traffic for your cluster, ensure that it is set up to handle the {{site.data.reuse.es_name}} endpoints. + +On the {{site.data.reuse.openshift_short}}, {{site.data.reuse.es_name}} uses routes. +If you are using OpenShift, ensure your [router](https://docs.openshift.com/container-platform/4.16/networking/routes/route-configuration.html){:target="_blank"} is set up as required. + +On other Kubernetes platforms, {{site.data.reuse.es_name}} uses ingress for [external access](../configuring/#configuring-access). You can configure ingress to provide load balancing through an ingress controller. Ensure your [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/){:target="_blank"} is set up as required for your Kubernetes platform. + +## Connecting clients + +For instructions about connecting a client to your {{site.data.reuse.es_name}} instance, see [connecting clients](../../getting-started/connecting). + +## Setting up access + +Secure your installation by [managing the access](../../security/managing-access/) your users and applications have to your {{site.data.reuse.es_name}} resources. + +For example, if you are using {{site.data.reuse.icpfs}}, associate your {{site.data.reuse.fs}} teams with your {{site.data.reuse.es_name}} instance to grant access to resources based on roles. + +## Scaling your Kafka Environment + +Depending on the size of the environment that you are installing, consider scaling and sizing options. You might also need to change scale and size settings for your services over time. For example, you might need to add additional Kafka brokers over time. + +See [how to scale your Kafka environment](../../administering/scaling). + +## Considerations for GDPR readiness + +Consider [the requirements for GDPR](../../security/gdpr-considerations/), including [encrypting your data](../../security/encrypting-data/) for protecting it from loss or unauthorized access. + +## Enable metrics for monitoring + +To display metrics in the monitoring dashboards of the {{site.data.reuse.es_name}} UI, ensure that you enable the **Monitoring** dashboard by following the instructions in the [post-upgrade tasks](../upgrading/#enable-metrics-for-monitoring) before accessing the dashboard. diff --git a/_es_11.5/02-installing/09-backup-restore.md b/_es_11.5/02-installing/09-backup-restore.md new file mode 100644 index 00000000..10da1fed --- /dev/null +++ b/_es_11.5/02-installing/09-backup-restore.md @@ -0,0 +1,278 @@ +--- +title: "Backing up and restoring on OpenShift" +excerpt: "Find out how you can back up and restore your Event Streams static configuration." +categories: installing +slug: backup-restore +toc: true +--- + +A complete disaster recovery mechanism for {{site.data.reuse.es_name}} can be achieved by using the OpenShift APIs for Data Protection (OADP) and MirrorMaker 2.0. + +OADP is used for creating a backup of {{site.data.reuse.es_name}} static configurations such as custom resources and operator-related configurations. The backup can be used to restore the configuration in the same cluster or a different cluster. High-frequency data updates such as Kafka topic data are not included in these backups. + +- If Kafka topic custom resources are included in the {{site.data.reuse.es_name}} installation, the topic configuration can be backed up and restored by using OADP. +- The actual messages within the Kafka topics cannot be preserved through the OADP mechanism. However, after you restore your {{site.data.reuse.es_name}} static configurations onto a new cluster, the topic data can be effectively replicated by implementing MirrorMaker 2.0, which provides a robust backup system for disaster recovery. +- When you restore the {{site.data.reuse.es_name}} static configurations within your cluster, the topic data remains intact while the persistent storage remains available, ensuring data retention during the restore process. + +Find out more about how to set up an OADP operator in an OpenShift cluster for a successful backup and restore. + +## Before you begin + +Consider the following when planning to back up and restore your {{site.data.reuse.es_name}} configurations: + +- Additional ConfigMaps that you create are not included as part of the {{site.data.reuse.es_name}} backup process. For example, if you have created ConfigMaps for accessing the {{site.data.reuse.es_name}} features such as monitoring with Prometheus, the ConfigMaps will not be backed up. Ensure the ConfigMaps are created again before restoring your {{site.data.reuse.es_name}} configurations. + +- When performing a restoration on a new cluster, ensure that the bootstrap address is updated to reflect the address of the new cluster if the bootstrap address is specified in any of the restored custom resources, such as Kafka Connect or Kafka Bridge. + +Ensure that the following prerequisites are met before you back up and restore {{site.data.reuse.es_name}} configurations by using OADP: + +- A storage mechanism, such as Amazon Simple Storage Service (Amazon S3), to securely retain backed up data. For more information about the supported storage types, see the [{{site.data.reuse.openshift_short}} documentation](https://docs.openshift.com/container-platform/4.16/backup_and_restore/application_backup_and_restore/installing/about-installing-oadp.html#oadp-s3-compatible-backup-storage-providers-supported){:target="_blank"}. + +- Ensure that the OADP operator is installed in your cluster along with a `DataProtectionApplication` instance in `Ready` state. Follow the instructions in the [{{site.data.reuse.openshift_short}} documentation](https://docs.openshift.com/container-platform/4.16/backup_and_restore/application_backup_and_restore/installing/oadp-installing-operator.html){:target="_blank"} to install OADP. + +## Backing up configurations of {{site.data.reuse.es_name}} + +Follow the instructions to back up your {{site.data.reuse.es_name}} configurations. + +### Pause reconciliation of the Kafka users + +Before backing up {{site.data.reuse.es_name}} resources from the cluster, pause the reconciliation of the Kafka users. This is to avoid scenarios where the Kafka user custom resources are restored first, causing new secrets to be generated before the ones from the backup are restored. To disable the reconciliation of all Kafka users before backing up, run the following command: + +```shell +oc annotate KafkaUser -n --all strimzi.io/pause-reconciliation="true" +``` + +### Applying backup labels + +{{site.data.reuse.es_name}} uses backup labels on resources to ensure that all the important resources are included in the backup. Complete the following steps to add backup labels that are specific to operator and instance resources: + +1. {{site.data.reuse.openshift_cli_login}} +2. Run the following commands to label the operator resources such as `operatorgroup`, `catalogsource`, and `Subscription`: + + ```shell + kubectl label catsrc "${eventstreams-catalog-name}" -n openshift-marketplace backup.eventstreams.ibm.com/component=catalogsource --overwrite=true + + kubectl label operatorgroup -n ${namespace} $(oc get operatorgroup -n ${namespace} | grep "${namespace}" | awk '{ print $1}') backup.eventstreams.ibm.com/component=operatorgroup --overwrite=true + + kubectl label subscription -n ${namespace} $(oc get subscription -n ${namespace} | grep "ibm-eventstreams" | awk '{ print $1}') backup.eventstreams.ibm.com/component=subscription --overwrite=true + ``` + +3. Ensure that the {{site.data.reuse.es_name}} custom resource YAML and the related operands are deployed with backup labels. + + ```yaml + labels: + backup.eventstreams.ibm.com/component: {OperandName} + ``` + + In {{site.data.reuse.es_name}} 11.2.5 and later, backup labels are included in the custom resource YAML samples. Apply the custom resource YAML to deploy your resources with backup labels. + + In {{site.data.reuse.es_name}} versions earlier than 11.2.5 and for the upgrade scenarios from an older version to the 11.2.5 version, you can add labels to your custom resources by running the following commands. + + ```shell + kubectl label eventstreams -n ${namespace} --all backup.eventstreams.ibm.com/component=eventstreams --overwrite=true + kubectl label kafkaconnects.eventstreams.ibm.com -n ${namespace} --all backup.eventstreams.ibm.com/component=kafkaconnect --overwrite=true + + kubectl label kafkatopics.eventstreams.ibm.com -n ${namespace} --all backup.eventstreams.ibm.com/component=kafkatopic --overwrite=true + kubectl label kafkausers.eventstreams.ibm.com -n ${namespace} --all backup.eventstreams.ibm.com/component=kafkauser --overwrite=true + + kubectl label kafkabridges.eventstreams.ibm.com -n ${namespace} --all backup.eventstreams.ibm.com/component=kafkabridge --overwrite=true + kubectl label kafkaconnectors.eventstreams.ibm.com -n ${namespace} --all backup.eventstreams.ibm.com/component=kafkaconnector --overwrite=true + kubectl label kafkarebalances.eventstreams.ibm.com -n ${namespace} --all backup.eventstreams.ibm.com/component=kafkarebalance --overwrite=true + ``` + +### Configuring the backup custom resource + +The backup custom resource is responsible for generating backup files for {{site.data.reuse.es_name}} resources and storing them in the designated storage location. Apply the following backup custom resource YAML to create the backup of all application resources that have been labeled for backup. + +The following `Backup` custom resource backs up all the {{site.data.reuse.es_name}} custom resource configurations, cluster certificates, and Kafka user secrets from the installed namespace: + + ```yaml + apiVersion: velero.io/v1 + kind: Backup + metadata: + name: + namespace: openshift-adp + spec: + ttl: 720h0m0s + defaultVolumesToRestic: false + includeClusterResources: true + storageLocation: + includedNamespaces: + - + - openshift-marketplace + orLabelSelectors: + - matchExpressions: + - key: backup.eventstreams.ibm.com/component + operator: In + values: + - catalogsource + - operatorgroup + - subscription + - eventstreams + - kafkaconnect + - kafkatopic + - kafkauser + - kafkabridge + - kafkaconnector + - kafkarebalance + - matchLabels: + eventstreams.ibm.com/component-type: certificate-authority + - matchLabels: + eventstreams.ibm.com/kind: KafkaUser + ``` + +Where: + +- `` is the name of the backup that you are creating. +- `` is the name of the namespaces to be backed up. You can back up more than one namespace at the same time. +- `` is the name of the `backupstoragelocation` instance that was created when you installed the `DataProtectionApplication` instance. You can get the name of your storage location by running the following command in your cluster: + + ```shell + oc get backupStorageLocations -n openshift-adp + ``` + +After the backup is successfully completed, the status of your backup instance will be `completed`. + +## Restoring configurations of {{site.data.reuse.es_name}} + +{{site.data.reuse.es_name}} configurations that are [backed up](#backing-up-configurations-of-event-streams) can be restored within your existing cluster or in a new cluster by using the restore custom resource. + +### In-place recovery + +In-place recovery in {{site.data.reuse.es_name}} refers to recovering configurations and metadata within the same {{site.data.reuse.openshift_short}} cluster without the need of setting up a secondary cluster for moving data and configurations. Such recovery is only possible if the [persistent storage](../planning/#planning-for-persistent-storage) configured for Kafka and ZooKeeper remains intact and can be reused by the restored configuration. + +To restore your {{site.data.reuse.es_name}} configurations within your cluster after you have a successful backup of {{site.data.reuse.es_name}} configurations in your storage location, apply the following YAML to restore {{site.data.reuse.es_name}} configurations from the previously created backup: + + ```yaml + apiVersion: velero.io/v1 + kind: Restore + metadata: + name: + namespace: openshift-adp + spec: + backupName: + includeClusterResources: true + existingResourcePolicy: update + hooks: {} + includedNamespaces: + - + - openshift-marketplace + itemOperationTimeout: 1h0m0s + orLabelSelectors: + - matchExpressions: + - key: backup.eventstreams.ibm.com/component + operator: In + values: + - catalogsource + - operatorgroup + - subscription + - eventstreams + - kafkaconnect + - kafkatopic + - kafkauser + - kafkabridge + - kafkaconnector + - kafkarebalance + - matchLabels: + eventstreams.ibm.com/component-type: certificate-authority + - matchLabels: + eventstreams.ibm.com/kind: KafkaUser + ``` + +Where: + +- `` is the name of your restore YAML. +- `` is the name of backup that you created earlier. +- `` is the name of namespaces to be restored. You can restore more than one namespace at the same time. + +Wait for the restore process to be completed. After the status of your restore instance is `completed`, all the application instances are restored. + + +### Verifying backup content + +You can use [the Velero CLI](https://velero.io/docs/v1.8/basic-install/#install-the-cli){:target="_blank"} to verify the contents of the backup and ensure that all the required resources are backed up. To verify your backup by using the Velero CLI, run the following command: + +```shell +velero backup describe --details -n openshift-adp +``` + +Running this command displays the list of resources that are backed up under the `Resource List` section. For example: + +```shell +Resource List: + apiextensions.k8s.io/v1/CustomResourceDefinition: + - eventstreams.eventstreams.ibm.com + - eventstreamsgeoreplicators.eventstreams.ibm.com + - kafkabridges.eventstreams.ibm.com + - kafkaconnectors.eventstreams.ibm.com + - kafkaconnects.eventstreams.ibm.com + - kafkamirrormaker2s.eventstreams.ibm.com + - kafkanodepools.eventstreams.ibm.com + - kafkarebalances.eventstreams.ibm.com + - kafkas.eventstreams.ibm.com + - kafkatopics.eventstreams.ibm.com + - kafkausers.eventstreams.ibm.com + - strimzipodsets.core.eventstreams.ibm.com + eventstreams.ibm.com/v1beta2/EventStreams: + - /minimal-prod + eventstreams.ibm.com/v1beta2/KafkaTopic: + - /bookings + - /orders + - /reservations + - /transactions + eventstreams.ibm.com/v1beta2/KafkaUser: + - /adminuser + - /reservationstarter + - /user2 + - /user3 + - /user4 + operators.coreos.com/v1/OperatorGroup: + - /-9sqtt + operators.coreos.com/v1alpha1/CatalogSource: + - openshift-marketplace/ibm-operator-catalog + operators.coreos.com/v1alpha1/Subscription: + - /ibm-eventstreams + v1/Namespace: + - + - openshift-marketplace + v1/Secret: + - /adminuser + - /minimal-prod-clients-ca + - /minimal-prod-clients-ca-cert + - /minimal-prod-cluster-ca + - /minimal-prod-cluster-ca-cert + - /minimal-prod-ibm-es-ac-reg + - /minimal-prod-ibm-es-georep-source-user + - /minimal-prod-ibm-es-kafka-user + - /reservationstarter + - /user2 + - /user3 + - /user4 +``` + +### Restoring in a new cluster + +Follow the steps to restore your {{site.data.reuse.es_name}} configurations in a new cluster. + +1. As mentioned in [before you begin](#before-you-begin), install the OADP operator and the corresponding `DataProtectionApplication` instance in your new cluster and point to the same storage location. + +2. After the `DataProtectionApplication` instance is created, the list of all the backups that are created in your old cluster is displayed in the new cluster. + +3. Apply the [restore YAML file](#in-place-recovery) and wait for the restoration process to be completed. + +4. After restoration is completed, verify that all the {{site.data.reuse.es_name}} resources are restored and functioning as expected. + +### Enable reconciliation of the Kafka users + +Enable the reconciliation of all Kafka users after restoring your {{site.data.reuse.es_name}} configurations in a new cluster by running the following command: + +```shell +oc annotate KafkaUser -n --all strimzi.io/pause-reconciliation- +``` + +## Migrating topic data + +The backup and restoration process by using OADP is primarily focused on safeguarding and recovering {{site.data.reuse.es_name}} configurations. However, for comprehensive disaster recovery, it is essential to extend these efforts to include the backup and restoration of topic data. + +In {{site.data.reuse.es_name}}, data replication can be effectively achieved through the use of MirrorMaker 2.0. + diff --git a/_es_11.5/02-installing/09-disaster-recovery.md b/_es_11.5/02-installing/09-disaster-recovery.md new file mode 100644 index 00000000..367ba5af --- /dev/null +++ b/_es_11.5/02-installing/09-disaster-recovery.md @@ -0,0 +1,72 @@ +--- +title: "Configuring disaster recovery topologies" +excerpt: "Find out more about deploying and configuring Event Streams for disaster recovery." +categories: installing +slug: disaster-recovery +toc: true +--- + +You cam use MirrorMaker 2.0 to mirror topics from one {{site.data.reuse.es_name}} cluster to another. + +You can configure {{site.data.reuse.es_name}} for disaster recovery (DR) by using multiple {{site.data.reuse.es_name}} instances in different locations. Two key topologies to consider as starting points for planning a DR solution suitable for your business are: + +- Active-Passive +- Active-Active + +## Active-Passive topology + +In the Active-Passive topology, there are two Kafka clusters; one active and one passive, which are in two different locations. The active cluster is the primary cluster where the data is processed, while the passive cluster serves as a backup for disaster recovery purposes. + +MirrorMaker 2.0 provides unidirectional data replication between the 2 Kafka clusters. This is one way to mitigate the risks that are associated with any outages that might cause data loss and interrupt ongoing business activities. + +![Active - passive topology diagram.]({{ 'images' | relative_url }}/ActivePassive.png "Diagram that shows how active-passive replication works.") + +This approach is typically used when there is a need for data recovery or business continuity because the secondary {{site.data.reuse.es_name}} cluster provides a backup to the primary cluster if there is a disaster or outage. Unidirectional replication ensures that the secondary cluster has a copy of the data, but this {{site.data.reuse.es_name}} instance is not directly accessed by producer and consumer applications. + +If there is a failure in the active cluster, the passive cluster can be activated to take over processing of data. To implement an Active-Passive topology, MirrorMaker 2.0 must be configured on the {{site.data.reuse.openshift}} cluster hosting the passive cluster that will allow the data from the primary active cluster to be copied across to the secondary cluster. + +In an Active-Passive topology, the primary {{site.data.reuse.es_name}} cluster, Cluster-1, is hosting Topic-A so all producer and consumer applications for Topic-A are connected to Cluster-1. + +Cluster-2 maintains a backup of the data from Cluster-1. This backup is ready so that if there is an outage of the active cluster the system can be restarted with little data loss. The backup cluster is not accessed directly by any of the producer or consumer applications. + +![Active passive with application.]({{ 'images' | relative_url }}/ActivePassiveApplication.png "Active passive with application") + +If there is business need to always have access to the data, some applications can be connected to the secondary cluster during an outage to access the backed-up data while the primary cluster might be inaccessible. + +The number of {{site.data.reuse.es_name}} MirrorMaker 2.0 instances that are needed depends mainly on the requirements of your DR solution. For example: + +- How much data are you putting through your primary cluster? +- How critical is the access to each topic? +- How much tolerance is there for the replication lag across your system? + +For more information about how to set up MirrorMaker 2.0 on your cluster, see the article [Demystifying Kafka MirrorMaker 2: Use cases and architecture](https://developers.redhat.com/articles/2023/11/13/demystifying-kafka-mirrormaker-2-use-cases-and-architecture){:target="_blank"}. + +## Active-Active topology + +In the Active-Active topology, there are two {{site.data.reuse.es_name}} clusters that are both active, and applications produce and consume from both the active clusters. The Active-Active topology can be useful for scenarios where high availability across different geographical regions and having a cluster always active is critical for business continuity. + +MirrorMaker 2.0 is configured in both directions between the two active clusters. Each cluster process data independently for the applications that are directly linked to it, while also serving as a backup for the other cluster. This helps to ensure that a failure in one of the clusters does not completely stop business processes. Furthermore, historical data from the disrupted cluster can still be used by applications that are connected to the remaining active cluster. + +![Active - active topology diagram.]({{ 'images' | relative_url }}/ActiveActive.png "Diagram that shows architecture of an active-active topology.") + +In the previous diagram, there is Topic-A on Cluster-1 and it is being replicated to a copy called `Cluster-1.Topic-A` on to Cluster-2 and vice versa. + +With the Active-Active topology, you can achieve: + +- Availability of data across different geographies +- Lower latency by connecting clients to a more local {{site.data.reuse.es_name}} +- Disruption isolation by distributing data processing across multiple clusters in different availability zones (for more information, see [considerations for multizone deployments](../../installing/multizone-considerations/)). + +Further to the DR capabilities offered by the Active-Active topology, a consumer application that is connected to Cluster-1 can be configured to read data that is produced to different clusters as if it were coming from one topic. You can achieve this by setting the topics list to use a name with a wildcard character like `*Topic-1`. This means that the client reads from both Topic-1 and `Cluster-2.Topic-1` and all messages are processed regardless of where they were originally produced. + +![Single topic across 2 instances diagram.]({{ 'images' | relative_url }}/SingleTopic2Clusters1.png "Diagram that shows a single consumer for *topic-1.") + +In the previous diagram, consumers that are connected to Cluster-1 can access the data from Cluster-2 without having to connect directly to Cluster-2. + +Active-Active topology can be useful for: + +- Clusters that are in different geographic locations +- Applications that care about the location where the data comes from +- Applications that are independent of location and prefer a broader view of the data across different {{site.data.reuse.es_name}} clusters. + +![Single topic across 2 instances diagram.]({{ 'images' | relative_url }}/SingleTopic2Clusters2.png "Diagram that shows consumer with conceptual topic across 2 clusters.") diff --git a/_es_11.5/02-installing/10-moving-from-oss-kafka.md b/_es_11.5/02-installing/10-moving-from-oss-kafka.md new file mode 100644 index 00000000..83dbd95b --- /dev/null +++ b/_es_11.5/02-installing/10-moving-from-oss-kafka.md @@ -0,0 +1,38 @@ +--- +title: "Migrating from open-source Apache Kafka to Event Streams" +excerpt: "Learn about the steps required to move from an open-source Kafka implementation to using Event Streams." +categories: installing +slug: moving-from-oss-kafka +toc: true +--- + +If you are using open-source Apache Kafka as your event-streaming platform, you can move to {{site.data.reuse.es_name}} and [benefit](../../about/overview/) from its features and enterprise-level support. + +## Prerequisites + + +Ensure you have an {{site.data.reuse.es_name}} deployment available. See the instructions for [installing]({{ 'installpagedivert' | relative_url }}) on your platform. + +For many of the tasks, you can use the [Kafka console tools](https://kafka.apache.org/quickstart){:target="_blank"}. Many of the console tools work with {{site.data.reuse.es_name}}, as described in the [using console tools](../../getting-started/using-kafka-console-tools/) topic. + +## Create the required topics in {{site.data.reuse.es_name}} + +In {{site.data.reuse.es_name}}, create the same set of topics that have been deployed in the open-source Kafka cluster. + +To list these topics, run the following Kafka console tool: + +`./kafka-topics.sh --bootstrap-server : --describe` + +For each existing topic, [create a new topic](../../getting-started/creating-topics) in {{site.data.reuse.es_name}} with the same name. Ensure you use the same partition and replica settings, as well as any other non-default settings that were applied to the existing topic in your open-source Kafka instance. + +## Change producer configuration + +Change the configuration for applications that produce messages to your open-source Kafka cluster to connect to {{site.data.reuse.es_name}} instead as described in [connecting clients](../../getting-started/connecting). + +If you are using the Kafka console tools, see the instructions for the example console producer in [using the console tools](../../getting-started/using-kafka-console-tools/#using-the-console-tools-with-ibm-event-streams) to change where the messages are produced to. + +## Change consumer configuration + +Change the configuration for applications that consume messages from your open-source Kafka cluster to connect to {{site.data.reuse.es_name}} instead as described in [connecting clients](../../getting-started/connecting). + +If you are using the Kafka console tools, see the instructions for the example console consumer in [using the console tools](../../getting-started/using-kafka-console-tools/#using-the-console-tools-with-ibm-event-streams) to change where messages are consumed from. diff --git a/_es_11.5/02-installing/11-uninstalling.md b/_es_11.5/02-installing/11-uninstalling.md new file mode 100644 index 00000000..23f71cd3 --- /dev/null +++ b/_es_11.5/02-installing/11-uninstalling.md @@ -0,0 +1,244 @@ +--- +title: "Uninstalling" +excerpt: "Uninstalling Event Streams." +categories: installing +slug: uninstalling +toc: true +--- + +You can remove the {{site.data.reuse.es_name}} from your platform as follows: + + +## Uninstalling an {{site.data.reuse.es_name}} instance by using the CLI +You can delete an {{site.data.reuse.es_name}} installation using the Kubernetes command-line tool (`kubectl`): + +1. {{site.data.reuse.cncf_cli_login}} +2. Ensure you are using the namespace where your {{site.data.reuse.es_name}} instance is located: + + ```shell + kubectl config set-context --current --namespace= + ``` + +3. Run the following command to display the {{site.data.reuse.es_name}} instances: + + ```shell + kubectl get eventstreams -n + ``` + +4. Run the following command to delete your instance: + + ```shell + kubectl delete eventstreams -n + ``` + +### Check uninstallation progress +Run the following command to check the progress: + +```shell +kubectl get pods --selector app.kubernetes.io/instance= +``` + +Pods will initially display a **STATUS** `Terminating` and then be removed from the output as they are deleted. + +```shell +$ kubectl get pods --selector app.kubernetes.io/instance=minimal-prod +> +NAME READY STATUS RESTARTS AGE +minimal-prod-entity-operator-77dfff7c79-cnrx5 0/2 Terminating 0 5h35m +minimal-prod-ibm-es-admapi-b49f976c9-xhsrv 0/1 Terminating 0 5h35m +minimal-prod-ibm-es-recapi-6f6bd784fc-jvf9z 0/1 Terminating 0 5h35m +minimal-prod-ibm-es-ac-reg-6dffdb54f9-dfdpl 0/3 Terminating 0 5h35m +minimal-prod-ibm-es-ui-5dd7496dbc-qks7m 0/2 Terminating 0 5h35m +minimal-prod-kafka-0 2/2 Terminating 0 5h36m +minimal-prod-zookeeper-0 0/2 Terminating 0 5h37m +``` + +### Removing persistence resources +If you had enabled persistence for the {{site.data.reuse.es_name}} instance but set the `deleteClaim` storage property to `false`, you will need to manually remove the associated Persistent Volumes (PVs) and Persistent Volume Claims (PVCs) that were created at installation time. + +The `deleteClaim` property is configured in the `EventStreams` custom resource and can be set to `true` during installation to ensure the PVs and PVCs are automatically removed when the instance is deleted. + +For Kafka and ZooKeeper, this property can be found as follows: + - `spec.strimziOverrides.kafka.storage.deleteClaim` + - `spec.strimziOverrides.zookeeper.storage.deleteClaim` + +For other components, this property can be found as follows: +- `spec..storage.deleteClaim` + +**Important:** This change will cause data to be removed during an upgrade. + +For example, to configure automatic deletion for the Kafka storage when uninstalling: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + # ... + storage: + type: persistent-claim + # ... + deleteClaim: true +``` + +To remove any remaining storage, delete the PVCs first then delete any remaining PVs. + +Delete the Persistent Volume Claims (PVCs): + +1. Run the following command to list the remaining PVCs associated with the deleted instance: + + ```shell + kubectl get pvc --selector app.kubernetes.io/instance= + ``` + +2. Run the following to delete a PVC: + + ```shell + kubectl delete pvc + ``` + +Delete remaining Persistent Volumes (PVs): + +1. Run the following command to list the remaining PVs: + + ```shell + kubectl get pv + ``` + +2. Run the following command to delete any PVs that were listed in the **Volume** column of the deleted PVCs. + + ```shell + kubectl delete pv + ``` + +**Note:** Take extreme care to select the correct PV name to ensure you do not delete storage associated with a different application instance. + + +## Uninstalling an {{site.data.reuse.es_name}} instance by using the OpenShift web console +To delete an {{site.data.reuse.es_name}} instance on {{site.data.reuse.openshift_short}}: + +1. {{site.data.reuse.openshift_ui_login}} +2. {{site.data.reuse.task_openshift_navigate_installed_operators}} +3. {{site.data.reuse.task_openshift_select_operator}} +4. In the **Operator Details** panel, select the **Event Streams** tab to show the {{site.data.reuse.es_name}} instances in the selected namespace. +5. Click ![More options icon]({{ 'images' | relative_url }}/more_options.png "More options icon at end of each row."){:height="30px" width="15px"} **More options** next to the instance to be deleted to open the actions menu. +6. Click the **Delete EventStreams** menu option to open the confirmation panel. +7. Check the namespace and instance name and click **Delete** to shutdown the associated pods and delete the instance. + +### Check uninstallation progress +1. {{site.data.reuse.openshift_ui_login}} +2. Expand the **Workloads** dropdown and select **Pods** to open the **Pods** dashboard. +3. Click **Select All Filters** to display pods in any state. +4. Enter the name of the {{site.data.reuse.es_name}} instance being deleted in the **Filter by name** box. +5. Wait for all the {{site.data.reuse.es_name}} pods to be displayed as **Terminating** and then be removed from the list. + +### Removing persistence resources +If you had enabled persistence for the {{site.data.reuse.es_name}} instance but set the `deleteClaim` storage property to `false`, you will need to manually remove the associated Persistent Volumes (PVs) and Persistent Volume Claims (PVCs) that were created at installation time. + +The `deleteClaim` property is configured in the `EventStreams` custom resource and can be set to `true` during installation to ensure the PVs and PVCs are automatically removed when the instance is deleted. + +For Kafka and ZooKeeper, this property can be found as follows: + - `spec.strimziOverrides.kafka.storage.deleteClaim` + - `spec.strimziOverrides.zookeeper.storage.deleteClaim` + +For other components, this property can be found as follows: +- `spec..storage.deleteClaim` + +**Important:** This change will cause data to be removed during an upgrade. + +For example, to configure automatic deletion for the Kafka storage when uninstalling: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + # ... + storage: + type: persistent-claim + # ... + deleteClaim: true +``` + +To remove any remaining storage, delete the PVCs first then delete any remaining PVs. + +Delete the Persistent Volume Claims (PVCs): + +1. {{site.data.reuse.openshift_ui_login}} +2. Expand the **Storage** dropdown and select **Persistent Volume Claims** to open the **Persistent Volume Claims** page. +3. In the **Project** dropdown select the required namespace. +4. Click **Select All Filters** to display PVCs in any state. +5. Enter the name of the {{site.data.reuse.es_name}} instance in the **Filter by name** box. +6. For each PVC to be deleted, make a note of the **Persistent Volume** listed for that PVC and then click ![More options icon]({{ 'images' | relative_url }}/more_options.png "More options icon at end of each row."){:height="30px" width="15px"} **More options** to open the actions menu. +7. Click the **Delete Persistent Volume Claim** menu option to open the confirmation panel. +8. Check the PVC name and namespace, then click **Delete** to remove the PVC. + +Delete remaining Persistent Volumes (PVs): + +1. {{site.data.reuse.openshift_ui_login}} +2. Expand the **Storage** dropdown and select **Persistent Volumes** to open the **Persistent Volumes** page. +3. In the **Project** dropdown select the required namespace. +4. For each PV you made a note of when deleting PVCs, click ![More options icon]({{ 'images' | relative_url }}/more_options.png "More options icon at end of each row."){:height="30px" width="15px"} **More options** to open the actions menu. +5. Click the **Delete Persistent Volume** menu option to open the confirmation panel. +6. Check the PV name and click **Delete** to remove the PV. + +**Note:** Take extreme care to select the correct PV name to ensure you do not delete storage associated with a different application instance. + +## Uninstalling an {{site.data.reuse.es_name}} operator on {{site.data.reuse.openshift_short}} +To delete an {{site.data.reuse.es_name}} operator: + +1. {{site.data.reuse.openshift_ui_login}} +2. Expand **Operators** and click **Installed Operators**. +3. In the **Project** dropdown select the required namespace. For cluster-wide operators, select the `openshift-operators` project. +4. Click ![More options icon]({{ 'images' | relative_url }}/more_options.png "More options icon at end of each row."){:height="30px" width="15px"} **More options** next to the {{site.data.reuse.es_name}} operator to be deleted to open the actions menu. +5. Click the **Uninstall Operator** menu option to open the confirmation panel. +6. Check the namespace and operator name, then click **Remove** to uninstall the operator. + +The {{site.data.reuse.es_name}} Custom Resource Definitions (CRDs) are not deleted automatically on {{site.data.reuse.openshift_short}}. You must manually delete any CRDs that you do not want: + +1. {{site.data.reuse.openshift_ui_login}} +2. Expand **Administration** and click **Custom Resource Definitions**. +3. Enter `eventstreams` in the **Filter by name** box to filter the CRDs associated with {{site.data.reuse.es_name}}. +4. Click ![More options icon]({{ 'images' | relative_url }}/more_options.png "More options icon at end of each row."){:height="30px" width="15px"} **More options** next to the CRD to be deleted to open the actions menu. +5. Click the **Delete Custom Resource Definition** menu option to open the confirmation panel. +6. Check the name of the CRD and click **Delete** to remove the CRD. + + +## Uninstalling an {{site.data.reuse.es_name}} operator on other Kubernetes platforms + +To delete an {{site.data.reuse.es_name}} operator: + +1. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to select the namespace the operator is installed on: + + ```shell + kubectl config set-context --current --namespace= + ``` + +3. Run the following command to list the helm releases installed on that namespace: + + ```shell + helm list + ``` + +4. Find the {{site.data.reuse.es_name}} operator release, it should have the chart as `ibm-eventstreams-operator-` + + **Important:** When multiple {{site.data.reuse.es_name}} operators are installed on the same cluster, all the operators share the same custom resource definitions (CRDs). Do not delete these global resources that other operators on the cluster depend upon. The {{site.data.reuse.es_name}} operator that was installed first on the cluster must not be uninstalled if you have other {{site.data.reuse.es_name}} operators deployed on the same cluster. + +5. Run the following command to uninstall the {{site.data.reuse.es_name}} operator and the {{site.data.reuse.es_name}} Custom Resource Definitions (CRDs): + + ```shell + helm uninstall + ``` + +## Uninstalling {{site.data.reuse.icpfs}} on OpenShift + +If you have {{site.data.reuse.icpfs}}, see the [{{site.data.reuse.fs}} documentation](https://www.ibm.com/docs/en/cloud-paks/foundational-services/3.23?topic=online-uninstalling-foundational-services){:target="_blank"} for information about uninstalling it. diff --git a/_es_11.5/02-installing/12-upgrading.md b/_es_11.5/02-installing/12-upgrading.md new file mode 100644 index 00000000..8c7d3db9 --- /dev/null +++ b/_es_11.5/02-installing/12-upgrading.md @@ -0,0 +1,396 @@ +--- +title: "Upgrading" +excerpt: "Upgrade your installation to the latest version." +categories: installing +slug: upgrading +toc: true +--- + +Upgrade your {{site.data.reuse.es_name}} installation as follows. The {{site.data.reuse.es_name}} operator handles the upgrade of your {{site.data.reuse.es_name}} instance. + +## Upgrade paths + +You can upgrade {{site.data.reuse.es_name}} to the [latest 11.5.x version]({{ 'support/matrix/#event-streams' | relative_url }}) directly from any 11.4.x version by using operator version 3.5.x. The upgrade procedure depends on whether you are upgrading to a major, minor, or patch level version, and what your catalog source is. + +If you are upgrading from {{site.data.reuse.es_name}} version 11.3.x or earlier, you must first [upgrade your installation to 11.4.x]({{ 'es/es_11.4' | relative_url }}/installing/upgrading/) and then follow these instructions to upgrade from 11.4.x to 11.5.x. + +- On OpenShift, you can upgrade to the latest version by using operator channel v3.5. Review the general upgrade [prerequisites](#prerequisites) before following the instructions to [upgrade on OpenShift](#upgrading-on-the-openshift-container-platform). + + **Note:** If your operator upgrades are set to automatic, patch level upgrades are completed automatically. This means that the {{site.data.reuse.es_name}} operator is upgraded to the latest 3.5.x version when it is available in the catalog, and your {{site.data.reuse.es_name}} instance is then also automatically upgraded, unless you [set a schedule for the upgrade](#scheduling-the-upgrade-of-an-instance) by pausing the reconciliation. + +- On other Kubernetes platforms, you must update the Helm repository for any level version update (any digit update: major, minor, or patch), and then upgrade by using the Helm chart. Review the general upgrade [prerequisites](#prerequisites) before following the instructions to [upgrade on other Kubernetes platforms](#upgrading-on-other-kubernetes-platforms-by-using-helm). + +## Prerequisites + +- The images for {{site.data.reuse.es_name}} release 11.5.x are available in the IBM Cloud Container Registry. Ensure you redirect your catalog source to use `icr.io/cpopen` as described in [Implementing ImageContentSourcePolicy to redirect to the IBM Container Registry](https://www.ibm.com/docs/en/cloud-paks/1.0?topic=clusters-migrating-from-docker-container-registry#implementing-imagecontentsourcepolicy-to-redirect-to-the-ibm-container-registry){:target="_blank"}. + +- Ensure that you have installed a supported container platform and system. For supported container platform versions and systems, see the [support matrix]({{ 'support/matrix/#event-streams' | relative_url }}). + +- To upgrade successfully, your {{site.data.reuse.es_name}} instance must have more than one ZooKeeper node or have persistent storage enabled. If you upgrade an {{site.data.reuse.es_name}} instance with a single ZooKeeper node that has ephemeral storage, all messages and all topics will be lost and both ZooKeeper and Kafka pods will move to an error state. To avoid this issue, increase the number of ZooKeeper nodes before upgrading as follows: + + + ```yaml + apiVersion: eventstreams.ibm.com/v1beta2 + kind: EventStreams + metadata: + name: example-pre-upgrade + namespace: myproject + spec: + # ... + strimziOverrides: + zookeeper: + replicas: 3 + ``` + +- If you [installed the {{site.data.reuse.es_name}} operator]({{ 'installpagedivert' | relative_url }}) to manage instances of {{site.data.reuse.es_name}} in any namespace (one per namespace), then you might need to control when each of these instances is upgraded to the latest version. You can control the updates by pausing the reconciliation of the instance configuration as described in the following sections. + +- If you are running {{site.data.reuse.es_name}} as part of {{site.data.reuse.cp4i}}, ensure you meet the following requirements: + + - Follow the [upgrade steps for {{site.data.reuse.cp4i}}](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=upgrading){:target="_blank"} before upgrading {{site.data.reuse.es_name}}. + - If you are planning to configure {{site.data.reuse.es_name}} with Keycloak, ensure you have the {{site.data.reuse.cp4i}} 2023.4.1 (operator version 7.2.0) or later [installed](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=installing){:target="_blank"}, including the required dependencies. + + **Note:** After upgrading {{site.data.reuse.es_name}} to the latest version, if you are changing authentication type from IAM to Keycloak, modify the `EventStreams` custom resource as described in [post-upgrade tasks](#configure-your-instance-to-use-keycloak). + +- Ensure all applications connecting to your instance of {{site.data.reuse.es_name}} that use the schema registry are using Apicurio client libraries version 2.6.2 or later before migrating. + +**Note:** There is no downtime during the {{site.data.reuse.es_name}} upgrade. The Kafka pods are rolled one at a time, so a Kafka instance will always be present to serve traffic. However, if the number of brokers you have matches the `min.insync.replicas` value set for any of your topics, then that topic will be unavailable to write to while the Kafka pods are rolling. + +### Scheduling the upgrade of an instance + +In 11.1.x and later, the {{site.data.reuse.es_name}} operator handles the upgrade of your {{site.data.reuse.es_name}} instance automatically after the operator is upgraded. No additional step is required to change the instance (product) version. + +If your operator manages more than one instance of {{site.data.reuse.es_name}}, you can control when each instance is upgraded by pausing the reconciliation of the configuration settings for each instance, running the upgrade, and then unpausing the reconciliation when ready to proceed with the upgrade for a selected instance. + +#### Pausing reconciliation by using the CLI + + 1. {{site.data.reuse.cncf_cli_login}} + 2. To apply the annotation first to the `EventStreams` and then to the `Kafka` custom resource, run the following command, where `` is either `EventStreams` or `Kafka`: + + ```shell + kubectl annotate -n eventstreams.ibm.com/pause-reconciliation='true' + ``` + + 3. Follow the steps to [upgrade on OpenShift](#upgrading-on-the-openshift-container-platform). + +#### Unpausing reconciliation by using the CLI + +To unpause the reconciliation and continue with the upgrade of an {{site.data.reuse.es_name}} instance, run the following command to first remove the annotations from the `Kafka` custom resource, and then from the `EventStreams` custom resource, where `` is either `Kafka` or `EventStreams`: + +```shell +kubectl annotate -n eventstreams.ibm.com/pause-reconciliation- +``` + +When the annotations are removed, the configuration of your instance is updated, and the upgrade to the latest version of {{site.data.reuse.es_name}} completes. + +#### Pausing reconciliation by using the OpenShift web console + +1. {{site.data.reuse.openshift_ui_login}} +2. Expand **Operators** in the navigation on the left, and click **Installed Operators**. + + ![Operators > Installed Operators]({{ 'images' | relative_url }}/rhocp_menu_installedoperators.png "Screen capture showing how to select Operators > Installed Operators from navigation menu"){:height="50%" width="50%"} + +3. From the **Project** list, select the namespace (project) the instance is installed in. +4. Locate the operator that manages your {{site.data.reuse.es_name}} instance in the namespace. It is called **{{site.data.reuse.es_name}}** in the **Name** column. Click the **{{site.data.reuse.es_name}}** link in the row. +5. Select the instance you want to pause and click the `YAML` tab. +6. In the `YAML` for the custom resource, add `eventstreams.ibm.com/pause-reconciliation: 'true'` to the `metadata.annotations` field as follows: + + ```yaml + apiVersion: eventstreams.ibm.com/v1beta2 + kind: EventStreams + metadata: + name: + namespace: + annotations: + eventstreams.ibm.com/pause-reconciliation: 'true' + spec: + # ... + ``` + +7. This annotation also needs to be applied to the corresponding `Kafka` custom resource. Expand **Home** in the navigation on the left, click **API Explorer**, and type `Kafka` in the `Filter by kind...` field. Select `Kafka`. +8. From the **Project** list, select the namespace (project) the instance is installed in and click the **Instances** tab. +9. Select the instance with the name `` (the same as the {{site.data.reuse.es_name}} instance). +10. In the `YAML` for the custom resource, add `eventstreams.ibm.com/pause-reconciliation: 'true'` to the `metadata.annotations` field as follows: + + ```yaml + apiVersion: eventstreams.ibm.com/v1beta2 + kind: Kafka + metadata: + name: + namespace: + annotations: + eventstreams.ibm.com/pause-reconciliation: 'true' + ``` + +11. Follow the steps to [upgrade on OpenShift](#upgrading-on-the-openshift-container-platform). + +#### Unpausing reconciliation by using the OpenShift web console + +To unpause the reconciliation and continue with the upgrade of an {{site.data.reuse.es_name}} instance, first remove the annotations from the `Kafka` custom resource, and then from the `EventStreams` custom resource. When the annotations are removed, the configuration of your instance is updated, and the upgrade to the latest version of {{site.data.reuse.es_name}} completes. + +## Upgrading on the {{site.data.reuse.openshift_short}} + +Upgrade your {{site.data.reuse.es_name}} instance running on the {{site.data.reuse.openshift_short}} by using the CLI or web console as follows. + +### Planning your upgrade + +Complete the following steps to plan your upgrade on OpenShift. + +- Determine which Operator Lifecycle Manager (OLM) channel is used by your existing Subscription. You can check the channel you are subscribed to in the [web console](#upgrading-subscription-ui) (see **Update channel** section), or by using the CLI as follows (this is the [subscription created during installation](../installing/#install-the-event-streams-operator)): + + 1. Run the following command to check your subscription details: + + ```shell + oc get subscription + ``` + + 2. Check the `CHANNEL` column for the channel you are subscribed to, for example, v3.4 in the following snippet: + + ``` + NAME PACKAGE SOURCE CHANNEL + ibm-eventstreams ibm-eventstreams ibm-eventstreams-catalog v3.4 + ``` + +- If your existing Subscription does not use the v3.5 channel, your upgrade is a change in a minor version. Complete the following steps to upgrade: + 1. Ensure the [catalog source for new version is available](#making-new-catalog-source-available). + 2. Change your Subscription to the `v3.5` channel by using [the CLI](#upgrading-subscription-by-using-the-cli) or [the web console](#upgrading-subscription-ui). The channel change will upgrade your operator, and then the operator will upgrade your {{site.data.reuse.es_name}} instance automatically. + +- If your existing Subscription is already on the v3.5 channel, your upgrade is a change to the patch level (third digit) only. [Make the catalog source for your new version available](#making-new-catalog-source-available) to upgrade to the latest level. If you installed by using the IBM Operator Catalog with the `latest` label, new versions are automatically available. The operator will upgrade your {{site.data.reuse.es_name}} instance automatically. + +### Making new catalog source available + +Before you can upgrade to the latest version, the catalog source for the new version must be available on your cluster. Whether you have to take action depends on how you set up [version control](../installing/#decide-version-control-and-catalog-source) for your deployment. + +- Latest versions: If your catalog source is the IBM Operator Catalog, latest versions are always available when published, and you do not have to make new catalog sources available. + +- Specific versions: If you used the CASE bundle to install catalog source for a specific previous version, you must download and use a new CASE bundle for the version you want to upgrade to. + - If you previously used the CASE bundle for an online install, [apply the new catalog source](../installing/#adding-specific-versions) to update the `CatalogSource` to the new version. + - If you used the CASE bundle for an offline install that uses a private registry, follow the instructions in [installing offline](../offline/#download-the-case-bundle) to remirror images and update the `CatalogSource` for the new version. + - In both cases, wait for the `status.installedCSV` field in the `Subscription` to update. It eventually reflects the latest version available in the new `CatalogSource` image for the currently selected channel in the `Subscription`: + - In the {{site.data.reuse.openshift_short}} web console, the current version of the operator is displayed under `Installed Operators`. + - If you are using the CLI, check the status of the `Subscription` custom resource, the `status.installedCSV` field shows the current operator version. + + +### Upgrading Subscription by using the CLI + +If you are using the OpenShift command-line interface (CLI), the `oc` command, complete the steps in the following sections to upgrade your {{site.data.reuse.es_name}} installation. + +1. {{site.data.reuse.openshift_cli_login}} +2. Ensure the required {{site.data.reuse.es_name}} Operator Upgrade Channel is available: + + ```shell + oc get packagemanifest ibm-eventstreams -o=jsonpath='{.status.channels[*].name}' + ``` + +2. Change the subscription to move to the required update channel, where `vX.Y` is the required update channel (for example, `v3.5`): + + ```shell + oc patch subscription -n ibm-eventstreams --patch '{"spec":{"channel":"vX.Y"}}' --type=merge + ``` + +All {{site.data.reuse.es_name}} pods that need to be updated as part of the upgrade will be gracefully rolled. Where required, ZooKeeper pods will roll one at a time, followed by Kafka brokers rolling one at a time. + +### Upgrading Subscription by using the web console {#upgrading-subscription-ui} + +If you are using the {{site.data.reuse.openshift_es_name}} web console, complete the steps in the following sections to upgrade your {{site.data.reuse.es_name}} installation. + +1. {{site.data.reuse.openshift_ui_login}} +2. Expand **Operators** in the navigation on the left, and click **Installed Operators**. + + ![Operators > Installed Operators]({{ 'images' | relative_url }}/rhocp_menu_installedoperators.png "Screen capture showing how to select Operators > Installed Operators from navigation menu"){:height="50%" width="50%"} +3. From the **Project** list, select the namespace (project) the instance is installed in. +4. Locate the operator that manages your {{site.data.reuse.es_name}} instance in the namespace. It is called **{{site.data.reuse.es_name}}** in the **Name** column. Click the **{{site.data.reuse.es_name}}** link in the row. +4. Click the **Subscription** tab to display the **Subscription details** for the {{site.data.reuse.es_name}} operator. +5. Click the version number link in the **Update channel** section (for example, **v3.4**). The **Change Subscription update channel** dialog is displayed, showing the channels that are available to upgrade to. +6. Select **v3.5** and click the **Save** button on the **Change Subscription Update Channel** dialog. + +All {{site.data.reuse.es_name}} pods that need to be updated as part of the upgrade will be gracefully rolled. Where required, ZooKeeper pods will roll one at a time, followed by Kafka brokers rolling one at a time. + +**Note:** The number of containers in each Kafka broker will reduce from 2 to 1 as the TLS-sidecar container will be removed from each broker during the upgrade process. + + + +## Upgrading on other Kubernetes platforms by using Helm + +If you are running {{site.data.reuse.es_name}} on Kubernetes platforms that support the Red Hat Universal Base Images (UBI) containers, you can upgrade {{site.data.reuse.es_name}} by using the Helm chart. + +### Planning your upgrade + +Complete the following steps to plan your upgrade on other Kubernetes platforms. + +- Determine the chart version for your existing deployment: + + 1. Change to the namespace where your {{site.data.reuse.es_name}} instance is installed: + + ```shell + kubectl config set-context --current --namespace= + ``` + + 2. Run the following command to check what version is installed: + + ```shell + helm list + ``` + + 3. Check the version installed in the `CHART` column, for example, `-3.4.0` in the following snippet: + + ``` + NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION + ibm-eventstreams es 1 2023-11-20 11:49:27.221411789 +0000 UTC deployed ibm-eventstreams-operator-3.4.0 3.4.0 + ``` + +- Check the latest chart version that you can upgrade to: + + 1. {{site.data.reuse.cncf_cli_login}} + 2. Add the [IBM Helm repository](https://github.com/IBM/charts/tree/master/repo/ibm-helm){:target="_blank"}: + + ```shell + helm repo add ibm-helm https://raw.githubusercontent.com/IBM/charts/master/repo/ibm-helm + ``` + + 3. Update the Helm repository: + + ```shell + helm repo update ibm-helm + ``` + + 4. Check the version of the chart you will be upgrading to is the intended version: + + ```shell + helm show chart ibm-helm/ibm-eventstreams-operator + ``` + + Check the `version:` value in the output, for example: `version: 3.5.0` + +- If the chart version for your existing deployment is earlier than 3.4.x, you must first [upgrade your installation to 11.4.x]({{ 'es/es_11.4' | relative_url }}/installing/upgrading/) and then follow these instructions to upgrade to chart version 3.5.x. + +- If your existing installation is in an offline environment, you must carry out the steps in the offline install instructions to [download the CASE bundle](../offline/#download-the-case-bundle) and [mirror the images](../offline/#mirror-the-images) for the new version you want to upgrade to, before running any `helm` commands. + +- Complete the steps in [Helm upgrade](#upgrading-by-using-helm) to update your Custom Resource Definitions (CRDs) and operator charts to the latest version. The operator will then upgrade your {{site.data.reuse.es_name}} instance automatically. + +### Upgrading by using Helm + +You can upgrade your {{site.data.reuse.es_name}} on other Kubernetes platforms by using Helm. + +To upgrade {{site.data.reuse.es_name}} to the latest version, run the following command: + +```shell +helm upgrade \ + ibm-helm/ibm-eventstreams-operator \ +-n \ +--set watchAnyNamespace= +--set previousVersion= +``` + +Where: + +- `` is the name you provide to identify your operator. +- `` is the name of the namespace where you want to install the operator. +- `watchAnyNamespace=` determines whether the operator manages instances of {{site.data.reuse.es_name}} in any namespace or only a single namespace (default is `false` if not specified). For more information, see [choosing operator installation mode](../installing-on-kubernetes/#choosing-operator-installation-mode). +- `` is the version of the Helm chart being upgraded from. For example, if your Helm chart version is 3.4.0, set the field as: `--set previousVersion=3.4.0`. You can retrieve the version of your existing Helm chart by running the following command: + + ```shell + helm list --filter -n -o json | jq '.[0].app_version' + ``` + +## Post-upgrade tasks + +### Enable collection of producer metrics + +In {{site.data.reuse.es_name}} version 11.0.0 and later, a Kafka Proxy handles gathering metrics from producing applications. The information is displayed in the [**Producers** dashboard](../../administering/topic-health/). The proxy is optional and is not enabled by default. To enable metrics gathering and have the information displayed in the dashboard, [enable the Kafka Proxy](../../installing/configuring/#enabling-collection-of-producer-metrics). + +### Enable metrics for monitoring + +To display metrics in the monitoring dashboards of the {{site.data.reuse.es_name}} UI: + + +- If you are running {{site.data.reuse.es_name}} on the {{site.data.reuse.openshift_short}}, complete the following steps to enable the [dashboard](../../administering/cluster-health#viewing-the-preconfigured-dashboard): + + 1. Ensure that you [enable](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=administering-enabling-openshift-container-platform-monitoring){:target="_blank"} the monitoring stack. + + 1. To create a `ClusterRoleBinding` in the next step, obtain the ServiceAccount name for your instance. The ServiceAccount is named `-ibm-es-admapi`. For example, `authorized-instance-ibm-es-admapi` + + 1. Run the following command: + + ```shell + oc adm policy add-cluster-role-to-user cluster-monitoring-view -z -n + ``` + + Where `` is the ServiceAccount name for your instance that you obtained in the previous step. + +- If you are running {{site.data.reuse.es_name}} on other Kubernetes platforms, you can use any monitoring solution compatible with Prometheus and JMX formats to collect, store, visualize, and set up alerts based on metrics provided by Event Streams. + +### Configure your instance to use Keycloak + +If your existing instance is configured to use IAM and you want to use Keycloak, update your `EventStreams` custom resource as follows: + +1. Remove the `spec.requestIbmServices` section. +2. Set the `adminUI` [authentication type](../../installing/configuring/#configuring-ui-and-cli-security) to `integrationKeycloak`: + + ```yaml + # ... + spec: + # ... + adminUI: + authentication: + - type: integrationKeycloak + ``` + +### Upgrade the Kafka broker protocol version + +After successfully upgrading to {{site.data.reuse.es_name}} by completing all previous steps and verifying the cluster’s behavior and performance, if your {{site.data.reuse.es_name}} instance is configured with a specific version in the `inter.broker.protocol.version`, complete the following steps to upgrade the Kafka brokers to your Kafka version: + +1. In the `spec.strimziOverrides.kafka.config` section of your `EventStreams` custom resource, change the `inter.broker.protocol.version` value to the Kafka version that is supported in your {{site.data.reuse.es_name}} version. For example, if you are running on {{site.data.reuse.es_name}} 11.5.0, set the value to `3.7`. +1. Wait for the Kafka pods to roll. + +### Remove the `apicurio-registry-version` annotation + +Remove the `eventstreams.ibm.com/apicurio-registry-version='>=2.4'` annotation from your {{site.data.reuse.es_name}} custom resource with the following command: + +```shell +oc annotate --namespace EventStreams eventstreams.ibm.com/apicurio-registry-version- +``` + +### Update SCRAM Kafka User permissions + +{{site.data.reuse.es_name}} 11.5.0 and later uses `KafkaTopic` custom resources (CRs) and topic operator for managing topics through {{site.data.reuse.es_name}} UI and CLI. If access to the {{site.data.reuse.es_name}} UI and CLI has been configured with [SCRAM authentication](../../installing/configuring/#configuring-ui-and-cli-security), see the [managing access](../../security/managing-access/#managing-access-to-ui-and-cli-with-scram) to update the `KafkaUser` permissions accordingly. + +### Upgrading an {{site.data.reuse.es_name}} instance that uses Topic Operator + +After upgrading to {{site.data.reuse.es_name}} 11.5.0, perform the following tasks if the {{site.data.reuse.es_name}} instance was configured to use the Topic Operator before upgrading. + +#### Delete the internal topics that are not used anymore + +You can delete the custom resources of the internal topics `strimzi-store-topic` and `strimzi-topic-operator` as they are no longer used. + +```shell +kubectl delete $(kubectl get kt -n -o name | grep strimzi-store-topic) -n \ + && kubectl delete $(kubectl get kt -n -o name | grep strimzi-topic-operator) -n +``` +#### Discontinue management of other internal topics by the Topic Operator + +Internal topics such as `consumer-offsets` and `transaction-state` are used in Kafka but do not need to be managed by the Topic Operator. +In these cases, you can discontinue their management through the Topic Operator first, and then delete their custom resources without deleting the topics. + +For example: + +1. To discontinue management of the internal topics `consumer-offsets` and `transaction-state`, use the following command: + + ```shell + kubectl annotate $(kubectl get kt -n -o name | grep consumer-offsets) strimzi.io/managed="false" -n \ + && kubectl annotate $(kubectl get kt -n -o name | grep transaction-state) strimzi.io/managed="false" -n + ``` +Before proceeding to the next step, ensure that these topics are no longer managed by their custom resource after reconciliation. You can verify this by confirming that the `kafkaTopic` resource is in the `ready` status and the `metadata.generation` value matches the `status.observedGeneration` in the custom resource. + +2. Optionally, after these topics are no longer managed by their `kafkaTopic` resource, delete the corresponding custom resources using the following command: + + ```shell + kubectl delete $(kubectl get kt -n -o name | grep consumer-offsets) -n \ + && kubectl delete $(kubectl get kt -n -o name | grep transaction-state) -n + ``` + +For more information, see [Deleting internal topics used by the operator](https://strimzi.io/docs/operators/0.42.0/deploying#upgrading_from_a_strimzi_version_using_the_bidirectional_topic_operator). + +### Verifying the upgrade + +After the upgrade, verify the status of {{site.data.reuse.es_name}} by using the [CLI](../post-installation/#using-the-openshift-container-platform-cli) or the [UI](../post-installation/#using-the-openshift-container-platform-ui). + diff --git a/_es_11.5/03-getting-started/01-logging-in.md b/_es_11.5/03-getting-started/01-logging-in.md new file mode 100644 index 00000000..1d68bced --- /dev/null +++ b/_es_11.5/03-getting-started/01-logging-in.md @@ -0,0 +1,185 @@ +--- +title: "Logging in" +excerpt: "Log in to your Event Streams installation." +categories: getting-started +slug: logging-in +toc: true +--- + +You can configure {{site.data.reuse.es_name}} for external access to the {{site.data.reuse.es_name}} UI and CLI. For {{site.data.reuse.openshift_short}}, {{site.data.reuse.es_name}} uses routes to access the UI and CLI. You can use ingress to access the UI on other Kubernetes platforms. + +Log in to {{site.data.reuse.es_name}} UI and CLI as follows: + +1. Retrieve the URL of your instance. +2. Use the URL to log in to your instance. + +## Logging in to {{site.data.reuse.es_name}} UI + +You can retrieve the URL for accessing the {{site.data.reuse.es_name}} UI: + +- By using the {{site.data.reuse.openshift_short}} web console. +- By using the Kubernetes command-line tool (`kubectl`). + +### Using the {{site.data.reuse.openshift_short}} web console + +Use the OpenShift web console to retrieve the URL for your {{site.data.reuse.es_name}} UI as follows: + +1. {{site.data.reuse.openshift_ui_login}} +2. Expand **Operators** in the navigation on the left, and click **Installed Operators**. + + ![Operators > Installed Operators]({{ 'images' | relative_url }}/rhocp_menu_installedoperators.png "Screen capture showing how to select Operators > Installed Operators from navigation menu"){:height="50%" width="50%"} +3. Locate the operator that manages your {{site.data.reuse.es_name}} instance in the namespace. It is called **{{site.data.reuse.es_name}}** in the **NAME** column. +4. Click the **{{site.data.reuse.es_name}}** link in the row and click the **{{site.data.reuse.es_name}}** tab. This lists the **{{site.data.reuse.es_name}}** instances related to this operator. +5. Find your instance in the **Name** column and click the link for the instance. + + ![{{site.data.reuse.es_name}} > {{site.data.reuse.es_name}} > Instance]({{ 'images' | relative_url }}/find_your_instance.png "Screen capture showing how to select your instance by {{site.data.reuse.es_name}} > {{site.data.reuse.es_name}} > Instance"){:height="100%" width="100%"} +6. A link to the {{site.data.reuse.es_name}} UI is displayed under the label **Admin UI**. Click the link to open the {{site.data.reuse.es_name}} UI login page in your browser tab. +7. Log in to your {{site.data.reuse.es_name}} UI from a supported [web browser](../../installing/prerequisites/#ibm-event-streams-ui). Use your credentials provided to you by your cluster administrator. + A cluster administrator can manage access rights by following the instructions in [managing access](../../security/managing-access/#accessing-the-event-streams-ui-and-cli). + Enter your username and password to access the {{site.data.reuse.es_name}} UI. + +### Using the Kubernetes command-line tool (`kubectl`) + +To retrieve the URL for your {{site.data.reuse.es_name}} UI, use the following commands: + +1. {{site.data.reuse.cncf_cli_login}} +2. Find the URL of your services as follows. + - If you are running with route endpoint, run the following command: + + ```shell + kubectl get routes -n -l app.kubernetes.io/name=admin-ui + ``` + + - If you are running with ingress endpoint, run the following command: + + ```shell + kubectl get ingress -n -l app.kubernetes.io/name=admin-ui + ``` + + The following is an example output, and you use the value from the **HOST/PORT** column to log in to your UI in a web browser: + + ```shell + NAME HOST/PORT PATH SERVICES PORT TERMINATION WILDCARD + my-eventstreams-ibm-es-ui my-eventstreams-ibm-es-ui-myproject.apps.my-cluster.my-domain.com my-eventstreams-ibm-es-ui 3000 reencrypt None + ``` + +3. Enter the address in a web browser. Add `https://` in front of the **HOST/PORT** value. For example: + + ```shell + https://my-eventstreams-ibm-es-ui-myproject.apps.my-cluster.my-domain.com + ``` + +4. Log in to your {{site.data.reuse.es_name}} UI from a supported [web browser](../../installing/prerequisites/#ibm-event-streams-ui). Use your credentials provided to you by your cluster administrator. A cluster administrator can manage access rights by following the instructions in [managing access](../../security/managing-access/#accessing-the-event-streams-ui-and-cli). Enter your username and password to access the {{site.data.reuse.es_name}} UI. + + +## Logging in to {{site.data.reuse.es_name}} CLI + +You can retrieve the URL for accessing the {{site.data.reuse.es_name}} CLI: + +- By using the {{site.data.reuse.openshift_short}} web console. +- By using the Kubernetes command-line tool (`kubectl`). + +**Note:** Authentication through Keycloak is not supported in the {{site.data.reuse.es_name}} CLI. You can authenticate the {{site.data.reuse.es_name}} CLI with the SCRAM authentication and then proceed to use an {{site.data.reuse.es_name}} instance that is configured with Keycloak. + +### Using the {{site.data.reuse.openshift_short}} web console + +Use the OpenShift web console to retrieve the URL for your {{site.data.reuse.es_name}} CLI as follows: + +1. {{site.data.reuse.openshift_ui_login}} +2. Expand **Networking** in the navigation on the left, and select **Routes**. + + ![Networking > Routes]({{ 'images' | relative_url }}/rhocp_menu_routes.png "Screen capture showing how to select Networking > Routes from navigation menu"){:height="50%" width="50%"} +3. Expand the **Project** at the top of the page and select **ibm-common-services**. + + The following is an example output, and you use the value from the **Location** column for the **cp-console** entry to log in to your CLI in a terminal window: + + ![Project]({{ 'images' | relative_url }}/find_cp_console_route.png "Screen capture showing how to select the CLI route") +4. Enter the address on your login command in a terminal. For example: + + ```shell + cloudctl login -a https://cp-console.apps.my-cluster.my-domain.com + ``` + +5. Use your credentials provided to you by your cluster administrator. + A cluster administrator can manage access rights by following the instructions in [managing access](../../security/managing-access/#accessing-the-event-streams-ui-and-cli). + Enter your username and password to access the {{site.data.reuse.es_name}} CLI. +6. Initialize the {{site.data.reuse.es_name}} plugin by running the following command: + + ```shell + cloudctl es init -n + ``` + +### Using the Kubernetes command-line tool (`kubectl`) + +The process for initializing the {{site.data.reuse.es_name}} CLI is different depending on the [configured authentication type](../../installing/configuring/#configuring-ui-and-cli-security): + +- Salted Challenge Response Authentication Mechanism (SCRAM) +- {{site.data.reuse.icpfs}} Identity and Access Management (IAM) + + +#### Authentication type set to SCRAM + +When you have the authentication type set to SCRAM, obtain the required credentials and endpoints, and then initialize and log in to the {{site.data.reuse.es_name}} CLI as follows: + +1. Obtain the following credentials and endpoints from your cluster administrator: + - Username + - Password + - Admin API URL + - Schema registry URL + + A cluster administrator can retrieve credentials and manage access rights by following the instructions in [managing access](../../security/managing-access/#managing-access-to-kafka-resources). + +2. Using the credentials and information you obtained earlier, initialize the {{site.data.reuse.es_name}} plugin by running the following command: + + ```shell + kubectl es init --auth-type scram-sha-512 + ``` + + **Note:** For the {{site.data.reuse.es_name}} CLI plugin to be set up, ensure you add `auth-type` as `scram-sha-512`. If `auth-type` is not present, the cluster will set up IAM as the authentication type by default. For more information, see [configuring UI and CLI security](../../installing/configuring/#configuring-ui-and-cli-security). + +3. Enter the credentials and endpoints when prompted to log in to the {{site.data.reuse.es_name}} CLI. + + Alternatively, instead of waiting for the prompt, you can also run the command with all the credentials and endpoints as additional arguments in one command as follows: + + ```shell + kubectl es init --auth-type scram-sha-512 --username --password --api-url --schema-reg-url + ``` + +#### Authentication type set to IAM + +{{site.data.reuse.iam_note}} + +When you have the authentication type set to IAM, retrieve the login URL for the {{site.data.reuse.es_name}} CLI, then log in and initialize the CLI as follows: + +1. {{site.data.reuse.cncf_cli_login}} +2. Run the following command: + + ```shell + kubectl get routes -n ibm-common-services -l app.kubernetes.io/name=management-ingress + ``` + + The following is an example output, and you use the value from the **HOST/PORT** column for the **cp-console** entry to log in to your CLI in a terminal: + + ```shell + NAME HOST/PORT PATH SERVICES PORT TERMINATION WILDCARD + cp-console cp-console.apps.my-cluster.my-domain.com icp-management-ingress https reencrypt/Redirect None + cp-proxy cp-proxy.apps.my-cluster.my-domain.com nginx-ingress-controller https passthrough/Redirect None + ``` +3. Enter the address on your login command in a terminal. Add `https://` in front of the **HOST/PORT** value. For example: + ```shell + cloudctl login -a https://cp-console.apps.my-cluster.my-domain.com + ``` +4. Use your credentials provided to you by your cluster administrator. + A cluster administrator can manage access rights by following the instructions in [managing access](../../security/managing-access/#accessing-the-event-streams-ui-and-cli). + Enter your username and password to access the {{site.data.reuse.es_name}} CLI. +5. Initialize the {{site.data.reuse.es_name}} plugin by running the following command: + ```shell + cloudctl es init -n + ``` + + +## Logging out + +To log out of {{site.data.reuse.es_name}}: +1. In your {{site.data.reuse.es_name}} UI, click the user icon. +2. Click **Log out**. diff --git a/_es_11.5/03-getting-started/02-creating-topics.md b/_es_11.5/03-getting-started/02-creating-topics.md new file mode 100644 index 00000000..f22a8bae --- /dev/null +++ b/_es_11.5/03-getting-started/02-creating-topics.md @@ -0,0 +1,117 @@ +--- +title: "Creating a Kafka topic" +excerpt: "Create a Kafka topic to learn more about using Event Streams" +categories: getting-started +slug: creating-topics +toc: true +--- + +To use Kafka topics to store events in {{site.data.reuse.es_name}}, create and configure a Kafka topic. + +## By using the UI + +1. {{site.data.reuse.es_ui_login_nonadmin_samesection}} +2. Click **Home** in the primary navigation. +3. Click the **Create a topic** tile. +4. Enter a topic name in the **Topic name** field, for example, `my-topic`. + This is the name of the topic that an application will be producing to or consuming from. + + Click **Next**. +5. Enter the number of **Partitions**, for example, `1`. + Partitions are used for scaling and distributing topic data across the Apache Kafka brokers. + For the purposes of a basic starter application, using only 1 partition is sufficient. + + Click **Next**. +6. Select a **Message retention**, for example, **A day**. + This is how long messages are retained before they are deleted. + + Click **Next**. +7. Select a replication factor in **Replicas**, for example, **Replication factor: 1**. + This is how many copies of a topic will be made for high availability. For production environments, select **Replication factor: 3** as a minimum. + +8. Click **Create topic**. The topic is created and can be viewed from the **Topics** tab located in the primary navigation. + +**Note:** To view all configuration options you can set for topics, set **Show all available options** to **On**. + +**Note:** Kafka supports additional [topic configuration](https://kafka.apache.org/37/documentation/#topicconfigs){:target="_blank"} settings. Enable **Show all available options** to access more detailed configuration settings if required. + +## By using the CLI + +1. {{site.data.reuse.cncf_cli_login}} + +2. {{site.data.reuse.es_cli_init_111_samesection}} + +3. Run the following command to create a topic: + + ```shell + kubectl es topic-create --name --partitions --replication-factor + ``` + + For example, to create a topic called `my-topic` that has 1 partition, a replication factor of 1, and 1 day set for message retention time (provided in milliseconds): + + ```shell + kubectl es topic-create --name my-topic --partitions 1 --replication-factor 1 --config retention.ms=86400000 + ``` + + **Important:** Do not set `` to a greater value than the number of available brokers. + + +**Note:** To view all configuration options you can set for topics, use the help option as follows: + +```shell +kubectl es topic-create --help +``` + +Kafka supports additional [topic configuration](https://kafka.apache.org/37/documentation/#topicconfigs){:target="_blank"} settings. Extend the topic creation command with one or more `--config =` properties to apply additional configuration settings. The following additional properties are currently supported: + +* cleanup.policy +* compression.type +* delete.retention.ms +* file.delete.delay.ms +* flush.messages +* flush.ms +* follower.replication.throttled.replicas +* index.interval.bytes +* leader.replication.throttled.replicas +* max.message.bytes +* message.format.version +* message.timestamp.difference.max.ms +* message.timestamp.type +* min.cleanable.dirty.ratio +* min.compaction.lag.ms +* min.insync.replicas +* preallocate +* retention.bytes +* retention.ms +* segment.bytes +* segment.index.bytes +* segment.jitter.ms +* segment.ms +* unclean.leader.election.enable + +## By using YAML + +You can create a Kafka topic by creating a `KafkaTopic` custom resource in a YAML file, and then running `kubectl apply -f .yaml`. + +The following is an example `KafkaTopic` custom resource: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: KafkaTopic +metadata: + name: + labels: + eventstreams.ibm.com/cluster: + backup.eventstreams.ibm.com/component: kafkatopic + namespace: +spec: + partitions: 1 + replicas: 1 + config: + min.insync.replicas: '1' +``` + + + + + diff --git a/_es_11.5/03-getting-started/02a-managing-topics.md b/_es_11.5/03-getting-started/02a-managing-topics.md new file mode 100644 index 00000000..eaff5d0e --- /dev/null +++ b/_es_11.5/03-getting-started/02a-managing-topics.md @@ -0,0 +1,68 @@ +--- +title: "Managing Kafka topics" +excerpt: "Find out how you can edit and remove Kafka topics in Event Streams." +categories: getting-started +slug: managing-topics +toc: true +--- + +## Editing a Kafka topic + +### By using the CLI + +1. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to initialize the {{site.data.reuse.es_name}} CLI on the cluster: + + ```shell + kubecl es init + ``` +3. Run the following command to edit the configuration of a topic: + + ```shell + kubectl es topic-update --name --config KEY[=VALUE] + ``` + + For example: + - To update a topic called `my-topic` to have `min.insync.replicas` as `2`: + + ```shell + kubectl es topic-update --name my-topic --config min.insync.replicas=2 + ``` + + - To update the number of topic partitions to `5` : + + ```shell + kubectl es topic-partitions-set -n my-topic -p 5 + ``` + + **Note:** To view all configuration options you can set for topics, use the help option as follows: + + ```shell + kubectl es topic-update --help + ``` + + +## Deleting a Kafka topic + +To delete a Kafka topic, complete the following steps: + +### By using the UI + +1. {{site.data.reuse.es_ui_login_nonadmin}} +2. Click **Home** in the primary navigation. +3. Navigate to **Topics** tab. +4. Click **Topic Delete** button in the overflow menu of the topic you want to delete. The topic is deleted and is removed from the list of topics. + +### By using the CLI + +1. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to initialize the {{site.data.reuse.es_name}} CLI on the cluster: + + ```shell + kubectl es init + ``` +3. Run the following command to delete a topic: + + ```shell + kubectl es topic-delete --name + ``` diff --git a/_es_11.5/03-getting-started/03-generating-starter-app.md b/_es_11.5/03-getting-started/03-generating-starter-app.md new file mode 100644 index 00000000..edbcef13 --- /dev/null +++ b/_es_11.5/03-getting-started/03-generating-starter-app.md @@ -0,0 +1,57 @@ +--- +title: "Running a starter application" +excerpt: "Create a starter application to learn more about using Event Streams." +categories: getting-started +slug: generating-starter-app +toc: true +--- + +To learn more about how to create applications that can take advantage of {{site.data.reuse.es_name}} capabilities, you can use the starter application. The starter application can produce and consume messages, and you can specify the topic to send messages to and the contents of the message. + +## About the application +The starter application provides a demonstration of a Java application that uses the [Vert.x Kafka Client](https://vertx.io/docs/vertx-kafka-client/java/){:target="_blank"} to send events to, and receive events from, {{site.data.reuse.es_name}}. It also includes a user interface to easily view message propagation. The source code is provided in [GitHub](https://github.com/ibm-messaging/kafka-java-vertx-starter){:target="_blank"} to allow you to understand the elements required to create your own Kafka application. + +## Downloading the application +If you do not already have the application, download the JAR file from the {{site.data.reuse.es_name}} UI. +1. {{site.data.reuse.es_ui_login_nonadmin_samesection}} +2. Click the **Try the starter application** tile, or click **Toolbox** in the primary navigation, go to the **Starter application** section, and click **Get started**. +3. Click **Download JAR from GitHub**. +4. Download the JAR file for the latest release. + +## Generate security and configuration files +Before you can run the application, generate security and configuration files to connect to your {{site.data.reuse.es_name}} and the target topic. + +Some of the following steps depend on your [access permissions](../../security/managing-access/): +- If you have logged in to the {{site.data.reuse.es_name}} UI by using a Keycloak user ID or an {{site.data.reuse.icpfs}} Identity and Access Management (IAM) user ID, and are not permitted to generate credentials, you will not see the **Generate properties** button. You will have to obtain security and configuration files from your administrator before running the application. +- If you have logged in to the {{site.data.reuse.es_name}} UI by using a SCRAM `KafkaUser`, you will need `topic read` and `topic write` permissions to generate the properties as described in [managing access to the UI with SCRAM](../../security/managing-access/#managing-access-to-the-ui-and-cli-with-scram). If permitted, these properties will be generated with the credentials of the current user. No additional `KafkaUsers` will be generated. If you are not permitted to create topics, you will not be able to create a topic as mentioned in the steps later in this section, and will have to use a pre-existing topic. + +1. From the **Starter application** menu opened in [the previous step](#downloading-the-application), click **Generate properties** to open the side panel. +2. Enter an application name in the **Starter application name** field. This value is used by {{site.data.reuse.es_name}} to create a `KafkaUser`, which will provide your application with credentials to connect to {{site.data.reuse.es_name}} securely. + **Note:** This name must be unique to avoid potential clashes with pre-existing KafkaUser resources. + +3. Select a new or existing topic to connect to. + **Note:** When creating a new topic the name must be unique to avoid potential clashes with pre-existing topics. +4. Click the **Generate and download .zip** button to download the compressed file, then extract the contents to your preferred location. + +## Running the application + +Before running the application, ensure you have the following available: +1. The application [JAR file](#downloading-the-application). +2. A directory containing [security and configuration files](#generate-security-and-configuration-files) + +Run the following command to start the application: + +``` +java -Dproperties_path= -jar /demo-all.jar +``` + +Where: +- `configuration_properties_path` is the path to the directory containing the extracted security and configuration files. +- `jar_path` is the path to the downloaded application JAR file. + +Wait for the application to be ready. It will print out the following message: +``` +Application started in Xms +``` + +When the application is ready, access the UI by using the following URL: [http://localhost:8080](http://localhost:8080). Use the start button in the UI to produce messages and see them consumed. diff --git a/_es_11.5/03-getting-started/04-testing-loads.md b/_es_11.5/03-getting-started/04-testing-loads.md new file mode 100644 index 00000000..14072cb9 --- /dev/null +++ b/_es_11.5/03-getting-started/04-testing-loads.md @@ -0,0 +1,168 @@ +--- +title: "Creating and testing message loads" +excerpt: "Use the producer application as a workload generator to test message loads." +categories: getting-started +slug: testing-loads +toc: true +--- + +{{site.data.reuse.es_name}} provides a high-throughput producer application you can use as a workload generator to test message loads and help validate the performance capabilities of your cluster. + +You can use one of the predefined load sizes, or you can specify your own settings to test throughput. Then use the test results to ensure your cluster setup is appropriate for your requirements, or make changes as needed, for example, by changing your [scaling settings](../../administering/scaling/). + + +## Downloading + +You can [download the latest pre-built producer](https://github.com/IBM/event-streams-sample-producer/releases){:target="_blank"} application. + +Alternatively, you can [clone the project from GitHub](https://github.com/IBM/event-streams-sample-producer){:target="_blank"}. However, if you clone from GitHub, you have to [build the producer](#building). + + +## Building + +If you cloned the Git repository, build the producer as follows: + +1. {{site.data.reuse.maven_prereq}} +2. Ensure you have [cloned the Git project](https://github.com/IBM/event-streams-sample-producer){:target="_blank"}. +3. Open a terminal and change to the root directory of the `event-streams-sample-producer` project. +4. Run the following command: + + ```shell + mvn install + ``` + + You can also specify your root directory using the `-f` option as follows: + + ```shell + mvn install -f /pom.xml + ``` + +5. The `es-producer.jar` file is created in the `/target` directory. + + +## Configuring + +The producer application requires configuration settings that you can set in the provided `producer.config` template configuration file. + +**Note:** The `producer.config` file is located in the root directory. If you downloaded the pre-built producer, you have to run the `es-producer.jar` with the `-g` option to generate the configuration file. If you build the producer application yourself, the configuration file is created and placed in the root for you when building. + +Before running the producer to test loads, you must specify the `bootstrap.servers` and any required security configuration details in the configuration file. + +### Obtaining configuration details + +The bootstrap servers address can be obtained from the {{site.data.reuse.es_name}} UI as described in the following steps. Other methods to obtain the bootstrap servers address are described in [connecting client](../connecting/). + +1. {{site.data.reuse.es_ui_login_nonadmin_samesection}} +2. Click **Connect to this cluster** on the right. +3. Click the **Resources** tab. +4. Go to the **Kafka listener and credentials** section. +5. Copy the address from one of the **External** listeners. + +The producer application might require credentials for the listener chosen in the previous step. For more information about these credentials, see the information about [managing access](../../security/managing-access/). + +Obtain the required credentials as follows: +1. {{site.data.reuse.es_ui_login_nonadmin_samesection}} +2. Click **Connect to this cluster** on the right. +3. Go to the **Resources** tab. +4. Scroll down to the **Kafka listener and credentials** section. +5. Click the button next to the listener chosen as the `bootstrap.servers` configuration. If present, the button will either be labeled **Generate SCRAM credentials** or **Generate TLS credentials**. +6. Select **Produce messages, consume messages and create topics and schemas** and click **Next**. +7. Select **A specific topic**, enter the name of a topic to produce to and click **Next**. +8. Select **All consumer groups** and click **Next**. +9. Select **No transactional IDs** and click **Generate credentials**. +10. Retrieve the generated credentials:\\ + - If using SCRAM note down the **Username and password**. + - If using TLS click **Download certificates** and extract the contents of the resulting .zip file to a preferred location. + +Obtain the {{site.data.reuse.es_name}} certificate as follows: +1. {{site.data.reuse.es_ui_login_nonadmin_samesection}} +2. Click **Connect to this cluster** on the right. +3. Go to the **Resources** tab. +5. Scroll down to the **Certificates** section. +6. In the **PKCS12 certificate** section click **Download certificate**. +7. Note down the generated password displayed in the **Certificate password** section. + +### Updating the configuration file + +Before updating the file, obtain the credentials by following the steps in [Obtaining configuration details](#obtaining-configuration-details). + +Update the `producer.config` file with your configuration details using the following table as guidance. + +| Attribute | Description | +| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `bootstrap.servers` | The bootstrap address for the chosen external listener. | | +| `security.protocol` | Set to `SSL`. Can be omitted if the chosen external listener has TLS disabled. | +| `ssl.truststore.location` | The full path and name of the {{site.data.reuse.es_name}} PKCS12 certificate file. Can be omitted if the chosen external listener has TLS disabled. | +| `ssl.truststore.password` | The password for the {{site.data.reuse.es_name}} PKCS12 certificate file. Can be omitted if the chosen external listener has TLS disabled. | +| `sasl.mechanism` | Set to `SCRAM-SHA-512` if using SCRAM credentials, otherwise omitted. | +| `sasl.jaas.config` | Set to `org.apache.kafka.common.security.scram.ScramLoginModule required username="" password="";`, where `` and `` are replaced with the SCRAM credentials. Omitted if not using SCRAM credentials. | +| `ssl.keystore.location` | Set to the full path and name of the `user.p12` keystore file downloaded from the {{site.data.reuse.es_name}} UI. Omitted if not using TLS credentials. | +| `ssl.keystore.password` | Set to the password listed in the `user.password` file downloaded from the {{site.data.reuse.es_name}} UI. Omitted if not using TLS credentials. | + +## Running + +Create a load on your {{site.data.reuse.es_name}} Kafka cluster by running the `es-producer.jar` command. You can specify the load size based on the provided predefined values, or you can provide specific values for throughput and total messages to determine a custom load. + +### Using predefined loads + +To use a predefined load size from the producer application, use the `es-producer.jar` with the `-s` option: + +`java -jar target/es-producer.jar -t -s ` + +For example, to create a `large` message load based on the predefined `large` load size, run the command as follows: + +`java -jar target/es-producer.jar -t my-topic -s large` + +This example creates a `large` message load, where the producer attempts to send a total of 6,000,000 messages at a rate of 100,000 messages per second to the topic called `my-topic`. + +The following table lists the predefined load sizes the producer application provides. + +Size | Messages per second | Total messages | +-------- | ------------------- | -------------- | +`small` | 1000 | 60,000 | +`medium` | 10,000 | 600,000 | +`large` | 100,000 | 6,000,000 | + +### Specifying a load + +You can generate a custom message load using your own settings. + +For example, to test the load to the topic called `my-topic` with custom settings that create a total load of `60,000` **messages** with a **size of 1024 bytes each**, at a **maximum throughput rate of 1000 messages per second**, use the `es-producer.jar` command as follows: + +``` +java -jar target/es-producer.jar -t my-topic -T 1000 -n 60000 -r 1024 +``` + +The following table lists all the parameter options for the `es-producer.jar` command. + +| Parameter | Shorthand | Longhand | Type | Description | Default | +| --------------------- | --------- | --------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| Topic | -t | --topic | string | The name of the topic to send the produced message load to. | `loadtest` | +| Num Records | -n | --num-records | integer | The total number of messages to be sent as part of the load. **Note:** The `--size` option overrides this value if used together. | `60000` | +| Payload File | -f | --payload-file | string | File to read the message payloads from. This works only for UTF-8 encoded text files. Payloads are read from this file and a payload is randomly selected when sending messages. | | +| Payload Delimiter | -d | --payload-delimiter | string | Provides delimiter to be used when `--payload-file` is provided. This parameter is ignored if `--payload-file` is not provided. | `\n` | +| Throughput | -T | --throughput | integer | Throttle maximum message throughput to *approximately* *THROUGHPUT* messages per second. -1 sets it to as fast as possible. **Note:** The `--size` option overrides this value if used together. | `-1` | +| Producer Config | -c | --producer-config | string | Path to the producer configuration file. | `producer.config`| +| Print Metrics | -m | --print-metrics | boolean | Set whether to print out metrics at the end of the test. | `false` | +| Num Threads | -x | --num-threads | integer | The number of producer threads to run. | `1` | +| Size | -s | --size | string | Pre-defined combinations of message throughput and volume. If used, this option overrides any settings specified by the `--num-records` and `--throughput` options. | | +| Record Size | -r | --record-size | integer | The size of each message to be sent in bytes. | `100` | +| Help | -h | --help | N/A | Lists the available parameters. | | +| Gen Config | -g | --gen-config | N/A | Generates the configuration file required to run the tool (`producer.config`). | | + + +**Note:** You can override the parameter values by using the environment variables listed in the following table. This is useful, for example, when using containerization, and you are unable to specify parameters on the command line. + +| Parameter | Environment Variable | +| --------------------- | -------------------- | +| Throughput | ES_THROUGHPUT | +| Num Records | ES_NUM_RECORDS | +| Size | ES_SIZE | +| Record Size | ES_RECORD_SIZE | +| Topic | ES_TOPIC | +| Num threads | ES_NUM_THREADS | +| Producer Config | ES_PRODUCER_CONFIG | +| Payload File | ES_PAYLOAD_FILE | +| Payload Delimiter | ES_PAYLOAD_DELIMITER | + +**Note:** If you set the size using `-s` when running `es-producer.jar`, you can only override it if both the `ES_NUM_RECORDS` and `ES_THROUGHPUT` environment variables are set, or if `ES_SIZE` is set. diff --git a/_es_11.5/03-getting-started/05-client.md b/_es_11.5/03-getting-started/05-client.md new file mode 100644 index 00000000..525c9ae6 --- /dev/null +++ b/_es_11.5/03-getting-started/05-client.md @@ -0,0 +1,57 @@ +--- +title: "Creating Kafka client applications" +excerpt: "Create Kafka client applications to use with Event Streams." +categories: getting-started +slug: client +toc: true +--- + +The {{site.data.reuse.es_name}} UI provides help with creating an Apache Kafka Java client application and discovering connection details for a specific topic. + +## Creating an Apache Kafka Java client application + +You can create Apache Kafka Java client applications to use with {{site.data.reuse.es_name}}. + +Download the JAR file from {{site.data.reuse.es_name}}, and include it in your Java build and classpaths before compiling and running Kafka Java clients. + +1. {{site.data.reuse.es_ui_login_nonadmin_samesection}} +2. Click **Toolbox** in the primary navigation. +3. Go to the **Apache Kafka Java client** section and click **Find out more**. +4. Click the **Apache Kafka Client JAR** link to download the JAR file. The file contains the Java class files and related resources needed to compile and run client applications you intend to use with {{site.data.reuse.es_name}}. +5. [Download](https://www.slf4j.org/download.html){:target="_blank"} the JAR files for **SLF4J** required by the Kafka Java client for logging. +6. Include the downloaded JAR files in your Java build and classpaths before compiling and running your Apache Kafka Java client. +7. Ensure you [set up security](../connecting/#securing-the-connection). + +## Creating an Apache Kafka Java client application using Maven or Gradle + +If you are using Maven or Gradle to manage your project, you can use the following snippets to include the Kafka client JAR and dependent JARs on your classpath. + +- For Maven, use the following snippet in the `` section of your `pom.xml` file: + + ```xml + + org.apache.kafka + kafka-clients + 2.8.1 + + + org.slf4j + slf4j-api + 1.7.26 + + + org.slf4j + slf4j-simple + 1.7.26 + + ``` + +- For Gradle, use the following snippet in the `dependencies{}` section of your `build.gradle` file: + + ```gradle + implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '2.8.1' + implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.26' + implementation group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.26' + ``` + +- Ensure you [set up security](../connecting/#securing-the-connection). diff --git a/_es_11.5/03-getting-started/06-connecting-clients.md b/_es_11.5/03-getting-started/06-connecting-clients.md new file mode 100644 index 00000000..dcd248b2 --- /dev/null +++ b/_es_11.5/03-getting-started/06-connecting-clients.md @@ -0,0 +1,237 @@ +--- +title: "Connecting clients" +excerpt: "Find out how to discover connection details to connect your client applications to Event Streams." +categories: getting-started +slug: connecting +toc: true +--- + +Learn how to discover connection details to connect your clients to your {{site.data.reuse.es_name}} instance. + +## Obtaining the bootstrap address + +Use one of the following methods to obtain the bootstrap address for your connection to your {{site.data.reuse.es_name}} instance, choosing the listener **`type`** appropriate for your client. More information on configuring listener types can be found in the [configuring Kafka access section](../../installing/configuring/#kafka-access). + +### Using the CLI + +1. {{site.data.reuse.cncf_cli_login}} +2. To find the type and address of the Kafka bootstrap host for each listener, run the following command: + + ```shell + kubectl get eventstreams -o=jsonpath='{range .status.kafkaListeners[*]}{.type} {.bootstrapServers}{"\n"}{end}' + ``` + + Where `` is the name of your {{site.data.reuse.es_name}} instance. + +**Note:** If using an external Kafka listener on the {{site.data.reuse.openshift_short}}, the OpenShift route is an HTTPS address, and the port in use is 443. + +### Using the {{site.data.reuse.es_name}} UI + +1. {{site.data.reuse.es_ui_login_nonadmin_samesection}} +2. Click the **Connect to this cluster** tile. +3. Go to the **Kafka listener and credentials** section, and select the listener from the list. + + - Click the **External** tab for applications connecting from outside of the Kubernetes cluster. + - Click the **Internal** tab for applications connecting from inside the Kubernetes cluster. + + **Note:** The list reflects the listeners [configured](../../installing/configuring) in `spec.strimziOverrides.kafka.listeners`. For example, you will have external listeners displayed if you have any listeners configured with a type of `route` or `ingress`. If `spec.strimziOverrides.kafka.listeners` is empty for your instance (not configured), then no address is displayed here. + +### Using the {{site.data.reuse.es_name}} CLI + +**Note:** You can only use the {{site.data.reuse.es_name}} CLI to retrieve the address if your {{site.data.reuse.es_name}} instance has at least one external listener [configured](../../installing/configuring) in `spec.strimziOverrides.kafka.listeners`. + +1. [Install the {{site.data.reuse.es_name}} CLI plugin](../../installing/post-installation/#installing-the-event-streams-command-line-interface) if not already installed. + +2. {{site.data.reuse.es_cli_init_111_samesection}} + Make note of the **Event Streams bootstrap address** value. This is the Kafka bootstrap address that your application will use. + + **Note:** If you have multiple listeners defined in `spec.strimziOverrides.kafka.listeners`, only the external listener is displayed. If you only have internal listeners defined, nothing is displayed. + +### Using the {{site.data.reuse.openshift_short}} web console + +1. {{site.data.reuse.openshift_ui_login}} +2. {{site.data.reuse.task_openshift_navigate_installed_operators}} +3. {{site.data.reuse.task_openshift_select_operator}} +4. {{site.data.reuse.task_openshift_select_instance}} +5. Select the **YAML** tab. +6. Scroll down and look for `status.kafkaListeners`. +7. The `kafkaListeners` field will contain one or more listeners each with a `bootstrapServers` property. + Find the `type` of listener you want to connect to and use the `bootstrapServers` value from the entry. + +**Note:** If using an external Kafka listener, the OpenShift route is an HTTPS address, and the port in use is 443. + +## Securing the connection + +To connect client applications to a secured {{site.data.reuse.es_name}}, you must obtain the following: + +- A copy of the server-side public certificate to add to your client-side trusted certificates. +- SCRAM-SHA-512 (`username` and `password`) or Mutual TLS (user certificates) Kafka credentials. + +### Obtaining the server-side public certificate from the CLI + +To extract the server-side public certificate to a `ca.p12` file, run the following command: + +```shell +kubectl get secret -cluster-ca-cert -o jsonpath='{.data.ca\.p12}' | base64 -d > ca.p12 +``` + +Where `` is the name of your {{site.data.reuse.es_name}} instance. + +To extract the password for the certificate to a `ca.password` file, run the following command: + +```shell +kubectl get secret -cluster-ca-cert -o jsonpath='{.data.ca\.password}' | base64 -d > ca.password +``` + +**Note:** If a `PEM` certificate is required, run the following command to extract the certificate to a `ca.crt` file: + +```shell +kubectl get secret -cluster-ca-cert -o jsonpath='{.data.ca\.crt}' | base64 -d > ca.crt +``` + +**Important:** If you have configured [external CA certificates](../../installing/configuring/#providing-external-ca-certificates) in the `externalCACertificates` secret, these will not be included in the secret `-cluster-ca-cert`. To obtain the external CA certificates, use secret name `-combined-cluster-ca-certs` instead, or obtain them from the UI or CLI as described in the following sections. + +### Obtaining the server-side public certificate from the {{site.data.reuse.es_name}} UI + +1. {{site.data.reuse.es_ui_login_nonadmin_samesection}} +2. Click **Connect to this cluster** on the right. +3. From the **Certificates** section, download the server certificate. If you are using a Java client, use the **PKCS12 certificate**, remembering to copy the truststore password presented during download. Otherwise, use the **PEM certificate**. + +**Note**: If you have configured [external CA certificates](../../installing/configuring/#providing-external-ca-certificates) in the `externalCACertificates` secret, these will be included in the downloaded server certificate. + +### Obtaining the server-side public certificate from the {{site.data.reuse.es_name}} CLI + +1. [Install the {{site.data.reuse.es_name}} CLI plugin](../../installing/post-installation/#installing-the-event-streams-command-line-interface) if not already installed. + +2. {{site.data.reuse.es_cli_init_111_samesection}} +3. Use the `certificates` command to download the cluster's public certificate in the required format: + + ```shell + kubectl es certificates --format p12 + ``` + + The truststore password will be displayed in the output for the command. The following example has a truststore password of `mypassword`: + + ```shell + $ kubectl es certificates --format p12 + + Trustore password is mypassword + Certificate successfully written to /es-cert.p12. + OK + ``` + + **Note:** You can optionally change the format to download a `PEM` Certificate if required. + +**Note**: If you have configured [external CA certificates](../../installing/configuring/#providing-external-ca-certificates) in the `externalCACertificates` secret, these will be included in the downloaded server certificate. + +### Obtaining the server-side public certificate from the {{site.data.reuse.openshift_short}} web console + +1. {{site.data.reuse.openshift_ui_login}} +2. {{site.data.reuse.task_openshift_navigate_installed_operators}} +3. {{site.data.reuse.task_openshift_select_operator}} +4. {{site.data.reuse.task_openshift_select_instance}} +5. Select the **Resources** tab. +6. To filter only secrets, deselect all resource types with the exception of **Secret**. +7. Locate and select the `-cluster-ca-cert` secret. Where `` is the name of your {{site.data.reuse.es_name}} instance. +8. In the **Secret Overview** panel scroll down to the **Data** section. Then, click the copy button to transfer the `ca.p12` certificate to the clipboard. The password can be found under `ca.password`. + +**Note:** For a `PEM` certificate, click the copy button for `ca.crt` instead. + +### Generating or Retrieving Client Credentials + +See the [assigning access to applications](../../security/managing-access/#managing-access-to-kafka-resources) section to learn how to create new application credentials or retrieve existing credentials. + + +### Configuring your SCRAM client + +Add the [truststore certificate details](#securing-the-connection) and the [SCRAM credentials](#generating-or-retrieving-client-credentials) to your Kafka client application to set up a secure connection from your application to your {{site.data.reuse.es_name}} instance. + +You can configure a Java application as follows: + +```shell +Properties properties = new Properties(); +properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, ""); +properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); +properties.put(SslConfigs.SSL_PROTOCOL_CONFIG, "TLSv1.3"); +properties.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, ""); +properties.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, ""); +properties.put(SaslConfigs.SASL_MECHANISM, "SCRAM-SHA-512"); +properties.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.scram.ScramLoginModule required " + + "username=\"\" password=\"\";"); +``` + +| Property Placeholder | Description | +| :-------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------- | +| `` | [Bootstrap servers address](#obtaining-the-bootstrap-address) | +| `` | Path to your truststore certificate. This must be a fully qualified path. As this is a Java application, the PKCS12 certificate is used. | +| `` | Truststore password. | +| `` | SCRAM username. | +| `` | SCRAM password. | + +### Configuring your Mutual TLS client + +Add the [truststore and keystore certificate details](#securing-the-connection) to your Kafka client application to set up a secure connection from your application to your {{site.data.reuse.es_name}} instance. + +You can configure a Java application as follows: + +```shell +Properties properties = new Properties(); +properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, ""); +properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); +properties.put(SslConfigs.SSL_PROTOCOL_CONFIG, "TLSv1.3"); +properties.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, ""); +properties.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, ""); +properties.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, ""); +properties.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, ""); +``` + +| Property Placeholder | Description | +| :-------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------- | +| `` | [Bootstrap servers address](#obtaining-the-bootstrap-address). | +| `` | Path to your truststore certificate. This must be a fully qualified path. As this is a Java application, the PKCS12 certificate is used. | +| `` | Truststore password. | +| `` | Path to `user.p12` keystore file from [credentials zip archive](#generating-or-retrieving-client-credentials). | +| `` | The `user.p12` keystore password found in the `user.password` file in the [credentials zip archive](#generating-or-retrieving-client-credentials). | + +### Configuring your OAuth client + +Add the [truststore certificate details](#securing-the-connection) and the [OAuth credentials](#generating-or-retrieving-client-credentials) to your Kafka client application to set up a secure connection from your application to your {{site.data.reuse.es_name}} instance. + +You can configure a Java application as follows: + +```shell +Properties properties = new Properties(); +properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, ""); +properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); +properties.put(SslConfigs.SSL_PROTOCOL_CONFIG, "TLSv1.3"); +properties.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, ""); +properties.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, ""); +properties.put(SaslConfigs.SASL_MECHANISM, "OAUTHBEARER"); +properties.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required " + + "oauth.ssl.truststore.location=\"\" " + + "oauth.ssl.truststore.type=PKCS12 " + + "oauth.ssl.truststore.password=\"\" " + + "oauth.client.id=\"\" " + + "oauth.client.secret=\"\" " + + "oauth.token.endpoint.uri=\"\";"); +``` + +| Property Placeholder | Description | +| :-------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------- | +| `` | [Bootstrap servers address](#obtaining-the-bootstrap-address). | +| `` | Path to your truststore certificate. This must be a fully qualified path. As this is a Java application, the PKCS12 certificate is used. | +| `` | Truststore password.| +| `` | Path to the OAuth server truststore certificate. This must be a fully qualified path. As this is a Java application, the PKCS12 certificate is used.| +| `` | OAuth server truststore password.| +| `` | OAuth client ID.| +| `` | OAuth client ID password.| +| `` | The URI of the OAuth server token endpoint. | + +### Obtaining Java code samples from the {{site.data.reuse.es_name}} UI + +For a Java application, you can copy the connection code snippet from the {{site.data.reuse.es_name}} UI by doing the following: + +1. {{site.data.reuse.es_ui_login_nonadmin_samesection}} +2. Click the **Connect to this cluster** tile. +3. Click the **Sample code** tab. +4. Copy the snippet from the **Sample connection code** section into your Java Kafka client application. Uncomment the relevant sections and replace the property placeholders with the values from the relevant table for [SCRAM](#configuring-your-scram-client) or [Mutual TLS](#configuring-your-mutual-tls-client). diff --git a/_es_11.5/03-getting-started/06-using-kafka-console-tools.md b/_es_11.5/03-getting-started/06-using-kafka-console-tools.md new file mode 100644 index 00000000..dbb11db5 --- /dev/null +++ b/_es_11.5/03-getting-started/06-using-kafka-console-tools.md @@ -0,0 +1,102 @@ +--- +title: "Using Apache Kafka console tools" +excerpt: "Using Apache Kafka console tools with Event Streams." +categories: getting-started +slug: using-kafka-console-tools +toc: true +--- + +Apache Kafka comes with a variety of console tools for simple administration and messaging operations. You can find these console tools in the `bin` directory of your [Apache Kafka download](https://www.apache.org/dyn/closer.cgi?path=/kafka/3.4.0/kafka_2.13-3.4.0.tgz){:target="_blank"}. + +You can use many of them with {{site.data.reuse.es_name}}, although {{site.data.reuse.es_name}} does not permit connection to its ZooKeeper cluster. As Kafka has developed, many of the tools that previously required connection to ZooKeeper no longer have that requirement. {{site.data.reuse.es_name}} has its own [command-line interface (CLI)](../../installing/post-installation/#installing-the-event-streams-command-line-interface) and this offers many of the same capabilities as the Kafka tools in a simpler form. + +The following table shows which Apache Kafka (release 3.x or later) console tools work with {{site.data.reuse.es_name}} and whether there are CLI equivalents. + +| Console tool | Works with {{site.data.reuse.es_name}} | CLI equivalent | +|:-----------------|:-----------------|:-----------------| +| `kafka-acls.sh` | Yes | | +| `kafka-broker-api-versions.sh` | Yes | | +| `kafka-configs.sh --entity-type topics` | No | `kubectl es topic-update` | +| `kafka-configs.sh --entity-type brokers` | No | `kubectl es broker-config` | +| `kafka-configs.sh --entity-type brokers --entity-default` | No | `kubectl es cluster-config` | +| `kafka-configs.sh --entity-type clients` | No | No - see the `KafkaUser` [quota support](../../administering/quotas/) | +| `kafka-configs.sh --entity-type users` | No | No | +| `kafka-console-consumer.sh` | Yes | | +| `kafka-console-producer.sh` | Yes | | +| `kafka-consumer-groups.sh --list` | Yes | `kubectl es groups` | +| `kafka-consumer-groups.sh --describe` | Yes | `kubectl es group` | +| `kafka-consumer-groups.sh --reset-offsets` | Yes | `kubectl es group-reset` | +| `kafka-consumer-groups.sh --delete` | Yes | `kubectl es group-delete` | +| `kafka-consumer-perf-test.sh` | Yes | | +| `kafka-delete-records.sh` | Yes | `kubectl es topic-delete-records` | +| `kafka-preferred-replica-election.sh` | No | | +| `kafka-producer-perf-test.sh` | Yes | | +| `kafka-streams-application-reset.sh` | Yes | | +| `kafka-topics.sh --list` | Yes | `kubectl es topics` | +| `kafka-topics.sh --describe` | Yes | `kubectl es topic` | +| `kafka-topics.sh --create` | Yes | `kubectl es topic-create` | +| `kafka-topics.sh --delete` | Yes | `kubectl es topic-delete` | +| `kafka-topics.sh --alter --config` | Yes | `kubectl es topic-update` | +| `kafka-topics.sh --alter --partitions` | Yes | `kubectl es topic-partitions-set` | +| `kafka-topics.sh --alter --replica-assignment` | Yes | `kubectl es topic-partitions-set` | +| `kafka-verifiable-consumer.sh` | Yes | | +| `kafka-verifiable-producer.sh` | Yes | | + +## Using the console tools with {{site.data.reuse.es_name}} + +The console tools are Kafka client applications and connect in the same way as regular applications. + +Follow the [instructions for securing a connection](../../getting-started/connecting/#securing-the-connection) to obtain: +* Your cluster’s broker URL +* The truststore certificate +* An API key + +Many of these tools perform administrative tasks and will need to be authorized accordingly. + +Create a properties file based on the following example: + +```shell +security.protocol=SASL_SSL +ssl.protocol=TLSv1.3 +ssl.truststore.location= +ssl.truststore.password= +sasl.mechanism=PLAIN +sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="token" password=""; +``` + +Replace: +* `` with the path to your truststore file +* `` with `"password"` +* `` with your API key + + +### Example - console producer + +You can use the Kafka console producer tool with {{site.data.reuse.es_name}}. + +After you've created the properties file as described previously, you can run the console producer in a terminal as follows: + +```shell +./kafka-console-producer.sh --broker-list --topic --producer.config +``` + +Replace: +* `` with your cluster's broker URL +* `` with the name of your topic +* `` with the name of your properties file including full path to it + + +### Example - console consumer + +You can use the Kafka console consumer tool with {{site.data.reuse.es_name}}. + +After you've created the properties file as described previously, you can run the console consumer in a terminal as follows: + +```shell +./kafka-console-consumer.sh --bootstrap-server --topic --from-beginning --consumer.config +``` + +Replace: +* `` with your cluster's broker URL +* `` with the name of your topic +* `` with the name of your properties file including full path to it diff --git a/_es_11.5/03-getting-started/07-sharing-topic.md b/_es_11.5/03-getting-started/07-sharing-topic.md new file mode 100644 index 00000000..82a612bc --- /dev/null +++ b/_es_11.5/03-getting-started/07-sharing-topic.md @@ -0,0 +1,38 @@ +--- +title: "Sharing topics with Event Endpoint Management" +excerpt: "Find out how to integrate Event Streams with an Event Endpoint Management instance." +categories: getting-started +slug: sharing-topic +toc: true +--- + +After you [set up a connection](../../installing/integrating-eem/) with {{site.data.reuse.eem_name}}, you can share your topic from the {{site.data.reuse.es_name}} UI. + + +Complete the following steps to share your topic with {{site.data.reuse.eem_name}}: + +1. {{site.data.reuse.es_ui_login}} +2. Click the **Topics** tab in the primary navigation and then select a topic from the list of topics. +3. Click **Share for reuse**. +3. In the **Select an instance** section, select the {{site.data.reuse.eem_name}} instance that you want to share your {{site.data.reuse.es_name}} topic with, and then click **Next**. +4. In the **Provide access token** section, enter the [access token]({{ 'eem/security/api-tokens/#creating-a-token' | relative_url }}) that you created earlier, and click **Next**. +5. In the **Review details** section, enter a topic alias. + + - Optional: If available, a sample message of your topic is displayed in the **Sample message** text box. You can also add a sample message now, or edit the sample. + + **Note:** Ensure that you review the sample message in case sensitive information is displayed, as the sample will be available in the {{site.data.reuse.eem_name}} catalog for other users to view. + - Optional: To select a schema from the schema registry, click **Select**. + + **Important:** In {{site.data.reuse.es_name}} 11.3.1 and earlier, if you select a nested Avro schema with custom record type, you must replace the custom record type name in it with the entire schema definition before publishing the topic as an option in {{site.data.reuse.eem_name}}. You can find the schema with the required definition in your schema registry. For more information about editing nested Avro schemas, see [Event Endpoint Management documentation]({{ 'eem/describe/managing-topics/#editing-nested-avro-schemas' | relative_url }}). + + + **Note:** If the schema registry is empty, the **Select** option is unavailable. + +6. Click **Share for reuse** to share your topic with {{site.data.reuse.eem_name}}. + +You are redirected to the {{site.data.reuse.eem_name}} UI from where you can [create]({{ 'eem/describe/managing-topics#create_option' | relative_url}}) and [publish]({{ 'eem/describe/publishing-topics' | relative_url}}) the topic data as an option. After you have published the option, consumers can [subscribe]({{ 'eem/consume-subscribe/approval-requests' | relative_url}}) to an option as required. + +**Note:** When you create an event source in {{site.data.reuse.eem_name}}, the following items are also created: + +- A Kafka user is created in {{site.data.reuse.es_name}} with read access to your topic, and the Kafka user is used by {{site.data.reuse.eem_name}}. +- A cluster definition is created in {{site.data.reuse.eem_name}} with information about your {{site.data.reuse.es_name}} instance and the Kafka user. diff --git a/_es_11.5/04-schemas/01-schemas-overview.md b/_es_11.5/04-schemas/01-schemas-overview.md new file mode 100644 index 00000000..e1d9f8f0 --- /dev/null +++ b/_es_11.5/04-schemas/01-schemas-overview.md @@ -0,0 +1,89 @@ +--- +title: "Schemas overview" +excerpt: "Learn about schemas and the schema registry, and understand how schemas can help manage your data more efficiently." +categories: schemas +slug: overview +toc: true +--- + +Apache Kafka can handle any data, but it does not validate the information in the messages. However, efficient handling of data often requires that it includes specific information in a certain format. Using schemas, you can define the structure of the data in a message, ensuring that both producers and consumers use the correct structure. + +Schemas help producers create data that conforms to a predefined structure, defining the fields that need to be present together with the type of each field. This definition then helps consumers parse that data and interpret it correctly. {{site.data.reuse.es_name}} supports schemas and includes a schema registry for using and managing schemas. + +It is common for all of the messages on a topic to use the same schema. The key and value of a message can each be described by a schema. + +![Schemas: key and value of a message diagram.]({{ 'images' | relative_url }}/Schema_Basics_1.svg "Diagram representing how a schema can help define a structure for the key and value pairs of a message.") + + + +## Schema registry + +Schemas are stored in internal Kafka topics by the [Apicurio Registry](https://www.apicur.io/registry/docs/apicurio-registry/2.6.x/index.html){:target="_blank"}, an open-source schema registry. In addition to storing a versioned history of schemas, Apicurio Registry provides an interface for retrieving them. Each {{site.data.reuse.es_name}} cluster has its own instance of Apicurio Registry providing schema registry functionality. + +![Event Streams Schemas architecture]({{ 'images' | relative_url }}/architectures/ibm-event-automation-diagrams-es-schemaregistry.svg "Diagram showing the architecture of the Event Streams schemas as part of IBM Event Automation.") + +Your producers and consumers validate the data against the specified schema stored in the schema registry. This is in addition to going through Kafka brokers. The schemas do not need to be transferred in the messages this way, meaning the messages are smaller than without using a schema registry. + +![Schema architecture diagram.]({{ 'images' | relative_url }}/Schema_registry_arch.png "Diagram showing a schema registry architecture. A producer is sending messages and a consumer is reading messages, while both are retrieving the schema from the schema registry.") + +If you are migrating to use {{site.data.reuse.es_name}} as your Kafka solution, and have been using a schema registry from a different provider, you can [migrate](../migrating/) to using {{site.data.reuse.es_name}} and the Apicurio Registry. + + + + +## Apache Avro data format + +Schemas are defined using [Apache Avro](https://avro.apache.org/){:target="_blank"}, an open-source data serialization technology commonly used with Apache Kafka. It provides an efficient data encoding format, either by using the compact binary format or a more verbose, but human-readable [JSON](https://www.json.org){:target="_blank"} format. + +The schema registry in {{site.data.reuse.es_name}} uses Apache Avro data formats. When messages are sent in the Avro format, they contain the data and the unique identifier for the schema used. The identifier specifies which schema in the registry is to be used for the message. + +Avro has support for a wide range of data types, including primitive types (`null`, `boolean`, `integer`, `long`, `float`, `double`, `bytes`, and `string`) and complex types (`record`, `enum`, `array`, `map`, `union`, and `fixed`). + +Learn more about how you can [create schemas](../creating) in {{site.data.reuse.es_name}}. + +![Schemas: Avro format.]({{ 'images' | relative_url }}/Schema_Basics_3.svg "Diagram showing a representation of a message sent in Avro format.") + + + +## Serialization and deserialization + +A producing application uses a serializer to produce messages conforming to a specific schema. As mentioned earlier, the message contains the data in Avro format, together with the schema identifier. + +A consuming application then uses a deserializer to consume messages that have been serialized using the same schema. When a consumer reads a message sent in Avro format, the deserializer finds the identifier of the schema in the message, and retrieves the schema from the schema registry to deserialize the data. + +This process provides an efficient way of ensuring that data in messages conform to the required structure. + +Serializers and deserializers that automatically retrieve the schemas from the schema registry as required are provided by {{site.data.reuse.es_name}}. If you need to use schemas in an environment for which serializers or deserializers are not provided, you can [use the command line or UI](../setting-nonjava-apps/#retrieving-the-schema-definition-from-the-schema-registry) directly to retrieve the schemas. + +![Schemas: Serializer and deserializer.]({{ 'images' | relative_url }}/Schema_Basics_4.svg "Diagram showing a representation of where a serializer and a deserializer fits into the Event Streams architecture.") + + + +## Versions and compatibility + +Whenever you add a schema, and any subsequent versions of the same schema, Apicurio Registry validates the format automatically and warns of any issues. You can evolve your schemas over time to accommodate changing requirements. You simply create a new version of an existing schema, and the registry ensures that the new version is compatible with the existing version, meaning that producers and consumers using the existing version are not broken by the new version. + +When you create a new version of the schema, you simply add it to the registry and version it. You can then set your producers and consumers that use the schema to start using the new version. Until they do, both producers and consumers are warned that a new version of the schema is available. + +![Schemas: versions.]({{ 'images' | relative_url }}/Schema_Basics_5.svg "Diagram showing a representation of schema versions.") + +## Lifecycle + +When a new version is used, you can deprecate the previous version. Deprecating means that producing and consuming applications still using the deprecated version are warned that a new version is available to upgrade to. When you upgrade your producers to use the new version, you can disable the older version so it can no longer be used, or you can remove it entirely from the schema registry. + +You can use the {{site.data.reuse.es_name}} UI or CLI to [manage the lifecycle](../manage-lifecycle/) of schemas, including registering, versioning, and deprecating. + +![Schemas: Avro format.]({{ 'images' | relative_url }}/Schema_Basics_6.svg "Diagram showing a representation of schema lifecycle stages.") + + + +## How to get started with schemas + +1. [Create schemas](../creating/#creating-schemas) +2. [Add schemas](../creating/#adding-schemas-to-the-registry) to schema registry +3. Set your [Java](../setting-java-apps/) or [non-Java](../setting-nonjava-apps/) applications to use schemas +4. Manage schema [lifecycle](../manage-lifecycle/) diff --git a/_es_11.5/04-schemas/02-creating-schemas.md b/_es_11.5/04-schemas/02-creating-schemas.md new file mode 100644 index 00000000..927c26bf --- /dev/null +++ b/_es_11.5/04-schemas/02-creating-schemas.md @@ -0,0 +1,121 @@ +--- +title: "Creating and adding schemas" +excerpt: "Learn how to create schemas and add them to the schema registry." +categories: schemas +slug: creating +toc: true +--- + +You can [create schemas](#creating-schemas) in Avro format and then use the {{site.data.reuse.es_name}} UI or CLI to [add them to the Apicurio Registry](#adding-schemas-to-the-registry). + + + + +## Creating schemas + +{{site.data.reuse.es_name}} supports Apache Avro schemas. Avro schemas are written in JSON to define the format of the messages. For more information about Avro schemas, see the [Avro documentation](http://avro.apache.org/docs/1.11.0/spec.html#schemas){:target="_blank"}. + +Apicurio Registry in {{site.data.reuse.es_name}} imports, stores, and uses Avro schemas to serialize and deserialize Kafka messages. Apicurio Registry supports Avro schemas using the `record` complex type. The `record` type can include multiple fields of any data type, primitive or complex. + +Define your Avro schema files and save them by using the `.avsc` or `.json` file extension. + +For example, the following Avro schema defines a `Book` record in the `org.example` namespace, and contains the `Title`, `Author`, and `Format` fields with different data types: + +```json +{ +    "type": "record", +    "name": "Book", +    "namespace": "org.example", +    "fields": [ +        {"name": "Title", "type": "string"}, +        {"name": "Author",  "type": "string"}, +        {"name": "Format", +         "type": { +                    "type": "enum", +                    "name": "Booktype", +                    "symbols": ["HARDBACK", "PAPERBACK"] +                 } +        } +    ] +} +``` + +## Adding schemas to the registry + +To use schemas in Kafka applications, import your schema definitions into the schema registry. Your applications can then retrieve the schemas from the registry as required. + +### Using the UI + +1. {{site.data.reuse.es_ui_login}} +2. Click **Schema registry** in the primary navigation, and then click **Add schema**. +3. Click **Upload definition** and select your Avro schema file. Avro schema files use the `.avsc` or `.json` file extensions.\\ + The file is loaded and its format validated. If the validation finds any problems with the file, a warning message is displayed. +4. Optional: Edit the **Schema name** and **Version** fields.\\ + - The `name` of the record defined in the Avro schema file is added to the **Schema name** field. You can edit this field to add a different name for the schema. Changing the **Schema name** field does not update the Avro schema definition itself. + - The value `1.0.0` is automatically added to the **Version** field as the initial version of the schema. You can edit this field to set a different version number for the schema. +5. Click **Add schema**. The schema is added to the list of schemas in the schema registry. + +### Using the CLI + +1. [Install the {{site.data.reuse.es_name}} CLI plugin](../../installing/post-installation/#installing-the-event-streams-command-line-interface) if not already installed. +2. {{site.data.reuse.es_cli_init_111}} +3. Run the following command to add a schema to the schema registry:\\ + + ```shell + kubectl es schema-add --name --version --file + ``` + +## Adding new schema versions + +Apicurio Registry can store multiple versions of the same schema. As your applications and environments evolve, your schemas need to change to accommodate the requirements. You can import, manage, and use different versions of a schema. As your schemas change, consider the options for [managing their lifecycle](../manage-lifecycle/). + +**Note:** A new version of a schema must be compatible with previous versions. This means that messages that have been serialized with an earlier version of a schema can be deserialized with a later version. To be compatible, fields in later versions of a schema cannot be removed, and any new schema field must have a default value. + +For example, the following Avro schema defines a new version of the `Book` record, adding a `PageCount` field. By including a default value for this field, messages that were serialized with the previous version of this schema (which would not have a `PageCount` value) can still be deserialized using this version. + +```json +{ +    "type": "record", +    "name": "Book", +    "namespace": "org.example", +    "fields": [ +        {"name": "Title", "type": "string"}, +        {"name": "Author",  "type": "string"}, +        {"name": "Format", +         "type": { +                    "type": "enum", +                    "name": "Booktype", +                    "symbols": ["HARDBACK", "PAPERBACK"] +                 } +        }, + {"name": "PageCount",  "type": "int", "default": 0} +    ] +} +``` + +### Using the UI + +1. {{site.data.reuse.es_ui_login}} +2. Click **Schema registry** in the primary navigation. +3. Locate your schema in the list of registered schemas and click its name. The list of versions for the schema is displayed. +4. Click **Add new version** to add a new version of the schema. +5. Click **Upload definition** and select the file that contains the new version of your schema. Avro schema files use the `.avsc` or `.json` file extensions.\\ + The file is loaded and its format validated. If the validation finds any problems with the file, a warning message is displayed. +6. Set a value in the **Version** field to be the version number for this iteration of the schema. For the current list of all versions, click **View all versions**. +7. Click **Add schema**. The schema version is added to the list of all versions for the schema. + +### Using the CLI + +1. [Install the {{site.data.reuse.es_name}} CLI plugin](../../installing/post-installation/#installing-the-event-streams-command-line-interface) if not already installed. +2. {{site.data.reuse.es_cli_init_111}} +3. Run the following command to list all schemas in the schema registry, and find the schema name you want to add a new version to:\\ + + ```shell + kubectl es schemas + ``` + +4. Run the following command to add a new version of the schema to the registry:\\ + + ```shell + kubectl es schema-add --name --version --file + ``` diff --git a/_es_11.5/04-schemas/03-managing-schema-lifecycle.md b/_es_11.5/04-schemas/03-managing-schema-lifecycle.md new file mode 100644 index 00000000..999b7d42 --- /dev/null +++ b/_es_11.5/04-schemas/03-managing-schema-lifecycle.md @@ -0,0 +1,150 @@ +--- +title: "Managing schema lifecycle" +excerpt: "Understand how to manage the lifecycle of schemas." +categories: schemas +slug: manage-lifecycle +toc: true +--- + +Multiple versions of each schema can be stored in the Apicurio Registry. Kafka producers and consumers retrieve the right schema version they use from the registry based on a unique identifier and version. + +When a [new schema version](../creating/#adding-new-schema-versions) is added, you can set both the producer and consumer applications to use that version. You then have the following options to handle earlier versions. + +The lifecycle is as follows: + +1. Add schema +2. Add new schema version +3. Deprecate version or entire schema +4. Disable version or entire schema +5. Remove version or entire schema + +## Deprecating + +If you want your applications to use a new version of a schema, you can set the earlier version to **Deprecated**. When a version is deprecated, the applications using that version receive a message to warn them to stop using it. Applications can continue to use the old version of the schema, but warnings will be written to application logs about the schema version being deprecated. You can customize the message to be provided in the logs, such as providing information for what schema or version to use instead. + +Deprecated versions are still available in the registry and can be used again. + +**Note:** You can deprecate an entire schema, not just the versions of that schema. If the entire schema is set to deprecated, then all of its versions are reported as deprecated (including any new ones added). + +### Using the UI + +1. {{site.data.reuse.es_ui_login}} +2. Click **Schema registry** in the primary navigation. +3. Select the schema you want to deprecate from the list. +4. Set the entire schema or a selected version of the schema to be deprecated:\\ + - If you want to deprecate the entire schema and all its versions, click the **Manage schema** tab, and set **Mark schema as deprecated** to on. + - To deprecate a specific version, select it from the list, and click the **Manage version** tab for that version. Then set **Mark schema as deprecated** to on. + +Deprecated schemas and versions are marked with a **Deprecated** flag on the UI. + +You can re-activate a schema or its version by setting **Mark schema as deprecated** to off. + +### Using the CLI + +1. {{site.data.reuse.es_cli_init_111}} +2. Run the following command to deprecate a schema version: + + ```shell + kubectl es schema-modify --deprecate --name --version + ``` + + To deprecate an entire schema, do not specify the `--version ` option. + + To re-activate a schema version: + + ```shell + kubectl es schema-modify --activate --name --version + ``` + + To re-activate an entire schema, do not specify the `--version ` option. + +**Note:** `` is the integer ID that is displayed when listing schema versions using the following command: + +```shell +kubectl es schema +``` + +## Disabling + +If you want your applications to stop using a specific schema, you can set the schema version to **Disabled**. If you disable a version, applications will be prevented from producing and consuming messages using it. After being disabled, a schema can be enabled again to allow applications to use the schema. + +When a schema is disabled, applications that want to use the schema receive an error response. + +**Note:** You can disable a entire schema, not just the versions of that schema. If the entire schema is disabled, then all of its versions are disabled as well, which means no version of the schema can be used by applications (including any new ones added). + +### Using the UI + +1. {{site.data.reuse.es_ui_login}} +2. Click **Schema registry** in the primary navigation. +3. Select the schema you want to disable from the list. +4. Set the entire schema or a selected version of the schema to be disabled: + + - If you want to disable the entire schema and all its versions, click the **Manage schema** tab, and click **Disable schema**, then click **Disable**. + - To disable a specific version, select it from the list, and click the **Manage version** tab for that version. Then click **Disable version**, then click **Disable**. + + You can re-enable a schema by clicking **Enable schema**, and re-enable a schema version by clicking **Re-enable version**. + +### Using the CLI + +1. {{site.data.reuse.es_cli_init_111}} +2. Run the following command to disable a schema version: + + ```shell + kubectl es schema-modify --disable --name --version + ``` + + To disable an entire schema, do not specify the `--version ` option. + + To re-enable a schema version: + + ```shell + kubectl es schema-modify --enable --name --version + ``` + + To re-enable an entire schema, do not specify the `--version ` option. + +**Note:** `` is the integer ID that is displayed when listing schema versions using the following command: + +```shell +kubectl es schema +``` + + +## Removing + +If a schema version has not been used for a period of time, you can remove it from the schema registry. Removing a schema version means it will be permanently deleted from the schema registry of your {{site.data.reuse.es_name}} instance, and applications will be prevented from producing and consuming messages using it. + +**Important:** You cannot reverse the removal of a schema. This action is permanent. + +**Note:** You can remove an entire schema, including all of its versions. If the entire schema is removed, then all of its versions are permanently deleted from the schema registry of your {{site.data.reuse.es_name}} instance. + +### Using the UI + +1. {{site.data.reuse.es_ui_login}} +2. Click **Schema registry** in the primary navigation. +3. Select the schema you want to remove from the list. +4. Remove the entire schema or a selected version of the schema:\\ + - If you want to remove the entire schema and all its versions, click the **Manage schema** tab, and click **Remove schema**, then click **Remove**. + - To remove a specific version, select it from the list, and click the **Manage version** tab for that version. Then click **Remove version**, then click **Remove**. + + **Important:** This action is permanent and cannot be reversed. + + +### Using the CLI + +1. {{site.data.reuse.es_cli_init_111}} +2. Run the following command to remove a schema version:\\ + + ```shell + kubectl es schema-remove --name --version + ``` + + To remove an entire schema, do not specify the `--version ` option. + + **Important:** This action is permanent and cannot be reversed. + +**Note:** `` is the integer ID that is displayed when listing schema versions using the following command: + +```shell +kubectl es schema +``` diff --git a/_es_11.5/04-schemas/04-setting-java-apps-to-use-schemas.md b/_es_11.5/04-schemas/04-setting-java-apps-to-use-schemas.md new file mode 100644 index 00000000..d3ecd629 --- /dev/null +++ b/_es_11.5/04-schemas/04-setting-java-apps-to-use-schemas.md @@ -0,0 +1,286 @@ +--- +title: "Setting Java applications to use schemas" +excerpt: "Set up your Java applications to use schemas." +categories: schemas +slug: setting-java-apps +toc: true +--- + +If you have Kafka producer or consumer applications written in Java, use the following guidance to set them up to use schemas. + +**Note:** If you have Kafka clients written in other languages than Java, see the guidance about [setting up non-Java applications](../setting-nonjava-apps) to use schemas. + + + + +## Preparing the setup + +To use schemas stored in the Apicurio Registry in {{site.data.reuse.es_name}}, your client applications need to be able to serialize and deserialize messages based on schemas. + +- Producing applications use a serializer to produce messages conforming to a specific schema, and use unique identifiers in the message to determine which schema is being used. +- Consuming applications then use a deserializer to consume messages that have been serialized using the same schema. The schema is retrieved from the schema registry based on the unique identifiers in the message. + +The {{site.data.reuse.es_name}} UI provides help with setting up your Java applications to use schemas. + +**Note:** Apicurio Registry in {{site.data.reuse.es_name}} works with multiple schema registry `serdes` libraries, including the Apicurio Registry `serdes` library and the deprecated {{site.data.reuse.es_name}} schema registry `serdes` library. You can use the Apicurio Registry `serdes` library in your applications by adding Maven dependencies to your project `pom.xml` files. The following instructions and code snippets use the Apicurio Registry `serdes` library. For more information, also see the guidance about [setting up Java applications to use schemas with the Apicurio Registry serdes library](../setting-java-apps-apicurio-serdes). + +To set up your Java applications to use the Apicurio Registry and the Apicurio Registry `serdes` library, prepare the connection for your application as follows: + +1. {{site.data.reuse.es_ui_login}} +2. Ensure you have [added schemas](../creating) to the registry. +3. Click **Schema registry** in the primary navigation. +4. Select a schema from the list and click the row for the schema. +5. Click **Connect to the latest version**. Alternatively, if you want to use a different version of the schema, click the row for the schema version, and click **Connect to this version**. +6. Set the preferences for your connection in the **Configure the schema connection** section. Use the defaults or change them by clicking **Change configuration**. + + - For producers, set the method for **Message encoding**. + + - **Binary** (default): Binary-encoded messages are smaller and typically quicker to process. However the message data is not human-readable without an application that is able to apply the schema. + - **JSON**: JSON-encoded messages are human-readable and can still be used by consumers that are not using the {{site.data.reuse.es_name}} schema registry. + +7. If any configuration was changed, click **Save**. +8. Click **Generate credentials** to generate SCRAM or Mutual TLS credentials and follow the instructions.\\ + **Important:** Ensure you make note of the information provided upon completion as you will need this later.\\ + Alternatively you can generate credentials later by using the {{site.data.reuse.es_name}} [UI](../../security/managing-access#creating-a-kafkauser-in-the-event-streams-ui) or [CLI](../../security/managing-access#creating-a-kafkauser-in-the-event-streams-cli). +9. Click **Generate connection details**. +10. Click **Download certificate** to download the cluster PKCS12 certificate. This is the Java truststore file which contains the server certificate. Take a copy of the **Certificate password** for use with the certificate in your application. +11. If your project uses Maven, add the following dependency to your project `pom.xml` file to use the Apicurio Registry `serdes` library: + + ```xml + + io.apicurio + apicurio-registry-serdes-avro-serde + 2.6.2.Final + + ``` + + Alternatively, if your project does not use Maven, select the **Use JARs** tab and click **Java dependencies** to download the java dependencies JAR files to use for your application in its code. +12. Depending on your application, click the **Producer** or **Consumer** tab, and copy the sample Java code snippets displayed. The sample code snippets include the settings you configured to set up your applications to use the schema. +13. Add the required snippets into your application code as described in the following sections. + - [Setting up producers to use schemas](#setting-up-producers-to-use-schemas) + - [Setting up consumers to use schemas](#setting-up-consumers-to-use-schemas) + +## Setting up producers to use schemas + +1. Ensure you have [prepared](#preparing-the-setup) for the setup, including configuring connection settings, downloading Java dependencies if not using Maven, and copying code snippets for a producing application. +2. If your project does not use Maven, ensure you add the location of the JAR files to the build path of your producer Kafka application. +3. Use the code snippets you copied from the UI for a producer and add them to your application code. + +The code snippet from the **Imports** section includes Java imports to paste into your class, and sets up the application to use the Apicurio Registry `serdes` library, for example: + +```java +import java.util.Properties; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.StringSerializer; + +import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.avro.AvroKafkaSerializer; +import io.apicurio.registry.serde.avro.AvroKafkaSerdeConfig; +import io.apicurio.rest.client.config.ApicurioClientConfig; + +``` + +The code snippet from the **Connection properties** section specifies connection and access permission details for your {{site.data.reuse.es_name}} cluster and the Apicurio Registry, for example: + +```java +Properties props = new Properties(); +// TLS Properties +props.put(SslConfigs.SSL_PROTOCOL_CONFIG, "TLSv1.3"); +props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, ); +props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, ); + +// SCRAM authentication properties - uncomment to connect using Scram +//props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); +//props.put(SaslConfigs.SASL_MECHANISM, "SCRAM-SHA-512"); +//String saslJaasConfig = "org.apache.kafka.common.security.scram.ScramLoginModule required " +// + "username="" password="";"; +//props.put(SaslConfigs.SASL_JAAS_CONFIG, saslJaasConfig); + +// Mutual authentication properties - uncomment to connect using Mutual authentication +//props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); +//props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, ); +//props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, ); + +// Apicurio Registry connection +props.put(SerdeConfig.REGISTRY_URL, ); +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_LOCATION, ); +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_PASSWORD, ); +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_TYPE, "PKCS12"); + +// SCRAM authentication properties - uncomment to connect to Apicurio Registry using Scram +//props.put(SerdeConfig.AUTH_USERNAME, ); +//props.put(SerdeConfig.AUTH_PASSWORD, ); + +// Mutual authentication properties - uncomment to connect to Apicurio Registry +// using Mutual authentication +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_LOCATION, ) +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_PASSWORD, ); +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_TYPE, "PKCS12"); + +// Kafka connection +props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, ); +``` + +**Note:** Follow the instructions in the code snippet to uncomment lines. Replace `` with the path to the Java truststore file you downloaded earlier, `` with the truststore password which has the permissions needed for your application, `` with the bootstrap address (find out how to [obtain the address](../../getting-started/connecting/#obtaining-the-bootstrap-address)), and `` with the endpoint address for Apicurio Registry in {{site.data.reuse.es_name}}. For SCRAM, replace the `` and `` with the SCRAM username and password. For Mutual authentication, replace `` with the path to the `user.p12` file extracted from the `.zip` file downloaded earlier and `` with the contents of the `user.password` file in the same `.zip` file. + +For more information about the configuration keys and values to use with the {{site.data.reuse.es_name}} `serdes` library, see the `SchemaRegistryConfig` class in the [schema API reference]({{'schema-api' | relative_url }}){:target="_blank"}. + +The code snippet from the **Producer code** section defines properties for the producer application that set it to use the schema registry and the correct schema, for example: + +```java +// Set the value serializer for produced messages to use the Apicurio Avro serializer +props.put("key.serializer", StringSerializer.class); +props.put("value.serializer", AvroKafkaSerializer.class); + +// Set the encoding type used by the message serializer +props.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_BINARY); + +// Get a new Generic KafkaProducer +KafkaProducer producer = new KafkaProducer<>(props); + +// Read the schema for the Generic record from a local schema file +Schema.Parser schemaDefinitionParser = new Schema.Parser(); +Schema schema = schemaDefinitionParser.parse( + new File("path/to/schema.avsc")); + +// Get a new Generic record based on the schema +GenericRecord genericRecord = new GenericData.Record(schema); + +// Add fields and values to the genericRecord, for example: +// genericRecord.put("title", "this is the value for a title field"); + +// Prepare the record +ProducerRecord producerRecord = new ProducerRecord(, genericRecord); + +// Send the record to Kafka +producer.send(producerRecord); + +// Close the producer +producer.close(); +``` + +The Kafka configuration property `value.serializer` is set to `AvroKafkaSerializer.class`, telling Kafka to use the Apicurio Registry Avro Kafka serializer for message values when producing messages. You can also use the Apicurio Registry Avro Kafka serializer for message keys. + +For more information about the configuration keys and values to use with the Apicurio Registry `serdes` library, see the [Apicurio Registry documentation](https://www.apicur.io/registry/docs/apicurio-registry/2.6.x/index.html){:target="_blank"}. + +**Note:** Use the `put` method in the `GenericRecord` class to set field names and values in your message. + +**Note:** Replace `` with the name of the topic to produce messages to. + +## Setting up consumers to use schemas + +1. Ensure you have [prepared](#preparing-the-setup) for the setup, including configuring connection settings, downloading Java dependencies if not using Maven, and copying code snippets for a consuming application. +2. If your project does not use Maven, ensure you add the location of the JAR files to the build path of your consumer Kafka application. +3. Use the code snippets you copied from the UI for a consumer and add them to your application code. + +The code snippet from the **Imports** section includes Java imports to paste into your class, and sets up the application to use the Apicurio Registry `serdes` library, for example: + +```java +import java.time.Duration; +import java.util.Arrays; +import java.util.Properties; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.serialization.StringDeserializer; + +import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; +import io.apicurio.rest.client.config.ApicurioClientConfig; + +``` + +The code snippet from the **Connection properties** section specifies connection and access permission details for your {{site.data.reuse.es_name}} cluster, for example: + +```shell +Properties props = new Properties(); +// TLS Properties +props.put(SslConfigs.SSL_PROTOCOL_CONFIG, "TLSv1.3"); +props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, ); +props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, ); + +// SCRAM authentication properties - uncomment to connect using Scram +//props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); +//props.put(SaslConfigs.SASL_MECHANISM, "SCRAM-SHA-512"); +//String saslJaasConfig = "org.apache.kafka.common.security.scram.ScramLoginModule required " +// + "username="" password="";"; +//props.put(SaslConfigs.SASL_JAAS_CONFIG, saslJaasConfig); + +// Mutual authentication properties - uncomment to connect using Mutual authentication +//props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); +//props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, ); +//props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, ); + +// Apicurio Registry connection +props.put(SerdeConfig.REGISTRY_URL, ); +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_LOCATION, ); +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_PASSWORD, ); +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_TYPE, "PKCS12"); + +// SCRAM authentication properties - uncomment to connect to Apicurio Registry using Scram +//props.put(SerdeConfig.AUTH_USERNAME, ); +//props.put(SerdeConfig.AUTH_PASSWORD, ); + +// Mutual authentication properties - uncomment to connect to Apicurio Registry +// using Mutual authentication +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_LOCATION, ) +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_PASSWORD, ); +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_TYPE, "PKCS12"); + +// Kafka connection +props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, ); +``` + +**Note:** Follow the instructions in the code snippet to uncomment lines. Replace `` with the path to the Java truststore file you downloaded earlier, `` with the truststore password which has the permissions needed for your application, `` with the bootstrap address (find out how to [obtain the address](../../getting-started/connecting/#obtaining-the-bootstrap-address)), and `` with the endpoint address for Apicurio Registry in {{site.data.reuse.es_name}}. For SCRAM, replace the `` and `` with the SCRAM username and password. For Mutual authentication, replace `` with the path to the `user.p12` file extracted from the `.zip` file downloaded earlier and `` with the contents of the `user.password` file in the same `.zip` file. + +For more information about the configuration keys and values to use with the Apicurio Registry `serdes` library, see the [Apicurio Registry documentation](https://www.apicur.io/registry/docs/apicurio-registry/2.6.x/index.html){:target="_blank"}. + +The code snippet from the **Consumer code** section defines properties for the consumer application that set it to use the schema registry and the correct schema, for example: + +```shell + +// Set the value deserializer for consumed messages to use the Apicurio Avro deserializer +props.put("key.deserializer", StringDeserializer.class); +props.put("value.deserializer", AvroKafkaDeserializer.class); + +// Set the consumer group ID in the properties +props.put("group.id", ); + +// Get a new KafkaConsumer +KafkaConsumer consumer = new KafkaConsumer<>(props); + +// Subscribe to the topic +consumer.subscribe(Arrays.asList()); + +// Poll the topic to retrieve records +while(true) { + ConsumerRecords records = consumer.poll(Duration.ofSeconds(5)); + + for (ConsumerRecord record : records) { + GenericRecord genericRecord = record.value(); + // Get fields and values from the genericRecord, for example: + // String titleValue = genericRecord.get("title").toString(); + } +} +``` + +The Kafka configuration property `value.deserializer` is set to `AvroKafkaDeserializer.class`, telling Kafka to use the Apicurio Registry Avro Kafka deserializer for message values when consuming messages. You can also use the Apicurio Registry Avro Kafka deserializer for message keys. + +For more information about the configuration keys and values to use with the Apicurio Registry `serdes` library, see the [Apicurio Registry documentation](https://www.apicur.io/registry/docs/apicurio-registry/2.6.x/index.html){:target="_blank"}. + +**Note:** Use the `get` method in the `GenericRecord` class to get field names and values. + +**Note:** Replace `` with the name of the consumer group to use and `` with the name of the topic to consume messages from. + +## Other clients + +To configure other types of Kafka clients, such as Kafka Streams applications and Kafka Connect connectors, and for information about other configuration options for the Apicurio Registry `serdes` library, see the guidance about [setting up Java applications to use schemas with the Apicurio Registry serdes library](../setting-java-apps-apicurio-serdes). diff --git a/_es_11.5/04-schemas/05-setting-nonjava-apps-to-use-schemas.md b/_es_11.5/04-schemas/05-setting-nonjava-apps-to-use-schemas.md new file mode 100644 index 00000000..36dcb311 --- /dev/null +++ b/_es_11.5/04-schemas/05-setting-nonjava-apps-to-use-schemas.md @@ -0,0 +1,62 @@ +--- +title: "Setting non-Java applications to use schemas" +excerpt: "Set up your non-Java applications to use schemas." +categories: schemas +slug: setting-nonjava-apps +toc: true +--- + +If you have producer or consumer applications created in languages other than Java, use the following guidance to set them up to use schemas. You can also use the [REST producer API](../using-with-rest-producer) to send messages that are encoded with a schema. + + + +For a producer application: +1. Retrieve the schema definition that you will be using from Apicurio Registry in {{site.data.reuse.es_name}} and save it in a local file. +2. Use an Apache Avro library for your programming language to read the schema definition from the local file and encode a Kafka message with it. +3. Set the schema registry headers in the Kafka message, so that consumer applications can understand which schema and version was used to encode the message, and which encoding format was used. +4. Send the message to Kafka. + +For a consumer application: +1. Retrieve the schema definition that you will be using from Apicurio Registry in {{site.data.reuse.es_name}} and save it in a local file. +2. Consume a message from Kafka. +3. Check the headers for the Kafka message to ensure they match the expected schema ID and schema version ID. +4. Use the Apache Avro library for your programming language to read the schema definition from the local file and decode the Kafka message with it. + +## Retrieving the schema definition from the schema registry + +### Using the UI + +1. {{site.data.reuse.es_ui_login}} +2. Click **Schema registry** in the primary navigation and find your schema in the list. +3. Copy the schema definition into a new local file. + + - For the latest version of the schema, expand the row. Copy and paste the schema definition into a new local file. + - For a different version of the schema, click on the row and then select the version to use from the list of schema versions. Click the **Schema definition** tab and then copy and paste the schema definition into a new local file. + +### Using the CLI + +1. {{site.data.reuse.es_cli_init_111}} +2. Run the following command to list all the schemas in the schema registry: + + ```shell + kubectl es schemas + ``` + +3. Select your schema from the list and run the following command to list all the versions of the schema: + + ```shell + kubectl es schema + ``` + +4. Select your version of the schema from the list and run the following command to retrieve the schema definition for the version and copy it into a new local file: + + ```shell + kubectl es schema --version > .avsc + ``` + +**Note:** ``is the integer ID that is displayed when listing schema versions using the following command: + +```shell +kubectl es schema +``` + diff --git a/_es_11.5/04-schemas/06-migrating-to-registry.md b/_es_11.5/04-schemas/06-migrating-to-registry.md new file mode 100644 index 00000000..07d2a0fd --- /dev/null +++ b/_es_11.5/04-schemas/06-migrating-to-registry.md @@ -0,0 +1,114 @@ +--- +title: "Migrating existing applications to the Event Streams schema registry" +excerpt: "Read about how you can migrate to using the Event Streams schema registry." +categories: schemas +slug: migrating +toc: true +--- + +If you are using the Confluent Platform schema registry, {{site.data.reuse.es_name}} provides a migration path for moving your Kafka consumers and producers over to use the Apicurio Registry in {{site.data.reuse.es_name}}. + + + +## Migrating schemas to Apicurio Registry in {{site.data.reuse.es_name}} + +To migrate schemas, you can use schema auto-registration in your Kafka producer, or you can manually migrate schemas by downloading the schema definitions from the Confluent Platform schema registry and adding them to the Apicurio Registry in {{site.data.reuse.es_name}}. + +### Migrating schemas with auto-registration + +When using auto-registration, the schema will be automatically uploaded to the Apicurio Registry in {{site.data.reuse.es_name}}, and named with the subject ID (which is based on the subject name strategy in use) and a random suffix. + +Auto-registration is enabled by default in the Confluent Platform schema registry client library. To disable it, set the `auto.register.schemas` property to `false`. + +**Note:** To auto-register schemas in the Apicurio Registry in {{site.data.reuse.es_name}}, you need credentials that have producer permissions and permission to create schemas. You can generate credentials by using the {{site.data.reuse.es_name}} [UI](../../security/managing-access/#creating-a-kafkauser-in-the-event-streams-ui) or [CLI](../../security/managing-access/#creating-a-kafkauser-in-the-event-streams-cli). + +### Migrating schemas manually + +To manually migrate the schemas, download the schema definitions from the Confluent Platform schema registry, and add them to the Apicurio Registry in {{site.data.reuse.es_name}}. When manually adding schemas to the Apicurio Registry in {{site.data.reuse.es_name}}, the provided schema name must match the subject ID used by the Confluent Platform schema registry subject name strategy. + +If you are using the default `TopicNameStrategy`, the schema name must be `-<'value'|'key'>` + +If you are using the `RecordNameStrategy`, the schema name must be `.` + +For example, if you are using the default `TopicNameStrategy` as your subject name strategy, and you are serializing your data into the message value and producing to the **MyTopic** topic, then the schema name you must provide when adding the schema in the UI must be `MyTopic-value` + +For example, if you are using the `RecordNameStrategy` as your subject name strategy, and the schema definition file begins with the following, then the schema name you must provide when adding the schema in the UI must be `org.example.Book`: + +```json +{ +    "type": "record", +    "name": "Book", +    "namespace": "org.example", +    "fields": [ +... +``` + +If you are using the {{site.data.reuse.es_name}} CLI, run the following command when adding the schema: + +```shell +kubectl es schema-add --create --name org.example.Book --version 1.0.0 --file /path/to/Book.avsc +``` + +## Migrating a Kafka producer application + +To migrate a Kafka producer application that uses the Confluent Platform schema registry, secure the connection from your application to {{site.data.reuse.es_name}}, and add additional properties to enable the Confluent Platform schema registry client library to interact with the Apicurio Registry in {{site.data.reuse.es_name}}. + +1. Configure your producer application to [secure the connection](../../getting-started/connecting/#securing-the-connection) between the producer and {{site.data.reuse.es_name}}. +2. Retrieve the full URL for the {{site.data.reuse.es_name}} [API endpoint](../../connecting/rest-api/#prerequisites), including the host name and port number by using the following command: + + ```shell + kubectl es init + ``` + +3. Ensure you add the following schema properties to your Kafka producers: + + Property name | Property value + ---------------------|---------------- + `schema.registry.url` | `https://:` + `basic.auth.credentials.source` | `SASL_INHERIT` + + You can also use the following code snippet for Java applications: + + ```shell + props.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "https://:/apis/ccompat/v6"); + props.put(AbstractKafkaSchemaSerDeConfig.BASIC_AUTH_CREDENTIALS_SOURCE, "SASL_INHERIT"); + ``` + +4. Set the Java SSL truststore JVM properties to allow the Confluent Platform schema registry client library to make HTTPS calls to the Apicurio Registry in {{site.data.reuse.es_name}}. For example: + + ```shell + export KAFKA_OPTS="-Djavax.net.ssl.trustStore=/path/to/es-cert.jks \  +       -Djavax.net.ssl.trustStorePassword=password" + ``` + +## Migrating a Kafka consumer application + +To migrate a Kafka consumer application that uses the Confluent Platform schema registry, secure the connection from your application to {{site.data.reuse.es_name}}, and add additional properties to enable the Confluent Platform schema registry client library to interact with the Apicurio Registry in {{site.data.reuse.es_name}}. + +1. Configure your consumer application to [secure the connection](../../getting-started/connecting/#securing-the-connection) between the consumer and {{site.data.reuse.es_name}}. +2. Retrieve the full URL for the {{site.data.reuse.es_name}} [API endpoint](../../connecting/rest-api/#prerequisites), including the host name and port number by using the following command: + + ```shell + kubectl es init + ``` + +3. Ensure you add the following schema properties to your Kafka producers: + + Property name | Property value + ---------------------|---------------- + `schema.registry.url` | `https:///apis/ccompat/v6` + `basic.auth.credentials.source` | `SASL_INHERIT` + + You can also use the following code snippet for Java applications: + + ```shell + props.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "https:///apis/ccompat/v6"); + props.put(AbstractKafkaSchemaSerDeConfig.BASIC_AUTH_CREDENTIALS_SOURCE, "SASL_INHERIT"); + ``` + +4. Set the Java SSL truststore JVM properties to allow the Confluent Platform schema registry client library to make HTTPS calls to the Apicurio Registry in {{site.data.reuse.es_name}}. For example: + + ```shell + export KAFKA_OPTS="-Djavax.net.ssl.trustStore=/path/to/es-cert.jks \  +       -Djavax.net.ssl.trustStorePassword=password" + ``` diff --git a/_es_11.5/04-schemas/07-using-schemas-with-rest-producer.md b/_es_11.5/04-schemas/07-using-schemas-with-rest-producer.md new file mode 100644 index 00000000..f2e36264 --- /dev/null +++ b/_es_11.5/04-schemas/07-using-schemas-with-rest-producer.md @@ -0,0 +1,50 @@ +--- +title: "Using schemas with the REST producer API" +excerpt: "You can use schemas with the Event Streams REST producer API." +categories: schemas +slug: using-with-rest-producer +toc: false +--- + +You can use schemas when producing messages with the {{site.data.reuse.es_name}} [REST producer API](../../connecting/rest-api/). You simply add the following parameters to the API call: + +- `schemaname`: The name of the schema you want to use when producing messages. +- `schemaversion`: The schema version you want to use when producing messages. + +For example, to use cURL to produce messages to a topic with the producer API, and specify the schema to be used, run the curl command as follows: + + Using SCRAM Basic Authentication: + +```shell +curl -H "Authorization: " -H "Accept: application/json" -H "Content-Type: text/plain" -d '' --cacert es-cert.pem "https:///topics//records?schemaname=&schemaversion=" +``` + +Using TLS Mutual Authentication: + +```shell +curl -H "Accept: application/json" -H "Content-Type: text/plain" -d '' --cacert es-cert.pem --key user.key --cert user.crt "https:///topics//records?schemaname=&schemaversion=" +``` + +**Note:** Replace the values in brackets as follows: +- `` with the **SCRAM Basic Authentication token** which is generated using the {{site.data.reuse.es_name}} [UI](../../security/managing-access#creating-a-kafkauser-in-the-event-streams-ui). +- `` with the producer URL which is available from **Connect to this cluster** or **Connect to this topic** and copy the URL in the **Producer endpoint and credentials** section. +- `` with your topic name. +- `` with the name of your schema that is stored in Apicurio Registry in {{site.data.reuse.es_name}}. +- `` with the version of your schema that is stored in Apicurio Registry in {{site.data.reuse.es_name}}. +- `` with your avro encoded message. + +The `es-cert.pem` certificate is downloaded by running the following command: + +```shell +kubectl es certificates --format pem +``` + +The `user.key` and `user.crt` files are downloaded in a .zip file by clicking **Generate credentials** in the **Producer endpoint and credentials** section of **Connect to this cluster** or **Connect to this topic**, selecting **Mutual TLS certificate**, following the instructions in the wizard and clicking **Download certificates**. + +By adding these parameters to the API call, a lookup is done on the specified schema and its version to check if it is valid. If valid, the correct message headers are set for the produced message. + +**Important:** When using the producer API, the lookup does not validate the data in the request to see if it matches the schema. Ensure the message conforms to the schema, and that it has been encoded in the Apache Avro binary or JSON encoding format. If the message does not conform and is not encoded with either of those formats, consumers will not be able to deserialize the data. + +If the message has been encoded in the Apache Avro binary format, ensure the HTTP `Content-Type` header is set to `application/octet-stream`. + +If the message has been encoded in the Apache Avro JSON format, ensure the HTTP `Content-Type` header is set to `application/json`. diff --git a/_es_11.5/04-schemas/08-setting-java-apps-to-use-apicurio-serdes.md b/_es_11.5/04-schemas/08-setting-java-apps-to-use-apicurio-serdes.md new file mode 100644 index 00000000..b8a1bd16 --- /dev/null +++ b/_es_11.5/04-schemas/08-setting-java-apps-to-use-apicurio-serdes.md @@ -0,0 +1,490 @@ +--- +title: "Setting Java applications to use schemas with the Apicurio Registry serdes library" +excerpt: "Set up your Java applications to use schemas with the Apicurio Registry serdes library." +categories: schemas +slug: setting-java-apps-apicurio-serdes +toc: true +--- + +If you have Kafka producer or consumer applications written in Java, use the following guidance to set them up to use schemas and the Apicurio Registry `serdes` library. + +**Note:** If you have Kafka clients written in other languages than Java, see the guidance about [setting up non-Java applications](../setting-nonjava-apps) to use schemas. + + + + +## Preparing the setup + +To use schemas stored in the Apicurio Registry in {{site.data.reuse.es_name}}, your client applications need to be able to serialize and deserialize messages based on schemas. + +- Producing applications use a serializer to produce messages conforming to a specific schema, and use unique identifiers in the message headers to determine which schema is being used. +- Consuming applications then use a deserializer to consume messages that have been serialized using the same schema. The schema is retrieved from the schema registry based on the unique identifiers in the message headers. + +The {{site.data.reuse.es_name}} UI provides help with setting up your Java applications to use schemas. + +**Note:** Apicurio Registry in {{site.data.reuse.es_name}} works with multiple schema registry `serdes` libraries, including the Apicurio Registry `serdes` library. The following instructions and code snippets use the Apicurio Registry `serdes` library. + +To set up your Java applications to use the Apicurio Registry `serdes` library with Apicurio Registry in {{site.data.reuse.es_name}}, prepare the connection for your application as follows: + +1. {{site.data.reuse.es_ui_login}} +2. Ensure you have [added schemas](../creating) to the registry. +3. Click **Schema registry** in the primary navigation. +4. Select a schema from the list and click the row for the schema. +5. Click **Connect to the latest version**. Alternatively, if you want to use a different version of the schema, click the row for the schema version, and click **Connect to this version**. +6. Set the preferences for your connection in the **Configure the schema connection** section. Use the defaults or change them by clicking **Change configuration**. + + - For producers, set the method for **Message encoding**. + + - **Binary** (default): Binary-encoded messages are smaller and typically quicker to process. However the message data is not human-readable without an application that is able to apply the schema. + - **JSON**: JSON-encoded messages are human-readable and can still be used by consumers that are not using the {{site.data.reuse.es_name}} schema registry. + + - For consumers, set the **Message deserialization behavior** for the behavior to use when an application encounters messages that do not conform to the schema. + + - **Strict** (default): Strict behavior means the message deserializer will fail to process non-conforming messages, throwing an exception if one is encountered. + - **Permissive**: Permissive behavior means the message deserializer will return a null message when a non-conforming message is encountered. It will not throw an exception, and will allow a Kafka consumer to continue to process further messages. + +7. If any configuration was changed, click **Save**. +8. Click **Generate credentials** to generate SCRAM or Mutual TLS credentials and follow the instructions.\\ + **Important:** Ensure you make note of the information provided upon completion as you will need this later.\\ + Alternatively you can generate credentials later by using the {{site.data.reuse.es_name}} [UI](../../security/managing-access#creating-a-kafkauser-in-the-event-streams-ui) or [CLI](../../security/managing-access#creating-a-kafkauser-in-the-event-streams-cli). +9. Click **Generate connection details**. +10. Click **Download certificate** to download the cluster PKCS12 certificate. This is the Java truststore file which contains the server certificate. Take a copy of the **Certificate password** for use with the certificate in your application. +11. Add the following dependency to your project Maven `pom.xml` file to use the Apicurio Registry `serdes` library: + + ```xml + + io.apicurio + apicurio-registry-serdes-avro-serde + 2.6.2.Final + + ``` + +12. If you want to generate specific schema classes from your project Avro schema files, add the following Avro plugin to your project Maven `pom.xml` file, replacing `SCHEMA-FILE-NAME` with the name of your schema file. + + ```xml + + avro + + + + org.apache.avro + avro-maven-plugin + 1.9.2 + + + generate-sources + + schema + + + ${project.basedir}/src/main/resources/avro-schemas/ + + SCHEMA-FILE-NAME.avsc + + ${project.build.directory}/generated-sources + + + + + + + + ``` + +13. Add snippets into your application code as described in the following sections. + - [Setting up producers to use schemas](#setting-up-producers-to-use-schemas) + - [Setting up consumers to use schemas](#setting-up-consumers-to-use-schemas) + +If you are connecting to a bootstrap address that is configured to use [OAuth authentication](../../installing/configuring/#enabling-oauth), complete the following additional steps: + +1. Download the OAuth authentication PKCS12 certificate. This is the Java truststore file which contains the OAuth server certificate. You will also need to know the truststore password as this will need to be configured in your application. +2. You might need to configure the OAuth client libraries as dependencies in your project Maven `pom.xml` file, for example, to use the Strimzi OAuth client library: + + ```xml + + io.strimzi + kafka-oauth-client + 0.9.0 + + ``` + +## Additional configuration options for Apicurio `serdes` library + +The Apicurio `serdes` library supports the following configuration options that can be specified to change how data is serialized. + +| Key | Value | Description +| apicurio.registry.headers.enabled | false | Use this option to store the schema ID in the payload of the message rather than within the header. +| apicurio.registry.avro.encoding | JSON or BINARY | Specify whether to use BINARY (default) or JSON encoding within the Avro serializer + +**Note:** If you are using headers to store the schema ID, you can override the keys used in the header with the following values: + +- `apicurio.registry.headers.key.artifactId.name` +- `apicurio.registry.headers.value.artifactId.name` +- `apicurio.registry.headers.key.version.name` +- `apicurio.registry.headers.value.version.name` +- `apicurio.registry.headers.key.globalId.name` +- `apicurio.registry.headers.value.globalId.name` + +By setting these values you can change the name of the header that the Apicurio `serdes` library uses when adding headers for the `artifactId`, `version`, or `globalId` in the Kafka message. + + +## Setting up producers to use schemas + +1. Ensure you have [prepared](#preparing-the-setup) for the setup, including configuring connection settings, downloading Java dependencies if not using Maven, and copying code snippets for a producing application. +2. If your project does not use Maven, ensure you add the location of the JAR files to the build path of your producer Kafka application. +3. Add the following code snippets to your application code. + +### Imports + +```java +import java.io.File; +import java.util.Properties; + +import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.avro.AvroKafkaSerdeConfig; +import io.apicurio.rest.client.config.ApicurioClientConfig; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.config.SslConfigs; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +``` + +### Connection properties + +```java +Properties props = new Properties(); + +// TLS Properties +props.put(SslConfigs.SSL_PROTOCOL_CONFIG, "TLSv1.3"); +props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, ""); +props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, ""); + +//If your Kafka and Schema registry endpoints do not use the same authentication method, you will need +//to duplicate the properties object - and add the Schema Registry authentication and connection properties +//to 'props', and the Kafka authentication and connection properties to 'kafkaProps'. The different properties objects +//are then supplied to the Serializer and Producer/Consumer respectively. +//Uncomment the next two lines. +//Properties kafkaProps = new Properties(); +//kafkaProps.putAll(props); + +// TLS Properties for Apicurio Registry connection +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_LOCATION, ); +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_PASSWORD, ); +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_TYPE, "PKCS12"); + +// SCRAM authentication properties - uncomment to connect using Scram +//props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); +//props.put(SaslConfigs.SASL_MECHANISM, "SCRAM-SHA-512"); +//String saslJaasConfig = "org.apache.kafka.common.security.scram.ScramLoginModule required " +//+ "username=\"\" password=\"\";"; +//props.put(SaslConfigs.SASL_JAAS_CONFIG, saslJaasConfig); +// +// SCRAM authentication properties for Apicurio Registry connection +//props.put(SerdeConfig.AUTH_USERNAME, ); +//props.put(SerdeConfig.AUTH_PASSWORD, ); + +// OAuth authentication properties - uncomment to connect using OAuth +//props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); +//props.put(SaslConfigs.SASL_MECHANISM, "OAUTHBEARER"); +//String saslJaasConfig = "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required \ + oauth.ssl.truststore.location=> \ + oauth.ssl.truststore.type=PKCS12 \ + oauth.ssl.truststore.password= \ + oauth.ssl.endpoint.identification.algorithm="" \ + oauth.client.id="" \ + oauth.client.secret="" \ + oauth.token.endpoint.uri=""; +//props.put(SaslConfigs.SASL_JAAS_CONFIG, saslJaasConfig); +// You may need to provide a Callback Handler for OAuth. This example shows the Strimzi OAuth callback handler. +//props.put(SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS, "io.strimzi.kafka.oauth.client.JaasClientOauthLoginCallbackHandler") + +// Mutual authentication properties - uncomment to connect using Mutual authentication +//props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); +//props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, ""); +//props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, ""); +// +// Mutual authentication properties for Apicurio Registry connection +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_LOCATION, ) +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_PASSWORD, ); +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_TYPE, "PKCS12"); +// +// URL for Apicurio Registry connection +props.put(SerdeConfig.REGISTRY_URL, ); + +// Set the ID strategy to use the fully-qualified schema name (including namespace) +props.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, "io.apicurio.registry.serde.avro.strategy.RecordIdStrategy"); + +// Kafka connection +props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, ""); +``` + +**Note:** Uncomment lines depending on your authentication settings. Replace: + - `` with the path to the Java truststore file you downloaded earlier. + - `` with the truststore password which has the permissions needed for your application. + - `` with the bootstrap address (find out how to [obtain the address](../../getting-started/connecting/#obtaining-the-bootstrap-address)) + - `` with the endpoint address for Apicurio Registry in {{site.data.reuse.es_name}}. + - For SCRAM, replace `` and `` with the SCRAM username and password. + - For Mutual authentication, replace `` with the path to the `user.p12` file extracted from the `.zip` file downloaded earlier and `` with the contents of the `user.password` file in the same `.zip` file. + - For OAuth, replace `` with the path to the OAuth authentication server truststore file you downloaded earlier, `` with the password for this truststore, `` and `` with the OAuth client credentials, and `` with the OAuth servers token endpoint URI. + + **Note:** You can use OAuth for connection to the Kafka brokers if an OAuth listener has been [configured](../../installing/configuring/#enabling-oauth). However, you cannot use OAuth for the Apicurio Registry. You must use either SCRAM or Mutual TLS for the connection to the registry. + +### Producer code example + +```java +// Set the value serializer for produced messages to use the Apicurio Registry serializer +props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); +props.put("value.serializer", "io.apicurio.registry.serde.avro.AvroKafkaSerializer;"); + +// Get a new Generic KafkaProducer +KafkaProducer producer = new KafkaProducer<>(props); + +// Read in the local schema file +Schema.Parser schemaDefinitionParser = new Schema.Parser(); +Schema schema = schemaDefinitionParser.parse(new File("")); + +// Get a new Generic record based on the schema +GenericRecord genericRecord = new GenericData.Record(schema); + +// Add fields and values to the genericRecord, for example: +// genericRecord.put("title", "this is the value for a title field"); + +// Prepare the record +ProducerRecord producerRecord = + new ProducerRecord("", genericRecord); + +// Send the record to Kafka +producer.send(producerRecord); + +// Close the producer +producer.close(); +``` + +The Kafka configuration property `value.serializer` is set to `io.apicurio.registry.utils.serde.AvroKafkaSerializer`, telling Kafka to use the Apicurio Registry Avro serializer for message values when producing messages. You can also use the Apicurio Registry Avro serializer for message keys. + +**Note:** Use the `put` method in the `GenericRecord` class to set field names and values in your message. + +**Note:** Replace: + +- `` with the name of the topic to produce messages to. +- `` with the path to the Avro schema file. + +**Note:** If you want to retrieve the schema from the registry instead of loading the file locally, you can use the following code: + +```java +// Convert kafka connection properties to a Map +Map config = (Map) props; +// Create the client +RegistryRestClient client = RegistryRestClientFactory.create(REGISTRY_URL, config); + +try { + // Get the schema from apicurio and convert to an avro schema + Schema schema = new Schema.Parser().parse(client.getLatestArtifact(null, artifactId)); +} catch (Exception e) { + e.printStackTrace(); +} +``` + +## Setting up consumers to use schemas + +1. Ensure you have [prepared](#preparing-the-setup) for the setup, including configuring connection settings, downloading Java dependencies if not using Maven, and copying code snippets for a consuming application. +2. If your project does not use Maven, ensure you add the location of the JAR files to the build path of your consumer Kafka application. +3. Add the following code snippets to your application code. + +### Imports + +```java +import java.time.Duration; +import java.util.Arrays; +import java.util.Properties; + +import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.rest.client.config.ApicurioClientConfig; + +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.config.SslConfigs; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.KafkaConsumer; +``` + +### Connection properties + +```java +Properties props = new Properties(); + +// TLS Properties +props.put(SslConfigs.SSL_PROTOCOL_CONFIG, "TLSv1.3"); +props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, ""); +props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, ""); + +//If your Kafka and Schema registry endpoints do not use the same authentication method, you will need +//to duplicate the properties object - and add the Schema Registry authentication and connection properties +//to 'props', and the Kafka authentication and connection properties to 'kafkaProps'. The different properties objects +//are then supplied to the Serializer and Producer/Consumer respectively. +//Uncomment the next two lines. +//Properties kafkaProps = new Properties(); +//kafkaProps.putAll(props); + +// TLS Properties for Apicurio Registry connection +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_LOCATION, ); +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_PASSWORD, ); +props.put(ApicurioClientConfig.APICURIO_REQUEST_TRUSTSTORE_TYPE, "PKCS12"); + +// SCRAM authentication properties - uncomment to connect using Scram +//props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); +//props.put(SaslConfigs.SASL_MECHANISM, "SCRAM-SHA-512"); +//String saslJaasConfig = "org.apache.kafka.common.security.scram.ScramLoginModule required " +//+ "username=\"\" password=\"\";"; +//props.put(SaslConfigs.SASL_JAAS_CONFIG, saslJaasConfig); +// +// SCRAM authentication properties for Apicurio Registry connection +//props.put(SerdeConfig.AUTH_USERNAME, ); +//props.put(SerdeConfig.AUTH_PASSWORD, ); + +// Mutual authentication properties - uncomment to connect using Mutual authentication +//props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); +//props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, ""); +//props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, ""); +// +// Mutual authentication properties for Apicurio Registry connection +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_LOCATION, ) +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_PASSWORD, ); +//props.put(ApicurioClientConfig.APICURIO_REQUEST_KEYSTORE_TYPE, "PKCS12"); + +// URL for Apicurio Registry connection +props.put(SerdeConfig.REGISTRY_URL, ); + +// Kafka connection +props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, ""); +``` + +**Note:** Uncomment lines depending on your authentication settings. Replace: + - `` with the path to the Java truststore file you downloaded earlier. + - `` with the truststore password which has the permissions needed for your application. + - `` with the bootstrap address (find out how to [obtain the address](../../getting-started/connecting/#obtaining-the-bootstrap-address)) + - `` with the endpoint address for Apicurio Registry in {{site.data.reuse.es_name}}. + - For SCRAM, replace `` and `` with the SCRAM username and password. + - For Mutual authentication, replace `` with the path to the `user.p12` file extracted from the `.zip` file downloaded earlier and `` with the contents of the `user.password` file in the same `.zip` file. + +### Consumer code example + +```java +// Set the value deserializer for consumed messages to use the Apicurio Registry deserializer + +props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); +props.put("value.deserializer", "io.apicurio.registry.utils.serde.AvroKafkaDeserializer"); + +// Set the consumer group ID in the properties +props.put("group.id", ""); + +KafkaConsumer consumer = new KafkaConsumer<>(props); + +// Subscribe to the topic +consumer.subscribe(Arrays.asList("")); + +// Poll the topic to retrieve records +while(true) { + + ConsumerRecords records = consumer.poll(Duration.ofSeconds(5)); + + for (ConsumerRecord record : records) { + GenericRecord genericRecord = record.value(); + + // Get fields and values from the genericRecord, for example: + // String titleValue = genericRecord.get("title").toString(); + } +} +``` + +The Kafka configuration property `value.serializer` is set to `io.apicurio.registry.utils.serde.AvroKafkaDeserializer`, telling Kafka to use the Apicurio Registry Avro deserializer for message values when consuming messages. You can also use the Apicurio Registry Avro deserializer for message keys. + +**Note:** Use the `get` method in the `GenericRecord` class to get field names and values. + +**Note:** Replace: +- `` with the name of the consumer group to use. +- `` with the name of the topic to consume messages from. + +## Setting up Kafka Streams applications + +Kafka Streams applications can also use the Apicurio Registry `serdes` library to serialize and deserialize messages. In particular, the `io.apicurio.registry.utils.serde.AvroSerde` class can be used to provide the Apicurio Avro serializer and deserializer for the `"default.value.serde"` or `"default.key.serdes"` properties. Additionally, setting the `"apicurio.registry.use-specific-avro-reader"` property to `"true"` tells the Apicurio Registry `serdes` library to use specific schema classes that have been generated from your project Avro schema files. For example: + +```java +import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.avro.AvroSerde; +import io.apicurio.registry.serde.avro.AvroKafkaSerdeConfig; +import io.apicurio.registry.utils.serde.avro.AvroDatumProvider; + +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.StreamsConfig; + +import java.util.Properties; + + +... + +final Properties streamsConfiguration = new Properties(); + +// URL for Apicurio Registry connection (including basic auth parameters) +streamsConfiguration.put(SerdeConfig.REGISTRY_URL, ); + +// Specify default serializer and deserializer for record keys and for record values. +streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); +streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, AvroSerde.class); + +// Specify using specific (generated) Avro schema classes +streamsConfiguration.put(AvroKafkaSerdeConfig.USE_SPECIFIC_AVRO_READER, "true"); +``` + +## Setting up Kafka Connect connectors + +The Apicurio Registry `converter` library can be used in Kafka Connect to provide the converters that are required to convert between Kafka Connect's format and the Avro serialized format. Source connectors can use the converter to write Avro-formatted values, keys, and headers to Kafka. Sink connectors can use the converter to read Avro-formatted values, keys, and headers from Kafka. In particular, the `io.apicurio.registry.utils.converter.AvroConverter` class can be used to provide the Apicurio Avro converter for the `"key.converter"`, `"value.converter"`, or `"header.converter"` properties. For example: + +```java +key.converter=io.apicurio.registry.utils.converter.AvroConverter +key.converter.apicurio.registry.url=https://:@ + +value.converter=io.apicurio.registry.utils.converter.AvroConverter +value.converter.apicurio.registry.url=https://:@ +``` + +To use the Apicurio Registry `converter` library, add the following dependency to your project Maven `pom.xml` file: + +```xml + + io.apicurio + apicurio-registry-utils-converter + 2.6.2.Final + +``` + +Alternatively, if you are not building your connector, you can download the Apicurio converter artifacts from [Maven](https://repo1.maven.org/maven2/io/apicurio/apicurio-registry-distro-connect-converter/2.6.2.Final/apicurio-registry-distro-connect-converter-2.6.2.Final.tar.gz){:target="_blank"}. + +After downloading, extract the `tar.gz` file and place the folder with all the JARs into a subdirectory within the folder where you are building your `KafkaConnect` image. + +To enable Kafka properties to be pulled from a file, add the following to your `KafkaConnector` or `KafkaConnect` custom resource definition: + +```java +config.providers: file +config.providers.file.class: org.apache.kafka.common.config.provider.FileConfigProvider +``` + +Then reference the Kafka connection details in the `KafkaConnector` custom resource of your connector. For example, the following shows a value converter with SCRAM credentials specified in the custom resource: + +```java +value.converter.apicurio.registry.url: :@ +value.converter.apicurio.registry.request.ssl.truststore.location: "${file:/tmp/strimzi-connect.properties:ssl.truststore.location}" +value.converter.apicurio.registry.request.ssl.truststore.password: "${file:/tmp/strimzi-connect.properties:ssl.truststore.password}" +value.converter.apicurio.registry.request.ssl.truststore.type: "${file:/tmp/strimzi-connect.properties:ssl.truststore.type}" +``` diff --git a/_es_11.5/04-security/01-managing-access.md b/_es_11.5/04-security/01-managing-access.md new file mode 100644 index 00000000..d3e288a5 --- /dev/null +++ b/_es_11.5/04-security/01-managing-access.md @@ -0,0 +1,522 @@ +--- +title: "Managing access" +excerpt: "Managing access for users and applications." +categories: security +slug: managing-access +toc: true +--- + +You can secure your {{site.data.reuse.es_name}} resources by managing the access each user and application has to each resource. + +An {{site.data.reuse.es_name}} cluster can be configured to expose any number of internal or external Kafka listeners. These listeners provide the mechanism for Kafka client applications to communicate with the Kafka brokers. The bootstrap address is used for the initial connection to the cluster. The address will resolve to one of the brokers in the cluster and respond with metadata describing all the relevant connection information for the remaining brokers. + +Each Kafka listener providing a connection to {{site.data.reuse.es_name}} can be configured to authenticate connections with Mutual TLS, SCRAM-SHA-512, or OAuth authentication mechanisms. + +Additionally, the {{site.data.reuse.es_name}} cluster can be configured to authorize operations sent via an authenticated listener. + +**Note:** Schemas in the Apicurio Registry in {{site.data.reuse.es_name}} are a special case and are secured using the resource type of `topic` combined with a prefix of `__schema_`. You can control the ability of users and applications to create, delete, read, and update schemas. + +## Accessing the {{site.data.reuse.es_name}} UI and CLI + +When Kafka authentication is enabled, the {{site.data.reuse.es_name}} UI and CLI requires you to log in by using an {{site.data.reuse.icpfs}} Identity and Access Management (IAM) user, or by using a Kafka user configured with SCRAM-SHA-512 authentication, depending on the authentication type set for the `adminUI` in the `EventStreams` custom resource (see [configuring UI and CLI security](../../installing/configuring/#configuring-ui-and-cli-security)). + +{{site.data.reuse.iam_note}} For other Kubernetes platforms, you can log in by using a Kafka user configured with SCRAM-SHA-512 authentication. You can create a Kafka user by applying a `KafkaUser` [custom resource](#creating-a-kafkauser-by-using-yaml). + +### Managing access to the UI with Keycloak + +When access to the {{site.data.reuse.es_name}} UI is set up with [Keycloak authentication](../../installing/configuring/#configuring-ui-and-cli-security), access for groups and users is managed through Keycloak groups. If you have not previously created any group, you can use the administrative user credentials to [set up a group](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=groups-managing-users-in-keycloak){:target="_blank"}. + +When a Keycloak instance is set up, the Keycloak instance can be shared with the other capabilities in {{site.data.reuse.cp4i}}, allowing for a single identity management system within {{site.data.reuse.cp4i}}. + +Access to the {{site.data.reuse.es_name}} UI requires a Keycloak user with a [role](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=management-cloud-pak-integration-roles-permissions){:target="_blank"} of `eventstreams-admin` or `admin`. The role can be set for the user, or for the group the user is part of. + +The default initial administrator (`admin`) user credentials are stored in a secret within the namespace where you installed the {{site.data.reuse.fs}} operator. Follow the instructions in the [{{site.data.reuse.cp4i}} documentation](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=management-getting-initial-administrator-password){:target="_blank"} to retrieve the username and password. + +**Note:** For Keycloak requirements and limitations, see the [prerequisites](../../installing/prerequisites/#prereqs-keycloak). + +### Managing access to the UI and CLI with SCRAM + +When access to the {{site.data.reuse.es_name}} UI and CLI is set up with [SCRAM authentication](../../installing/configuring/#configuring-ui-and-cli-security), any Kafka users configured to use SCRAM-SHA-512 are able to log in to the UI and run CLI commands by using the Kafka user name and the SCRAM password, which is contained in the secret generated by the creation of the Kafka user (as described in [retrieving credentials](#retrieving-the-credentials-of-a-kafkauser)). + +When a SCRAM user logs in to the UI or runs CLI commands, the permissions of the `KafkaUser` determine which panels or functionality are available to the user. This allows the cluster administrator to grant different permissions to different users, permitting administrative actions through the UI or CLI based on the role of a specific user. + +The following table describes the UI panels and the permissions required to access them. + +| UI Panel | Permissions | Additional Information | +|:----------------|:----------------|:---------------------------| +| **Topics** | `topic.read` or `topic.describe` | - If the user also has `topic.create` permission, the **Topic Create** button is enabled.
- If the user has `topic.delete` permission, the **Topic Delete** button in the overflow menu is enabled. | +| **Topic Producer** | `topic.write` | | +| **Schema registry** | `schema.read` | If the user also has the `schema.alter` permission, then the **Add Schema** button is enabled. | +| **Metrics** | `cluster.alter` | | +| **Consumer groups** | `group.read` | | +| **Geo-replication** | `cluster.alter` | Geo-replication is enabled for SCRAM authentication. | +| **Generate credentials** | `cluster.alter` | **Generate credentials** buttons are enabled in the **Cluster connection** panel. | +| **Starter application** | `topic.read` and `topic.write` | When generating the starter application, the current user ID and password will be configured in the properties. `topic.create` permission is required to create new topics within the Starter App wizard. | + +The following tables describe the CLI commands and the permissions required to run them. + +| Topic CLI command | Permissions required | +|:----------------------|:---------------------| +| `kubectl es topics` | `topic.read` or `topic.describe` | +| `kubectl es topic` | `topic.read` and `topic.describeConfigs` | +| `kubectl es topic-create` | `topic.create` | +| `kubectl es topic-partitions-set`
`kubectl es topic-update` | `topic.alter` and `topic.alterConfigs` | +| `kubectl es topic-delete`
`kubectl es topic-delete-records` | `topic.delete` | + +| Consumer group CLI command | Permissions required | +|:---------------------------|:---------------------| +| `kubectl es groups` | `group.read` | +| `kubectl es group` | `group.read` and `topic.describe` | +| `kubectl es group-reset` | `group.read` and `topic.read` | +| `kubectl es group-delete` | `group.delete` | + +| Schema CLI command | Permissions required | +|:-------------------|:---------------------| +| `kubectl es schemas`
`kubectl es schema`
`kubectl es schema-export` | `schema.read` | +| `kubectl es schema-add`
`kubectl es schema-verify`
`kubectl es schema-modify` | `schema.read` and `schema.alter` | +| `kubectl es schema-remove`
`kubectl es schema-import` | `schema.alter` | + +| Geo-replication CLI command | Permissions required | +|:----------------------------|:---------------------| +| `kubectl es geo-cluster`
`kubectl es geo-cluster-add`
`kubectl es geo-cluster-connect`
`kubectl es geo-cluster-remove`
`kubectl es geo-clusters`
`kubectl es geo-replicator-create`
`kubectl es geo-replicator-delete`
`kubectl es geo-replicator-pause`
`kubectl es geo-replicator-restart`
`kubectl es geo-replicator-resume`
`kubectl es geo-replicator-topics-add`
`kubectl es geo-replicator-topics-remove` | `cluster.alter` | + +| Broker or cluster CLI command | Permissions required | +|:------------------------------|:---------------------| +| `kubectl es acls` | `cluster.describe` | +| `kubectl es broker` | `cluster.describe` and `cluster.describeConfigs` | +| `kubectl es broker-config`
`kubectl es cluster`
`kubectl es cluster-config` | `cluster.describeConfigs` | +| `kubectl es certificates`| No permissions required | + +{{site.data.reuse.es_cli_kafkauser_note}} +You can also use the {{site.data.reuse.es_name}} UI to generate the Kafka users. + +The following table describes the mapping of these permissions to the Kafka user ACL definitions. + +| Permissions | ACL Resource Type | ACL Operation | ACL Pattern Type | ACL Resource Name | +|:-------------------|-------------------|---------------|------------------|-------------------| +| `cluster.alter` | cluster | ALTER | N/A | N/A | +| `cluster.create` | cluster | CREATE | N/A | N/A | +| `cluster.describe` | cluster | DESCRIBE | N/A | N/A | +| `cluster.describeConfigs` | cluster | DESCRIBE_CONFIGS | N/A | N/A | +| `group.read` [^1] | group | READ | literal/prefix | '*', \ or \ | +| `group.delete` [^1] | group | DELETE | literal/prefix | '*', \ or \ | +| `schema.alter` [^2] | topic | ALTER | prefix | '\__schema\_' | +| `schema.read` [^2] | topic | READ | prefix | '\__schema\_' | +| `topic.create` [^1] | topic | CREATE | literal/prefix | '*', \ or \ | +| `topic.delete` [^1] | topic | DELETE | literal/prefix | '*', \ or \ | +| `topic.describe` [^1] | topic | DESCRIBE | literal/prefix | '*', \ or \ | +| `topic.read` [^1] | topic | READ | literal/prefix | '*', \ or \ | +| `topic.write` [^1] | topic | WRITE | literal/prefix | '*', \ or \ | +| `topic.describeConfigs` [^1] | topic | DESCRIBE_CONFIGS | literal/prefix | '*', \ or \ | + +[^1]: Topics and groups can be configured with an asterisk (*), meaning all topics and groups, a specific prefix name, or an exact name. These will influence which resources are listed within the panels, and they also determine whether certain actions are permitted, for example, whether a user can create a topic. + +[^2]: {{site.data.reuse.es_name}} Schema ACLs are topic resource ACLs, but with a specific prefix, `__schema_`. Schema permissions do not target a set of named schemas through prefixed names or a single schema through an explicitly defined name. Only define ACLs for schemas with the exact settings defined in the table earlier, permitting the operation against all schemas (otherwise, parts of the UI might not work as expected). + +For more information about ACLs, see the [authorization section](#authorization). + +### Managing access to the UI and CLI with IAM + +Access for groups and users is managed through IAM teams. If you have not previously created any teams, the administrative user credentials can be used to [set up a team](https://www.ibm.com/docs/en/cloud-paks/foundational-services/3.23?topic=users-managing-teams){:target="_blank"}. + +Access to the {{site.data.reuse.es_name}} UI and CLI requires an IAM user with a role of `Cluster Administrator`, `CloudPakAdministrator`, or `Administrator`. The role can be set for the user, or for the group the user is part of. + +The default cluster administrator (admin) IAM user credentials are stored in a secret within the `ibm-common-services` namespace. To retrieve the username and password: + +1. {{site.data.reuse.openshift_cli_login}} +2. Extract and decode the current {{site.data.reuse.icpfs}} admin username: + + ```shell + oc --namespace ibm-common-services get secret platform-auth-idp-credentials -o jsonpath='{.data.admin_username}' | base64 --decode + ``` + +3. Extract and decode the current {{site.data.reuse.icpfs}} admin password: + + ```shell + oc --namespace ibm-common-services get secret platform-auth-idp-credentials -o jsonpath='{.data.admin_password}' | base64 --decode + ``` + +**Note:** The password is auto-generated in accordance with the default password rules for {{site.data.reuse.icpfs}}. To change the username or password of the admin user, see the instructions about [changing the cluster administrator access credentials](https://www.ibm.com/docs/en/cloud-paks/foundational-services/3.23?topic=configurations-changing-cloud-pak-administrator-access-credentials#administrator-pwd){:target="_blank"}. + +The admin user has the `Cluster Administrator` role granting full access to all resources within the cluster, including {{site.data.reuse.es_name}}. + + +Any groups or users added to an IAM team with the `Cluster Administrator` role can log in to the {{site.data.reuse.es_name}} UI and CLI. Any groups or users with the `Administrator` role will not be able to log in until the namespace that contains the {{site.data.reuse.es_name}} cluster is [added as a resource](https://www.ibm.com/docs/en/cloud-paks/foundational-services/3.23?topic=teams-add-resources-team){:target="_blank"} for the IAM team. + +If Kafka authorization is enabled, operations by IAM users are automatically mapped to a Kafka principal with authorization for the required Kafka resources. + +**Note:** If you are using OAuth authorization in Kafka, ensure you add all admin user IDs to the `superUsers` property in the Kafka authorization configuration within the `EventStreams` custom resource, as described in [enabling OAuth authorization](../../installing/configuring/#enable-oauth-authorization). + +## Managing access to Kafka resources + +Each Kafka listener exposing an authenticated connection to {{site.data.reuse.es_name}} requires credentials to be presented when connecting. SCRAM-SHA-512 and Mutual TLS authentication credentials are created by using a `KafkaUser` custom resource, where the `spec.authentication.type` field has a value that matches the Kafka listener authentication type. + +OAuth authentication does not require any `KafkaUser` custom resources to be created. + +You can create a `KafkaUser` by using the {{site.data.reuse.es_name}} UI or CLI. It is also possible to create a `KafkaUser` by using the {{site.data.reuse.openshift_short}} UI or CLI, or the underlying Kubernetes API by applying a `KafkaUser` operand request. + +**Note:** You must have authenticated to the {{site.data.reuse.openshift_short}} UI with a Keycloak user or an IAM user to be able to generate Kafka users within the UI. To generate Kafka users within the UI as a SCRAM user, you must have an administrator role with the `cluster.alter` permission enabled. + +To assist in generating compatible `KafkaUser` credentials, the {{site.data.reuse.es_name}} UI indicates which authentication mechanism is being configured for each Kafka listener. + +**Warning:** Do not use or modify the internal {{site.data.reuse.es_name}} `KafkaUsers` named `-ibm-es-kafka-user`, `-ibm-es-georep-source-user`, `-ibm-es-iam-admin-kafka-user`, or `-ibm-es-ac-reg`. These are reserved to be used internally within the {{site.data.reuse.es_name}} instance. + +### Creating a `KafkaUser` in the {{site.data.reuse.es_name}} UI + +1. {{site.data.reuse.es_ui_login}} +2. Click the **Connect to this cluster** tile to view the **Cluster connection** panel. +3. Go to the **Kafka listener and Credentials** section. +4. To generate credentials for external clients, click **External**, or to generate credentials for internal clients, click **Internal**. +5. Click the **Generate SCRAM credential** or **Generate TLS credential** button next to the required listener to view the credential generation dialog. +6. Follow the instructions to generate credentials with desired permissions.\\ +**Note:** If your cluster does not have authorization enabled, the permission choices will not have any effect. + +The generated credential appears after the listener bootstrap address: +* For SCRAM credentials, two tabs are displayed: **Username and password** and **Basic Authentication**. The SCRAM username and password combination is used by Kafka applications, while the Basic Authentication credential is for use as an HTTP authorization header. +* For TLS credentials, a download button is displayed, providing a zip archive with the required certificates and keys. + +A `KafkaUser` will be created with the entered credential name. + +The cluster truststore is not part of the above credentials ZIP file archive. This certificate is required for all external connections, and is available to download from the **Cluster connection** panel under the **Certificates** heading within the {{site.data.reuse.openshift_short}} UI, or by running the command `kubectl es certificates`. Upon downloading the PKCS12 certificate, the certificate password will also be displayed. + +**Note:** If you have overridden the cluster CA certificates for any Kafka listener or REST endpoint to use an externally signed CA certificate, the download does not include the external certificate in the truststore unless you [provided the external CA certificate in a secret](../../installing/configuring/#providing-external-ca-certificates). + +Additionally, to retrieve endpoint details, view the `EventStreams` custom resource in the OpenShift web console as follows: + +1. {{site.data.reuse.openshift_ui_login}} +2. Expand **Operators** in the navigation on the left, and click **Installed Operators**. +3. Locate the operator that manages your {{site.data.reuse.es_name}} instance in the namespace. It is called **{{site.data.reuse.es_name}}** in the **NAME** column. +4. Click the **{{site.data.reuse.es_name}}** link in the row and click the **{{site.data.reuse.es_name}}** tab. This lists the **{{site.data.reuse.es_name}}** instances related to this operator. +5. Find your instance in the **Name** column and click the link for the instance. +6. Go to the menu of your {{site.data.reuse.es_name}} instance and select **Edit EventStreams**. +7. In the YAML view, scroll down to the `status.endpoints` section. +8. The admin API URL is the value in the `uri` property of the `admin` section. +9. The schema registry URL is the value in the `uri` property of the `apicurioregistry` section. + +![How to retrieve endpoints]({{ 'images/' | relative_url }}/endpoints.png "Screen capture showing where to retrieve the endpoints from."){:height="50%" width="50%"} + +### Creating a `KafkaUser` in the {{site.data.reuse.es_name}} CLI + +{{site.data.reuse.es_cli_kafkauser_note}} +You can also use the {{site.data.reuse.es_name}} UI to generate the Kafka users. + +1. {{site.data.reuse.cp_cli_login}} +2. {{site.data.reuse.es_cli_init_111}} +3. Use the `kafka-user-create` command to create a `KafkaUser` with the accompanying permissions.\\ +**Note:** If your cluster does not have authorization enabled, the permission choices will not have any effect. + +For example, to create SCRAM credentials with authorization to create topics and schemas, and also to produce to (including transactions) and consume from every topic: + +```shell +cloudctl es kafka-user-create \ + --name my-user \ + --consumer \ + --producer \ + --schema-topic-create \ + --all-topics \ + --all-groups \ + --all-txnids \ + --auth-type scram-sha-512 +``` + +For information about all options provided by the command, use the `--help` flag:\\ +`cloudctl es kafka-user-create --help` + +#### Creating a `KafkaUser` in the {{site.data.reuse.openshift_short}} UI + +**Note:** You must have authenticated to the {{site.data.reuse.openshift_short}} UI with a Keycloak user or an IAM user to be able to generate Kafka users within the UI. To generate Kafka users within the UI as a SCRAM user, you must have an administrator role with the `cluster.alter` permission enabled. + +Navigate to the **Event Streams** installed operator menu and select the **KafkaUser** tab. Click `Create KafkaUser`. The YAML view contains sample `KafkaUser` definitions to consume, produce, or modify every resource. + +### Creating a `KafkaUser` by using YAML + +You can create a Kafka user by creating a `KafkaUser` custom resource in a YAML file, and then running `kubectl apply -f .yaml`. + +For example, the following YAML creates SCRAM credentials for a Kafka user with authorization to create topics, groups, and schemas, and also to produce to and consume from every topic. It also provides administrator access to the {{site.data.reuse.es_name}} UI by generating a Kubernetes `Secret` containing the Base64-encoded SCRAM password for the `scram-sha-512` authentication type. You can use the password to [access the {{site.data.reuse.es_name}} UI](#managing-access-to-the-ui-and-cli-with-scram). + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: KafkaUser +metadata: + name: + namespace: + labels: + eventstreams.ibm.com/cluster: + backup.eventstreams.ibm.com/component: kafkauser +spec: + authentication: + type: scram-sha-512 + authorization: + type: simple + acls: + - resource: + type: cluster + operations: + - Alter + - resource: + type: topic + name: '*' + patternType: literal + operations: + - Create + - Delete + - Read + - Write + - resource: + type: topic + name: __schema_ + patternType: prefix + operations: + - Alter + - Read + - resource: + type: group + name: '*' + patternType: literal + operations: + - Read +``` + +### UI and CLI `KafkaUser` authorization + +`KafkaUsers` created by using the UI and CLI can only be configured with permissions for the most common operations against Kafka resources. You can later modify the created `KafkaUser` to [add additional ACL rules](#authorization). + +### Retrieving the credentials of a `KafkaUser` + +When a `KafkaUser` custom resource is created, the Entity Operator within {{site.data.reuse.es_name}} will create the principal in ZooKeeper with appropriate ACL entries. It will also create a Kubernetes `Secret` that contains the Base64-encoded SCRAM password for the `scram-sha-512` authentication type, or the Base64-encoded certificates and keys for the `tls` authentication type. + +You can retrieve the credentials by using the name of the `KafkaUser`. For example, you can retrieve the credentials as follows: + +1. Set the current namespace: + + ```shell + kubectl config set-context --current --namespace= + ``` + + +2. {{site.data.reuse.cncf_cli_login}} +3. Use the following command to retrieve the required `KafkaUser` data, adding the `KafkaUser` name and your chosen namespace: \\ + + ```shell + kubectl get kafkauser -o jsonpath='{"username: "}{.status.username}{"\nsecret-name: "}{.status.secret}{"\n"}' + ``` + + The command provides the following output: + - The principal username + - The name of the Kubernetes `Secret`, which includes the namespace, containing the SCRAM password or the TLS certificates. + +4. Decode the credentials. + + - For SCRAM, use the `secret-name` from the previous step to retrieve the password and decode it: + + ```shell + kubectl get secret -o jsonpath='{.data.password}' | base64 --decode + ``` + + - For TLS, get the credentials, decode them, and write each certificates and keys to files: + + ```shell + kubectl get secret -o jsonpath='{.data.ca\.crt}' | base64 --decode > ca.crt + ``` + + ```shell + kubectl get secret -o jsonpath='{.data.user\.crt}' | base64 --decode > user.crt + ``` + + ```shell + kubectl get secret -o jsonpath='{.data.user\.key}' | base64 --decode > user.key + ``` + + ```shell + kubectl get secret -o jsonpath='{.data.user\.p12}' | base64 --decode > user.p12 + ``` + + ```shell + kubectl get secret -o jsonpath='{.data.user\.password}' | base64 --decode + ``` + +5. Additionally, you can also retrieve endpoint details such as the admin API URL and the schema registry URL by using the following commands: + + - To retrieve the admin API URL from the `status.endpoints` list in the `EventStreams` custom resource, run the following command: + + ```shell + kubectl get eventstreams -n -o jsonpath="{.status.endpoints[?(@.name=='admin')].uri}" + ``` + + - To retrieve the schema registry URL from the `status.endpoints` list in the `EventStreams` custom resource, run the following command: + + ```shell + kubectl get eventstreams -n -o jsonpath="{.status.endpoints[?(@.name=='apicurioregistry')].uri}" + ``` + +The cluster truststore certificate is required for all external connections and is available to download from the **Cluster connection** panel under the **Certificates** heading, within the {{site.data.reuse.openshift_short}} UI, or by running the command `kubectl es certificates`. Upon downloading the PKCS12 certificate, the certificate password will also be displayed. + +Similarly, if you are using {{site.data.reuse.openshift_short}}, you can inspect these `KafkaUser` and `Secret` resources by using the web console. + +**Warning:** Do not use or modify the internal {{site.data.reuse.es_name}} `KafkaUsers` named `-ibm-es-kafka-user`, `-ibm-es-georep-source-user`, `-ibm-es-iam-admin-kafka-user`, or `-ibm-es-ac-reg`. These are reserved to be used internally within the {{site.data.reuse.es_name}} instance. + +## Authorization + +### What resource types can I secure? + +Within {{site.data.reuse.es_name}}, you can secure access to the following resource types, where the names in parentheses are the resource type names used in Access Control List (ACL) rules: +* Topics (`topic`): you can control the ability of users and applications to create, delete, read, and write to a topic. +* Consumer groups (`group`): you can control an application's ability to join a consumer group. +* Transactional IDs (`transactionalId`): you can control the ability to use the transaction capability in Kafka. +* Cluster (`cluster`): you can control operations that affect the whole cluster, including idempotent writes. + +**Note:** Schemas in the Apicurio Registry in {{site.data.reuse.es_name}} are a special case and are secured using the resource type of `topic` combined with a prefix of `__schema_`. You can control the ability of users and applications to create, delete, read, and update schemas. + +### What are the available Kafka operations? + +Access control in Apache Kafka is defined in terms of operations and resources. Operations are actions performed on a resource, and each operation maps to one or more APIs or requests. + +| Resource type | Operation | Kafka API | +|:----------------|:----------------|:---------------------------| +| topic | Alter | CreatePartitions | +| | AlterConfigs | AlterConfigs | +| | | IncrementalAlterConfigs | +| | Create | CreateTopics | +| | | Metadata | +| | Delete | DeleteRecords | +| | | DeleteTopics | +| | Describe | ListOffsets | +| | | Metadata | +| | | OffsetFetch | +| | | OffsetForLeaderEpoch | +| | DescribeConfigs | DescribeConfigs | +| | Read | Fetch | +| | | OffsetCommit | +| | | TxnOffsetCommit | +| | Write | AddPartitionsToTxn | +| | | Produce | +| | All | All topic APIs | +|   |   |   | +| group | Delete | DeleteGroups | +| | Describe | DescribeGroup | +| | | FindCoordinator | +| | | ListGroups | +| | Read | AddOffsetsToTxn | +| | | Heartbeat | +| | | JoinGroup | +| | | LeaveGroup | +| | | OffsetCommit | +| | | OffsetFetch | +| | | SyncGroup | +| | | TxnOffsetCommit | +| | All | All group APIs | +|   |   |   | +| transactionalId | Describe | FindCoordinator | +| | Write | AddOffsetsToTxn | +| | | AddPartitionsToTxn | +| | | EndTxn | +| | | InitProducerId | +| | | Produce | +| | | TxnOffsetCommit | +| | All | All txnid APIs | +|   |   |   | +| cluster | Alter | AlterReplicaLogDirs | +| | | CreateAcls | +| | | DeleteAcls | +| | AlterConfigs | AlterConfigs | +| | | IncrementalAlterConfigs | +| | ClusterAction | ControlledShutdown | +| | | ElectPreferredLeaders | +| | | Fetch | +| | | LeaderAndISR | +| | | OffsetForLeaderEpoch | +| | | StopReplica | +| | | UpdateMetadata | +| | | WriteTxnMarkers | +| | Create | CreateTopics | +| | | Metadata | +| | Describe | DescribeAcls | +| | | DescribeLogDirs | +| | | ListGroups | +| | | ListPartitionReassignments | +| | DescribeConfigs | DescribeConfigs | +| | IdempotentWrite | InitProducerId | +| | | Produce | +| | All | All cluster APIs | + +### Implicitly-derived operations + +Certain operations provide additional implicit operation access to applications. + +When granted `Read`, `Write`, or `Delete`, applications implicitly derive the `Describe` operation. +When granted `AlterConfigs`, applications implicitly derive the `DescribeConfigs` operation. + +For example, to `Produce` to a topic, the `Write` operation for the `topic` resource is required, which will implicitly derive the `Describe` operation required to get the topic metadata. + +### Access Control List (ACL) rules + +Access to resources is assigned to applications through an Access Control List (ACL), which consists of rules. An ACL rule includes an operation on a resource together with the additional fields listed in the following tables. A `KafkaUser` custom resource contains the binding of an ACL to a principal, which is an entity that can be authenticated by the {{site.data.reuse.es_name}} instance. + +An ACL rule adheres to the following schema: + +| Property | Type | Description | +|:-------------|:-------|:---------------------------------------------------------------------------------------------| +| `host` | string | The host from which the action described in the ACL rule is allowed. | +| `operations` | array | An array of strings that lists all the operations which will be allowed on the chosen resource. | +| `resource` | object | Indicates the resource for which the ACL rule applies. | + +The resource objects used in ACL rules adhere to the following schema: + +| Property | Type | Description | +|:--------------|:-------|:------------| +| `type` | string | Can be one of `cluster`, `group`, `topic` or `transactionalId`. | +| `name` | string | Identifies the value that the ACL rule will authenticate against when receiving incoming requests, rejecting anything that does not match. For example, the `topic` that can be accessed based on the topic name, or the `transactionalId` that can be used by a client. The `name` value can be used in combination with the `patternType` value to use the prefix pattern. | +| `patternType` | string | Describes the pattern used in the resource field. The supported types are `literal` and `prefix`. With literal pattern type, the resource field will be used as a definition of a full topic name. With prefix pattern type, the resource name will be used only as a prefix.
The default value is `literal`. | + +Using the information about schemas and resource-operations described in the previous tables, the `spec.authorization.acls` list for a `KafkaUser` can be created as follows: + +```yaml +# ... +spec: +# ... + authorization: + # ... + acls: + - host: '*' + resource: + type: topic + name: 'client-' + patternType: prefix + operations: + - Write +``` + +In this example, an application using this `KafkaUser` would be allowed to write to any topic beginning with **client-** (for example, **client-records** or **client-billing**) from any host machine. + +**Note:** The write operation also implicitly derives the required describe operation that Kafka clients require to understand the data model of a topic. + +The following is an example ACL rule that provides access to read Schemas: + +```yaml +- host: '*' + resource: + type: topic + name: '__schema_' + patternType: prefix + operations: + - Read +``` + +### Using OAuth + +Open Authorization (OAuth) is an open standard for authorization that allows client applications secure delegated access to specified resources. OAuth works over HTTPS and uses access tokens for authorization rather than credentials. + +Ensure you have enabled OAuth in your {{site.data.reuse.es_name}} [installation](../../installing/configuring/#enabling-oauth). + +You can [configure OAuth authentication](../../installing/configuring/#enable-oauth-authentication) by using either fast JWT token authentication or token introspection. + +**Note:** You cannot configure OAuth authentication with the {{site.data.reuse.es_name}} [REST producer API](../../connecting/rest-api/) or [Schema registry](../../schemas/setting-java-apps/#preparing-the-setup). Both endpoints can only be configured with SCRAM-SHA-512 or Mutual TLS authentication. + +You can also configure [OAuth authorization](../../installing/configuring/#enable-oauth-authorization). + +## Revoking access for an application + +As each application will be using credentials provided through a `KafkaUser` instance, deleting the instance will revoke all access for that application. Individual ACL rules can be deleted from a `KafkaUser` instance to remove the associated control on a resource operation. + + +#### Footnotes diff --git a/_es_11.5/04-security/02-encrypting-data.md b/_es_11.5/04-security/02-encrypting-data.md new file mode 100644 index 00000000..014ca7e9 --- /dev/null +++ b/_es_11.5/04-security/02-encrypting-data.md @@ -0,0 +1,18 @@ +--- +title: "Encrypting your data" +excerpt: "Encrypt your data to improve security." +categories: security +slug: encrypting-data +toc: true +--- + +The following encryption is always provided in {{site.data.reuse.es_name}}: + +- Network connections into the {{site.data.reuse.es_name}} deployment from external clients are secured using TLS. +- Kafka replication between brokers is also TLS encrypted. + +Consider the following for encryption as well: +- Internal Kafka listeners can be configured with or without encryption as described in [configuring access](../../installing/configuring/#configuring-access). +- The REST producer endpoint can be configured with or without encryption as described in [configuring access](../../installing/configuring/#configuring-access). + +In addition, you can supplement the existing data encryption with disk encryption where supported by your chosen storage provider. You can also encrypt messages within your applications before producing them. diff --git a/_es_11.5/04-security/03-secure-jmx-connections.md b/_es_11.5/04-security/03-secure-jmx-connections.md new file mode 100644 index 00000000..76f196b9 --- /dev/null +++ b/_es_11.5/04-security/03-secure-jmx-connections.md @@ -0,0 +1,119 @@ +--- +title: "Configuring secure JMX connections" +excerpt: "Connecting securely to Kafka JMX ports" +categories: security +slug: secure-jmx-connections +toc: true +--- + +## Java Management Extensions (JMX) + +Java Management Extensions (JMX) is a way of retrieving metrics from your specific processes dynamically at runtime. This can be used to get metrics that are specific to Java operations on Kafka + +To manage resources, management beans (MBeans) are used. MBeans represent a resource in the JVM. There are specific [MBean attributes](https://kafka.apache.org/37/documentation/#remote_jmx){:target="_blank"} you can use with Kafka. + +Metrics can be retrieved from applications running inside your Kubernetes cluster by connecting to an exposed JMX port. + +## Exposing a JMX port on Kafka + +You can expose the JMX port (`9999`) of each Kafka broker to be accessible to secure connections from within the Kubernetes cluster. This grants applications deployed inside the cluster read-only access to Kafka metrics. To expose the JMX port, set the `spec.strimziOverrides.kafka.jmxOptions` value to `{}`. This will create an open JMX port allowing any pod to read from it. + +The JMX port can be password-protected to prevent unauthorized pods from accessing it. It is good practice to secure the JMX port, as an unprotected port could allow a user to invoke an MBean operation on the Java JVM. To enable security for the JMX port, set the `spec.strimiziOverrrides.kafka.jmxOptions.authentication.type` field to `password`. For example: + +```yaml +#... +spec: + #... + strimziOverrides: + #... + kafka: + #... + jmxOptions: + authentication: + type: "password" + #... +``` + +This will cause the JMX port to be secured using a generated username and password. When the JMX port is password protected, a Kubernetes secret named `{{site.data.reuse.jmx-secret-name}}` is created inside the {{site.data.reuse.es_name}} namespace. The secret contains the following content: + +| Name | Description | +| -------------- | ---------------------------------------------------------- | +| `jmx_username` | The user that is authenticated to connect to the JMX port. | +| `jmx_password` | The password for the authenticated user. | + +## Connecting internal applications + +To connect your application to the Kafka JMX port, it must be deployed running inside the Kubernetes cluster. + +After your application is deployed, you can connect to each Kafka broker with the following URL pattern: +`-kafka-.-kafka-brokers.svc:9999` + +To connect to the JMX port, clients must use the following Java options: + +- `javax.net.ssl.trustStore=` +- `javax.net.ssl.trustStorePassword=` + +In addition, when initiating the JMX connection, if the port is secured then clients must provide the `username` and `password` from the JMX secret. For example, the `JConsole` UI provide a username/password box to enter the credentials. + +### Retrieving the truststore + +#### Using the {{site.data.reuse.es_name}} UI: + +1. {{site.data.reuse.es_ui_login}} +2. Click **Topics** in the primary navigation. +3. Click **Connect to this cluster**. +4. In the certificates section, click **download certificate**. + You will now have the required certificate and the password will be displayed in the UI. + +#### Using the {{site.data.reuse.es_name}} CLI: + +1. {{site.data.reuse.es_cli_init_111}} +2. Run the command `kubectl es certificates` to download the certificate. The password is displayed in the CLI. + +### Retrieving the JMX username and password + +#### Using the Kubernetes command-line tool (`kubectl`) + +1. {{site.data.reuse.cncf_cli_login}} +2. Run the following commands: + + ```shell + kubectl get secret {{site.data.reuse.jmx-secret-name}} -o jsonpath='{.data.jmx\-username}' -namespace | base64 -decode > jmx_username.txt + ``` + + ```shell + kubectl get secret {{site.data.reuse.jmx-secret-name}} -o jsonpath='{.data.jmx\-password}' -namespace | base64 -decode > jmx_password.txt + ``` + +These will output the `jmx_username` and `jmx_password` values into the respective `.txt` files. + +#### Mounting the JMX secret directly into a pod + +Mounting the secret will project the `jmx_username` and `jmx_password` values as files under the mount path folder. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: example-pod +spec: + volumes: + - name: jmx-secret + secret: + secretName: {{site.data.reuse.jmx-secret-name}} + containers: + - name: example-container + image: example-image + volumeMounts: + - name: jmx-secret + mountPath: /path/to/jmx-secret + readOnly: true +``` + +For more information, see [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-files-from-a-pod){:target="_blank"}. + +If the connecting application is not installed inside the {{site.data.reuse.es_name}} namespace, it must be copied to the application namespace using the following command: + +```shell +kubectl -n get secret {{site.data.reuse.jmx-secret-name}} -o yaml --export | kubectl -n apply -f - +``` diff --git a/_es_11.5/04-security/04-renewing-certificates.md b/_es_11.5/04-security/04-renewing-certificates.md new file mode 100644 index 00000000..822b9481 --- /dev/null +++ b/_es_11.5/04-security/04-renewing-certificates.md @@ -0,0 +1,86 @@ +--- +title: "Renewing certificates" +excerpt: "Learn about how to renew the certificates and keys in an existing Event Streams cluster." +categories: security +slug: renewing-certificates +toc: true +--- + +{{site.data.reuse.es_name}} uses 4 secrets to store the certificate authority (CA) certificates and keys for the cluster. + + Secret Name | Description | +--|-- +`\-cluster-ca` | The secret containing the cluster CA private key. This is used to generate the certificates presented at the Event Streams endpoints, and for internal Kafka to Zookeeper connections. | +`\-cluster-ca-cert` | The secret containing the cluster CA certificate. This is used to generate the certificates presented at the Event Streams endpoints, and for internal Kafka to Zookeeper connections. | +`\-clients-ca` | The secret containing the private key used to sign the Mutual TLS `KafkaUser` certificates. These are used by clients connecting to Kafka over a Mutual TLS authenticated listener. | +`\-clients-ca-cert` | The secret containing the CA certificate used to sign the Mutual TLS `KafkaUser` certificates. These are used by clients connecting to Kafka over a Mutual TLS authenticated listener. | + +## Renewing auto-generated self-signed CA certificates for existing installations + +By default, {{site.data.reuse.es_name}} uses self-signed CA certificates. These are automatically renewed when the default `renewalDays` (default is 60 days) and `validityDays` (default is 90 days) limits are met. + +To set exactly when certificates are renewed, you can configure the `renewalDays` and `validityDays` values under the `spec.strimziOverrides.clusterCa` and `spec.strimziOverrides.clientsCa` properties. Validity periods are expressed as a number of days after certificate generation. For more information, see [Certificate renewal and validity periods](https://strimzi.io/docs/operators/0.42.0/deploying.html#con-certificate-renewal-str){:target="_blank"}. + +You can define `maintenance windows` to ensure that the renewal of certificates are done at an appropriate time. For more information, see [Maintenance time windows for rolling update](https://strimzi.io/docs/operators/0.42.0/deploying.html#con-maintenance-time-window-definition-str){:target="_blank"}. + +You can also use the `strimzi.io/force-renew` annotation to manually renew the certificates. This can be necessary if you need to renew the certificates for security reasons outside of the defined renewal periods and maintenance windows. For more information, see [Manually renewing CA certificates](https://strimzi.io/docs/operators/0.42.0/deploying.html#proc-renewing-ca-certs-manually-str){:target="_blank"}. + +**Note:** The configuration settings for renewal periods and maintenance windows, and the annotation for manual renewal only apply to auto-generated self-signed certificates. If you provided your own CA certificates and keys, you must manually renew these certificates as described in the following sections. + +## Renewing your own CA certificates for existing installations + +If you provided your own CA certificates and keys, and need to renew only the CA certificate, complete the following steps. The steps provided demonstrate renewing the cluster CA certificate, but the steps are identical for renewing the clients CA certificate, with the exception of the secret name. + +1. Generate a new CA certificate by using the existing CA private key. The new certificate must have an identical CN name to the certificate it is replacing. Optionally, create a PKCS12 truststore with the new certificate if required. +2. Replace the value of the `ca.crt` in the `-cluster-ca-cert` secret with a base64-encoded string of the new certificate. Optionally, replace the `ca.p12` and `ca.password` values with the base64-encoded strings if required. +3. Increment the `eventstreams.ibm.com/ca-cert-generation` annotation value in the `-cluster-ca-cert` secret. If no annotation exists, add the annotation, and set the value to `1` with the following command: + + ```shell + kubectl annotate --namespace secret -cluster-ca-cert eventstreams.ibm.com/ca-cert-generation=1 + ``` + + When the operator reconciles the next time, the pods roll to process the certificate renewal. + +## Renewing your own CA certificates and private keys for existing installations + +If you provided your own CA Certificates and keys, and need to renew both the CA certificate and private key, complete the following steps. The steps provided demonstrate renewing the cluster CA certificate and key, but the steps are identical for renewing the clients CA certificate, with the exception of the secret name. + +1. Pause the operator's reconcile loop by running the following command: + + ```shell + kubectl annotate Kafka strimzi.io/pause-reconciliation="true" --overwrite=true + ``` + +2. Generate a new certificate and key pair. Optionally, create a PKCS12 truststore with the new certificate, if required. +3. Replace the value of the `ca.key` in the `-cluster-ca` secret with a base64-encoded string of the new key. +4. Increment the `eventstreams.ibm.com/ca-key-generation` annotation value in the `-cluster-ca` secret. If no annotation exists, add the annotation, setting the value to `1` with the following command: + + ```shell + kubectl annotate --namespace secret -cluster-ca eventstreams.ibm.com/ca-key-generation=1 + ``` + +5. Find the expiration date and time of the previous CA certificate by using OpenSSL or other certificate tooling. +6. Edit the `-cluster-ca-cert` secret. Rename the `ca.crt` element to be `ca-.crt`. + + This will take the format of `ca-YYYY-MM-DDTHH-MM-SSZ.crt` (for example, `ca-2022-05-24T10-20-30Z.crt`). Ensure the value of this element contains the base64-encoded string of the original CA certificate that you are renewing. + +7. Add element `ca.crt` to the `-cluster-ca-cert` secret with a base64-encoded string of the new certificate. Optionally, replace the `ca.p12` and `ca.password` values with the base64-encoded strings, if required. +8. Increment the `eventstreams.ibm.com/ca-cert-generation` annotation value in the `-cluster-ca-cert` secret. If no annotation exists, add the annotation, and set the value to `1` with the following command: + + ```shell + kubectl annotate --namespace secret -cluster-ca-cert eventstreams.ibm.com/ca-cert-generation=1 + ``` + +9. Resume the operator's reconcile loop by running the following command: + + ```shell + kubectl annotate Kafka strimzi.io/pause-reconciliation="false" --overwrite=true + ``` + + When the operator reconciles the next time, the pods roll to process the certificate renewal. The pods might roll multiple times during the renewal process. + +## Updating clients after certificate change + +If you renew the CA certificates, clients might need to update their truststore with the new certificate. + +To update your client settings after a certificate change, see how to [secure the connection](../../getting-started/connecting/#securing-the-connection). diff --git a/_es_11.5/04-security/05-verifying-signature.md b/_es_11.5/04-security/05-verifying-signature.md new file mode 100644 index 00000000..8ae5286e --- /dev/null +++ b/_es_11.5/04-security/05-verifying-signature.md @@ -0,0 +1,236 @@ +--- +title: "Verifying container image signatures" +excerpt: "Verify your CISO image signatures." +categories: security +slug: verifying-signature +toc: true +--- + +Digital signatures provide a way for consumers of content to ensure that what they download is both authentic (it originated from the expected source) and has integrity (it is what we expect it to be). All images for the {{site.data.reuse.es_name}} certified container in the IBM Entitled Registry are signed following the approach from Red Hat. + +You can use the signature to verify that the images came from IBM when they are pulled onto the system. + +## Before you begin + +- Ensure that the following command-line tools are installed on your computer. On Linux systems, these images can typically be installed by using the package manager. + + - [The OpenShift Container Platform CLI](https://docs.openshift.com/container-platform/4.16/cli_reference/openshift_cli/getting-started-cli.html){:target="_blank"} + - [The IBM Catalog Management Plug-in for IBM Cloud Paks (ibm-pak)](https://github.com/IBM/ibm-pak-plugin/releases/latest){:target="_blank"} + - [GNU Privacy Guard (GnuPG) version 2](https://gnupg.org/){:target="_blank"} + - [Skopeo](https://github.com/containers/skopeo){:target="_blank"} + +- On the computer where the command-line tools are installed, copy the following version-specific text block exactly as shown into a text editor, and save it in a file named `acecc-public.gpg`. The following text block represents the {{site.data.reuse.es_name}}-certified container public key in the GNU Privacy Guard format. + + ```console + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBGP9DMgBEADCSPlk3GN4qbs+kHFuYnKR3d25Tpv0w1FR04krE4eJZleGzv9V + ipZ21ywcKxE9Y9KOseKt4/QT+vsJbmDdrhZQNCqJzxGkL27lS/Dfqc6GfSas3qhY + Zghmd6+S9Wo4G9oIAAE7/wZcchltoEyYd4/VihuU5uKmcRZ8U/k7+lCMQV2qt0+l + PgCtyQ6dlbqtwm4gjNeHu6rz69Hk4PA9EN1h2J0yHtiTDYL4wV0tjbh7t6s/WYHW + KkLbDbXybWdTD7CitcZ3mneXI+ij5N0iB63HIrmVQKaP/bWsuWwaijGGZM+LjUf6 + MRBTjfiqLKrmlnaL2xwo200Z+vHV705et81aOefOLxCE2bqnXnhyMup5NE+U4BVx + 57aTVW4sL03OpSF0Lj+Jms2JjoNtG84qLv2w7XKozhc+yunv7wghnuaubqiAc3tO + 3pHuP92r8IVMcHkFsbGZAJxi891nYthIQAQwUNkvfgVC83y3BdPrxlwUu9qD9nhl + q56n7c571keXlTQ1fT/Km4o2yhG7JfkpJvvChTVTYx8g+ST4dEgxY3A8tmxTo0ss + pHBp1f3ry1QAyp72yjobar/xZOB1+O2YBmGHhM1cm4pZNdd7DcxJNYqgP+Y/o83l + UATX/QUInD1Jjz1aPiCplwd7J3rctEE+TZ1gSLwhLFBKrynOte+GVERO0wARAQAB + tCZRdWlubmlwaWFjIFByb2R1Y3RzIDxwc2lydEB1cy5pYm0uY29tPokCOgQTAQgA + JAUCY/0MyAIbDwULCQgHAgYVCgkICwIEFgIDAQIeAQUJAAAAAAAKCRDSz7jRu6g0 + HsY8EAC4nvJW5QPxhSY5oaUtA5bx5KLt4LQ8ib316C0lhZV+YilvkOkcXsfvLdon + Y61xrp/WBTjzmAm6X7EFypJI33Olo+pBtBq7lOOuwOH8Y/n/BM9qSSzL9EQmqO8r + rZ3Qk7jsSVLT18NhEX76dC72MnXJwK6p0A+4Gc7UiG1sSOZgi5pqrLWaWlABWVp8 + fVwWVRy7rcQMulJOxD6FRrt/HsIZm0hPQrxeXMLn9pKzKm3DVwSUD91xLjjh+/cQ + gRpIaV1AYvV8+74d29cBAaNMZigfrbSROfxWtgBpIKuBxCvEdSjzJe6G34Usdg0Q + Owcm+FHsyk/QTAClSueCz/fpUirYQM2SKweeelhYNQr0gII1P8LF0RpmKJ8I68dn + sZZI8JlIs0stz+kU4FZLPF+vMwl9Fq6/EJ0tNQcleWiDpH8Gz7hwCbpoLBnWyR0j + NDZ9xZ8KxUtfdchptja93t0hJpcTs97HyRtgKydguWT3wbvH31U2J+hR45/PQgYE + UV8yeGx9p2WNi2G4FbPw7f/781RZgZ4ZesTBzQrdZwfyz/brBBv2jWfg6bHmTGh3 + QuRoFeqg9GW3p2QGFIIazDZJ+XHAk7NKNH5m3ao9U/x0g/fth4iUKxIGp4c7A54L + fROemzTVxd8jHEGGUlgZ6Bi0NdSGOF1htWBDo88XnewLiKNpHw== + =jPze + -----END PGP PUBLIC KEY BLOCK----- + ``` + + +## Obtaining the container images + +Obtain the list of {{site.data.reuse.es_name}}-certified container images to verify as described in the following sections. + +### Prepare your bastion host + +Ensure you meet the following prerequisites before downloading the CASE archive and obtaining the images: + +- A computer with internet access on which you can run the required commands. This computer must also have access to the cluster, and is referred to as a **bastion host**. +- A cluster that is already set up and running a supported version of the {{site.data.reuse.openshift}}. For more information, see the [support matrix]({{ 'support/matrix/#event-streams' | relative_url }}) for supported versions. +- A private Docker registry that can be accessed by the cluster and the bastion host, and which will be used to store all images on your restricted network. + +If the cluster has a bastion host which has access to the public internet, then the following steps can be performed from the bastion host. + +**Note**: In the absence of a bastion host, prepare a portable device that has access to the public internet to download the CASE archive and images, and also has access to the target registry where the images will be mirrored. + +### Download the CASE archive + +Download the Container Application Software for Enterprises (CASE) archive. This archive, which is typically provided for installing within a restricted network, includes metadata and files that you will require later. + +Complete the following steps to download the CASE archive: + +1. {{site.data.reuse.openshift_cli_login}} +2. Configure the internal repository for downloading the CASE archive: + + ```shell + oc ibm-pak config repo 'default' -r "https://github.com/IBM/cloud-pak/raw/master/repo/case/" --enable + ``` + +3. Run the following command to download, validate, and extract the CASE archive. + + ```shell + oc ibm-pak get ibm-eventstreams + ``` + + Where `` is the location of the CASE archive. If you are running the command from the current location, set the path to the current directory (`.`). + The following output is displayed: + + ```shell + Downloading and extracting the CASE ... + - Success + Retrieving CASE version ... + - Success + Validating the CASE ... + Validating the signature for the ibm-eventstreams CASE... + - Success + Creating inventory ... + - Success + Finding inventory items + - Success + Resolving inventory items ... + Parsing inventory items + - Success + Download of CASE: ibm-eventstreams, version: 3.5.0 is complete + ``` + +4. Verify that the CASE archive and images `.csv` files have been generated for the {{site.data.reuse.es_name}}. For example, ensure you have the following files generated for the {{site.data.reuse.es_name}} CASE. + + ```shell + $ tree ~/.ibm-pak + + ├── config + │   └── config.yaml + ├── data + │   ├── cases + │   │   └── ibm-eventstreams + │   │   └── 3.5.0 + │   │   ├── caseDependencyMapping.csv + │   │   ├── charts + │   │   ├── ibm-eventstreams-3.5.0-airgap-metadata.yaml + │   │   ├── ibm-eventstreams-3.5.0-charts.csv + │   │   ├── ibm-eventstreams-3.5.0-images.csv + │   │   ├── ibm-eventstreams-3.5.0.tgz + │   │   └── resourceIndexes + │   │   └── ibm-eventstreams-resourcesIndex.yaml + │   └── mirror + └── logs + └── oc-ibm_pak.log + + 9 directories, 8 files + ``` + +### Obtain the files + +1. After meeting the required prerequisites and downloading the CASE archive, obtain the following files: + + - The downloaded CASE archives, which contain metadata for the container images required to deploy each {{site.data.reuse.es_name}} capability. Each CASE archive also contains the required scripts to mirror images to a private registry, and to configure the target cluster to use the private registry as a mirror. + - Generated comma-separated value (CSV) files listing the images. Obtain an IBM Entitled Registry entitlement key from the [IBM Container software library](https://myibm.ibm.com/products-services/containerlibrary){:target="_blank"}. The CSV files, combined with your entitlement key, are used for downloading or mirroring the images manually. + + To verify the image signatures for a {{site.data.reuse.es_name}}-certified container, use the file that is named in the format `ibm-eventstreams--images.csv`, where `v.r.m` represents the {{site.data.reuse.es_name}} CASE version. + +2. Use a shell script to parse through the CSV file and print out the list of "manifest list images" with their digests or tags. You can use the listed names when pulling and verifying image signatures. In the `tail` command, `/tmp/cases` represents the directory where you downloaded the CASE archive. + + - List images by digest: + + ```shell + tail -q -n +2 /tmp/cases/ibm-eventstreams-*-images.csv | while IFS="," read registry image_name tag digest mtype os arch variant insecure digest_source image_type groups; do + if [[ "$mtype" == "LIST" ]]; then + echo "$registry/$image_name@$digest" + fi + done + ``` + + - List images by tag: + + ```shell + tail -q -n +2 /tmp/cases/ibm-eventstreams-*-images.csv | while IFS="," read registry image_name tag digest mtype os arch variant insecure digest_source image_type groups; do + if [[ "$mtype" == "LIST" ]]; then + echo "$registry/$image_name:$tag" + fi + done + ``` + + **Note**: You can also copy the output to a file for ease of reference while verifying the image signatures. + +## Verifying the signature + +To verify the image signatures, complete the following steps: + +1. Import the {{site.data.reuse.es_name}}-certified container public key on the computer where you saved the public key to a file as described in the [Before you begin](#before-you-begin) section. + + + ```shell + sudo gpg --import acecc-public.gpg + ``` + + + **Note**: This step needs to be done only once on each computer that you use for signature verification. + +2. Calculate the fingerprint. + + + ```shell + fingerprint=$(sudo gpg --fingerprint --with-colons | grep fpr | tr -d 'fpr:') + ``` + + + This command stores the key's fingerprint in an environment variable called `fingerprint`, which is needed for the command to verify the signature. + + **Note:** When you exit your shell session, the variable will be deleted. The next time you log in to your computer, you can set the environment variable again by rerunning the command in this step. + +3. Log in to `skopeo` to access the entitled registry. Use `cp` as the username and your entitlement key as the password. +For example: + + ```shell + skopeo login cp.icr.io --username cp --password myEntitlementKey + ``` + +4. Create a directory (for example, `images`) for the image. Then use `skopeo` to pull the image into local storage, where `imageName` represents the image name. + + ```shell + mkdir images + skopeo copy docker:// dir:./images + ``` + + For example: + + ```shell + mkdir images + skopeo copy docker://icr.io/cpopen/ibm-eventstreams-catalog:3.0.0-00000000-000000 dir:./images + ``` + + This command downloads the `image` as a set of files and places them in the `images` directory, or in a directory that you specified. A manifest file named `images/manifest.json`, and a set of signature files named `images/signature-1`, `images/signature-2`, and `images/signature-3` are added to the directory. You will use these files to verify the signature in the next step. + +5. Verify the signature for each required image, where `imageName` is the name of the image and `signature-N` relates to a format for the name. + + ```shell + sudo skopeo standalone-verify ./images/manifest.json ${fingerprint} ./images/ + ``` + + For example: + + ```shell + sudo skopeo standalone-verify ./images/manifest.json icr.io/cpopen/ibm-eventstreams-catalog:3.0.0-00000000-000000 ${fingerprint} ./images/signature-1 + ``` + + You will receive a confirmation similar to the following: + + ```shell + Signature verified, digest sha256:0000000000000000000000000000000000000000000000000000000000000000 + ``` diff --git a/_es_11.5/04-security/06-network-policies.md b/_es_11.5/04-security/06-network-policies.md new file mode 100644 index 00000000..06023582 --- /dev/null +++ b/_es_11.5/04-security/06-network-policies.md @@ -0,0 +1,185 @@ +--- +title: "Network policies" +excerpt: "Learn about the network connections allowed for Event Streams each pod." +categories: security +slug: network-policies +toc: true +--- + +## Inbound network connections (ingress) + +[Network policies](https://kubernetes.io/docs/concepts/services-networking/network-policies/){:target="_blank"} are used to control inbound connections into pods. These connections can be from pods within the cluster, or from external sources. + +When you install an instance of {{site.data.reuse.es_name}}, the required network policies will be automatically created. To review the network policies that have been applied: + +1. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to display the installed network policies for a specific namespace: + + ```shell + kubectl get netpol -n + ``` + +The following tables provide information about the network policies that are applicable to each pod within the {{site.data.reuse.es_name}} instance. If a particular pod is not required by a given {{site.data.reuse.es_name}} configuration, the associated network policy will not be applied. + +**Note:** Where a network policy exposes a port to the {{site.data.reuse.es_name}} Cluster operator, it is configured to allow connections from any namespace. + +### Kafka pod + +| **Type** | **Origin** | **Port** | **Reason** | **Enabled in policy** | +|----------|----------------------------------------------------------------------------------------------|----------|------------------------------------|--------------------------------------------------------------------------------------------------------| +| TCP | REST API, REST Producer and Schema Registry pods | 8091 | Broker communication | Always | +| TCP | Kafka, Cluster operator, Entity operator, Kafka Cruise Control and Kafka Exporter pods | 9091 | Broker communication | Always | +| TCP | Anywhere (can be restricted by including `networkPolicyPeers` in the listener configuration) | 9092 | Kafka access for Plain listener | If the [Plain listener](../../installing/configuring/#kafka-access) is enabled | +| TCP | Anywhere (can be restricted by including `networkPolicyPeers` in the listener configuration) | 9093 | Kafka access for TLS listener | If the [TLS listener](../../installing/configuring/#kafka-access) is enabled | +| TCP | Anywhere (can be restricted by including `networkPolicyPeers` in the listener configuration) | 9094 | Kafka access for External listener | If the [External listener](../../installing/configuring/#kafka-access) is enabled | +| TCP | Anywhere | 9404 | Prometheus access to Kafka metrics | If [metrics](../../installing/configuring/#configuring-external-monitoring-through-prometheus) are enabled | +| TCP | Anywhere | 9999 | JMX access to Kafka metrics | If [JMX port is exposed](../secure-jmx-connections#exposing-a-jmx-port-on-kafka) | + +**Note:** If required, access to listener ports can be restricted to only those pods with specific labels by including additional configuration in the {{site.data.reuse.es_name}} custom resource under `spec.strimziOverrides.kafka.listeners..networkPolicyPeers`. + +### ZooKeeper pod + +| **Type** | **Origin** | **Port** | **Reason** | **Enabled in policy** | +|----------|--------------------------------------------------------------------------------|----------|-----------------------------------------|--------------------------------------------------------------------------------------------------------| +| TCP | Kafka, ZooKeeper, Cluster operator, Entity operator, Kafka Cruise Control pods | 2181 | ZooKeeper client connections | Always | +| TCP | Other ZooKeeper pods | 2888 | ZooKeeper follower connection to leader | Always | +| TCP | Other ZooKeeper pods | 3888 | ZooKeeper leader election | Always | +| TCP | Anywhere | 9404 | Exported Prometheus metrics | If [metrics](../../installing/configuring/#configuring-external-monitoring-through-prometheus) are enabled | + +### Geo-replicator pod + +| **Type** | **Origin** | **Port** | **Reason** | **Enabled in policy** | +|----------|-------------------------------------------------------------|----------|--------------------------------|--------------------------------------------------------------------------------------------------------| +| TCP | REST API pods | 8083 | Geo-replicator API traffic | Always | +| TCP | Cluster operator and other geo-replicator pods | 8083 | Geo-replicator cluster traffic | Always | +| TCP | Anywhere | 9404 | Exported Prometheus metrics | If [metrics](../../installing/configuring/#configuring-external-monitoring-through-prometheus) are enabled | + +### Schema registry pod + +| **Type** | **Origin** | **Port** | **Reason** | **Enabled in policy** | +|----------|----------------------------------------------------|----------|-------------------------|--------------------------------------------------------------------------------------------------| +| TCP | Anywhere | 9443 | External access to API | Always | +| TCP | Any pod in {{site.data.reuse.es_name}} instance | 7443 | TLS cluster traffic | If [internal TLS](../../installing/configuring/#configuring-encryption-between-pods) is enabled | +| TCP | Any pod in {{site.data.reuse.es_name}} instance | 7080 | Non-TLS cluster traffic | If [internal TLS](../../installing/configuring/#configuring-encryption-between-pods) is disabled | + +### Administration UI pod + +| **Type** | **Origin** | **Port** | **Reason** | **Enabled in policy** | +|----------|----------------------------------------|----------|--------------------------|-----------------------| +| TCP | Anywhere | 3000 | External access to UI | Always | + +### Administration server pod + +| **Type** | **Origin** | **Port** | **Reason** | **Enabled in policy** | +|----------|----------------------------------------------------|----------|-------------------------|--------------------------------------------------------------------------------------------------| +| TCP | Anywhere | 9443 | External access to API | Always | +| TCP | Any pod in {{site.data.reuse.es_name}} instance | 7443 | TLS cluster traffic | If [internal TLS](../../installing/configuring/#configuring-encryption-between-pods) is enabled | +| TCP | Any pod in {{site.data.reuse.es_name}} instance | 7080 | Non-TLS cluster traffic | If [internal TLS](../../installing/configuring/#configuring-encryption-between-pods) is disabled | + +### REST producer server pod + +| **Type** | **Origin** | **Port** | **Reason** | **Enabled in policy** | +|----------|----------------------------------------------------|----------|-------------------------|--------------------------------------------------------------------------------------------------| +| TCP | Anywhere | 9443 | External access to API | Always | +| TCP | Any pod in {{site.data.reuse.es_name}} instance | 7443 | TLS cluster traffic | If [internal TLS](../../installing/configuring/#configuring-encryption-between-pods) is enabled | +| TCP | Any pod in {{site.data.reuse.es_name}} instance | 7080 | Non-TLS cluster traffic | If [internal TLS](../../installing/configuring/#configuring-encryption-between-pods) is disabled | + +### Metrics collector pod + +| **Type** | **Origin** | **Port** | **Reason** | **Enabled in policy** | +|----------|----------------------------------------------------|----------|---------------------------------|--------------------------------------------------------------------------------------------------| +| TCP | Anywhere | 7888 | Exported Prometheus metrics | Always | +| TCP | Any pod in {{site.data.reuse.es_name}} instance | 7443 | TLS inbound metrics traffic | If [internal TLS](../../installing/configuring/#configuring-encryption-between-pods) is enabled | +| TCP | Any pod in {{site.data.reuse.es_name}} instance | 7080 | Non-TLS inbound metrics traffic | If [internal TLS](../../installing/configuring/#configuring-encryption-between-pods) is disabled | + +### Cluster operator pod + +| **Type** | **Origin** | **Port** | **Reason** | **Enabled in policy** | +|----------|------------|----------|----------------------------------------|------------------------| +| TCP | Anywhere | 8080 | Exported Prometheus metrics | Always | +| TCP | Anywhere | 8081 | EventStreams custom resource validator | Always | + +### Cruise Control pod + +| **Type** | **Origin** | **Port** | **Reason** | **Enabled in policy** | +|----------|------------------|----------|---------------|-----------------------| +| TCP | Cluster operator | 9090 | Access to API | Always | + +### Kafka Connect pod + +| **Type** | **Origin** | **Port** | **Reason** | **Enabled in policy** | +|----------|-------------------------------------------------------------|----------|----------------------------------|--------------------------------------------------------------------------------------------------------| +| TCP | Cluster operator and Kafka Connect pods | 8083 | Access to Kafka Connect REST API | Always | +| TCP | Anywhere | 9404 | Exported Prometheus metrics | If [metrics](../../installing/configuring/#configuring-external-monitoring-through-prometheus) are enabled | + + +## Outbound network connections (egress) + +The following tables provide information about the outbound network connections (egress) initiated by pods in an {{site.data.reuse.es_name}} installation. If a particular pod is not required by an {{site.data.reuse.es_name}} configuration, the associated outbound connection is not applicable. + +### Kafka pod + +| **Type** | **Destination** | **Pod Label** | **Port** | **Reason** | +|----------|-------------------|---------------------------------------------|----------|----------------------------------| +| TCP | Kafka | app.kubernetes.io/name=kafka | 9091 | Broker replication | +| TCP | ZooKeeper | app.kubernetes.io/name=zookeeper | 2181 | ZooKeeper communication | + +### ZooKeeper pod + +| **Type** | **Destination** | **Pod Label** | **Port** | **Reason** | +|----------|-------------------|---------------------------------------------|----------|----------------------------------| +| TCP | ZooKeeper | app.kubernetes.io/name=zookeeper | 2888 | ZooKeeper clustering | +| TCP | ZooKeeper | app.kubernetes.io/name=zookeeper | 3888 | ZooKeeper leader elections | + +### Geo-replicator pod + +| **Type** | **Destination** | **Pod Label** | **Port** | **Reason** | +|----------|-------------------|---------------------------------------------|----------|----------------------------------| +| TCP | Geo-replicator | app.kubernetes.io/name=kafka-mirror-maker-2 | 8083 | Geo-replicator cluster traffic | + +**Note:** Geo-replicator destination is external to the cluster. + +### Schema registry pod + +| **Type** | **Destination** | **Pod Label** | **Port** | **Reason** | +|----------|-------------------|---------------------------------------------|----------|----------------------------------| +| TCP | Kafka | app.kubernetes.io/name=kafka | 8091 | Broker communication | + +### Administration server pod + +| **Type** | **Destination** | **Pod Label** | **Port** | **Reason** | +|----------|-------------------|---------------------------------------------|----------|----------------------------------| +| TCP | Kafka | app.kubernetes.io/name=kafka | 8091 | Broker API traffic | +| TCP | Geo-replicator | app.kubernetes.io/name=kafka-mirror-maker-2 | 8083 | Geo-replicator API traffic | +| TCP | Kafka Connect | app.kubernetes.io/name=kafka-connect | 8083 | Kafka Connect API traffic | + +### REST producer server pod + +| **Type** | **Destination** | **Pod Label** | **Port** | **Reason** | +|----------|-------------------|---------------------------------------------|----------|----------------------------------| +| TCP | Kafka | app.kubernetes.io/name=kafka | 8091 | Producer messages | + +### Cluster operator pod + +| **Type** | **Destination** | **Pod Label** | **Port** | **Reason** | +|----------|-------------------|---------------------------------------------|----------|----------------------------------| +| TCP | Kafka | app.kubernetes.io/name=kafka | 9091 | Configuration | +| TCP | ZooKeeper | app.kubernetes.io/name=zookeeper | 2181 | Configuration | +| TCP | Geo-replicator | app.kubernetes.io/name=kafka-mirror-maker-2 | 8083 | Configuration | +| TCP | Kafka Connect | app.kubernetes.io/name=kafka-connect | 8083 | Configuration | +| TCP | Admin API | app.kubernetes.io/name=admin-api | 7443 | Configuration | +| TCP | Metrics collector | app.kubernetes.io/name=metrics | 7443 | Configuration | +| TCP | Rest producer | app.kubernetes.io/name=rest-producer | 7443 | Configuration | + +### Cruise Control pod + +| **Type** | **Destination** | **Pod Label** | **Port** | **Reason** | +|----------|------------------|----------------------------------------------|----------|----------------------------------| +| TCP | Kafka | app.kubernetes.io/name=kafka | 9091 | Broker communication | +| TCP | ZooKeeper | app.kubernetes.io/name=zookeeper | 2181 | ZooKeeper client connections | + +### Kafka Connect pod + +| **Type** | **Destination** | **Pod Label** | **Port** | **Reason** | +|----------|------------------|----------------------------------------------|----------|----------------------------------| +| TCP | Kafka Connect | app.kubernetes.io/name=kafka-connect | 8083 | Access to Kafka Connect REST API | diff --git a/_es_11.5/04-security/07-gdpr-considerations.md b/_es_11.5/04-security/07-gdpr-considerations.md new file mode 100644 index 00000000..380d4456 --- /dev/null +++ b/_es_11.5/04-security/07-gdpr-considerations.md @@ -0,0 +1,139 @@ +--- +title: "Considerations for GDPR" +excerpt: "Considerations for GDPR." +categories: security +slug: gdpr-considerations +toc: true +--- + +## Notice: + +Clients are responsible for ensuring their own compliance with various laws +and regulations, including the European Union General Data Protection Regulation. +Clients are solely responsible for obtaining advice of competent legal counsel as to +the identification and interpretation of any relevant laws and regulations that may +affect the clients’ business and any actions the clients may need to take to comply +with such laws and regulations. + +The products, services, and other capabilities +described herein are not suitable for all client situations and may have restricted +availability. IBM does not provide legal, accounting, or auditing advice or represent or +warrant that its services or products will ensure that clients are in compliance with +any law or regulation. + +## GDPR Overview + +### What is GDPR? + +_GDPR_ stands for General Data Protection Regulation. + +GDPR has been adopted by the European Union and will apply from May 25, 2018. + +### Why is GDPR important? + +GDPR establishes a stronger data protection regulatory framework for processing of personal data of individuals. GDPR brings: + +- New and enhanced rights for individuals +- Widened definition of personal data +- New obligations for companies and organisations handling personal data +- Potential for significant financial penalties for non-compliance +- Compulsory data breach notification + +This document is intended to help you in your preparations for GDPR readiness. + +### Read more about GDPR + +- [EU GDPR website](https://gdpr.eu/){:target="_blank"} +- [IBM GDPR website](https://www.ibm.com/data-responsibility/gdpr/){:target="_blank"} + +## Product Configuration for GDPR + +#### Configuration to support data handling requirements + +The GDPR legislation requires that personal data is strictly controlled and that the +integrity of the data is maintained. This requires the data to be secured against loss +through system failure and also through unauthorized access or via theft of computer equipment or storage media. +The exact requirements will depend on the nature of the information that will be stored or transmitted by {{site.data.reuse.es_name}}. +Areas for consideration to address these aspects of the GDPR legislation include: + +- Physical access to the assets where the product is installed +- [Encryption of data](../encrypting-data) both at rest and in flight +- [Managing access](../managing-access) to topics which hold sensitive material. + +## Data Life Cycle + +{{site.data.reuse.es_name}} is a general purpose pub-sub technology built on [Apache Kafka®](https://kafka.apache.org/){:target="_blank"} which can +be used for the purpose of connecting applications. Some of these applications may be IBM-owned but others may be third-party products +provided by other technology suppliers. As a result, {{site.data.reuse.es_name}} can be used to exchange many forms of data, +some of which could potentially be subject to GDPR. + +### What types of data flow through {{site.data.reuse.es_name}}? + +There is no one definitive answer to this question because use cases vary through application deployment. + +### Where is data stored? + +As messages flow through the system, message data is stored on physical storage media configured by the deployment. It may also reside in logs collected +by pods within the deployment. This information may include data governed by GDPR. + +### Personal data used for online contact with IBM + +{{site.data.reuse.es_name}} clients can submit online comments/feedback requests to contact IBM about {{site.data.reuse.es_name}} in a variety of +ways, primarily: + +- Public issue reporting and feature suggestions via {{site.data.reuse.es_name}} Git Hub portal +- Private issue reporting via IBM Support + +Typically, only the client name and email address are used to enable personal replies for the subject of the contact. The use of personal data conforms to the [IBM Online Privacy Statement](https://www.ibm.com/privacy/us/en/){:target="_blank"}. + +## Data Collection + +{{site.data.reuse.es_name}} can be used to collect personal data. When assessing your use of {{site.data.reuse.es_name}} and the demands +of GDPR, you should consider the types of personal data which in your circumstances are passing through the system. You +may wish to consider aspects such as: + +- How is data being passed to an {{site.data.reuse.es_name}} topic? Has it been encrypted or digitally signed beforehand? +- What type of storage has been configured within the {{site.data.reuse.es_name}}? Has [encryption](../encrypting-data) been enabled? +- How does data flow between nodes in the {{site.data.reuse.es_name}} deployment? Has internal network traffic been [encrypted](../encrypting-data)? + +## Data Storage + +When messages are published to topics, {{site.data.reuse.es_name}} will store the message data on stateful media within the cluster for +one or more nodes within the deployment. Consideration should be given to [securing this data](../encrypting-data) when at rest. + +The following items highlight areas where {{site.data.reuse.es_name}} may indirectly persist application provided data which +users may also wish to consider when ensuring compliance with GDPR. + +- Kubernetes activity logs for containers running within the Pods that make up the {{site.data.reuse.es_name}} deployment +- Logs captured on the local file system for the Kafka container running in the Kakfa pod for each node + +By default, messages published to topics are retained for a week after their initial receipt, but this can be configured by modifying [Kafka broker settings](https://kafka.apache.org/37/documentation/#brokerconfigs){:target="_blank"}. These settings are [configured](../../installing/configuring/#applying-kafka-broker-configuration-settings) using the `EventStreams` custom resource. + +## Data Access + +The Kafka core APIs can be used to access message data within the {{site.data.reuse.es_name}} system: + +- [Producer](http://kafka.apache.org/37/documentation/#producerapi){:target="_blank"} API to allow data to be sent to a topic +- [Consumer](http://kafka.apache.org/37/documentation/#consumerapi){:target="_blank"} API to allow data to be read from a topic +- [Streams](http://kafka.apache.org/37/documentation/#streamsapi){:target="_blank"} API to allow transformation of data from an input topic to an output topic +- [Connect](http://kafka.apache.org/37/documentation/#connectapi){:target="_blank"} API to allow connectors to continually move data in or out of a topic from an external system + +For more information about controlling access to data stored in {{site.data.reuse.es_name}}, see [managing access](../managing-access). + +Cluster-level configuration and resources, including logs that might contain message data, are accessible through your Kubernetes platform. + +[Access and authorization controls](https://kubernetes.io/docs/reference/access-authn-authz/controlling-access/){:target="_blank"} can be used to control which users are able to access this cluster-level information. + +## Data Processing + +### Encryption of connection to {{site.data.reuse.es_name}} + +Connections to {{site.data.reuse.es_name}} are [secured using TLS](../../installing/configuring/#configuring-access). If you want to [use your own CA certificates](../../installing/configuring/#using-your-own-certificates) instead of those generated by the operator, you can provide them in the `EventStreams` custom resource settings. + +### Encryption of connections within {{site.data.reuse.es_name}} + +Internal communication between {{site.data.reuse.es_name}} pods is [encrypted by default](../../installing/configuring/#configuring-encryption-between-pods) using TLS. + +## Data Monitoring + +{{site.data.reuse.es_name}} provides a range of [monitoring](../../administering/cluster-health/) features that users can exploit to gain a better understanding of how applications are performing. diff --git a/_es_11.5/04-security/08-fips.md b/_es_11.5/04-security/08-fips.md new file mode 100644 index 00000000..b1b27806 --- /dev/null +++ b/_es_11.5/04-security/08-fips.md @@ -0,0 +1,34 @@ +--- +title: "Enabling Federal Information Processing Standards (FIPS)" +excerpt: "Find out how to set up Event Streams in a FIPS-compliant manner." +categories: security +slug: fips +toc: true +--- + +Find out how to set up {{site.data.reuse.es_name}} to be FIPS-compliant by using a boundary approach that is enabled by the ["FIPS Wall"](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=reference-fips-compliance){:target="_blank"}. + +## Requirements + +To run a FIPS-compliant {{site.data.reuse.es_name}} deployment, ensure that you have a [FIPS-enabled OpenShift Container Platform cluster](https://docs.openshift.com/container-platform/4.16/installing/installing-fips.html#installing-fips-mode_installing-fips){:target="_blank"} available with {{site.data.reuse.es_name}} version of 11.3.0 or later installed. + +### Configuring {{site.data.reuse.es_name}} for FIPS + +To configure your {{site.data.reuse.es_name}} instance for deployment within a FIPS-compliant boundary wall, set the following options in the {{site.data.reuse.es_name}} custom resource that defines your instance: + +1. Restrict external access to Kafka listeners, the Apicurio Registry, the Admin API, and the REST Producer by setting the value of `type` to `internal` in the following sections: + - Kafka listeners: `spec.strimziOverrides.kafka.listeners` + - Apicurio Registry: `spec.apicurioRegistry.endpoints` + - Admin API: `spec.adminApi.endpoints` + - REST Producer: `spec.restProducer.endpoints` + +1. Disable the {{site.data.reuse.es_name}} UI by removing the `spec.adminUI` section. + +For more information about these configuration options, see [configuring](../../installing/configuring/#configuring-access). + +## Limitations + +The FIPS-complaint {{site.data.reuse.es_name}} deployment limits the use of the following features: + +- External endpoints are not supported for Kafka listeners, the Apicurio Registry, the Admin API, and the REST Producer. +- {{site.data.reuse.es_name}} UI, geo-replication, and CLI cannot be used. diff --git a/_es_11.5/05-atopic-mirroring/01-about-topic-mirroring.md b/_es_11.5/05-atopic-mirroring/01-about-topic-mirroring.md new file mode 100644 index 00000000..517d6955 --- /dev/null +++ b/_es_11.5/05-atopic-mirroring/01-about-topic-mirroring.md @@ -0,0 +1,101 @@ +--- +title: "About topic mirroring" +excerpt: "Learn about mirroring topics between your clusters." +categories: mirroring +slug: about +toc: true +--- + + +Sometimes there is a need to copy messages from a topic in one Kafka cluster to an equivalent topic in another cluster. This is known as topic mirroring. There are a number of possible reasons to perform topic mirroring, including data distribution, disaster recovery, and infrastructure maintenance. + +Two capabilities exist within {{site.data.reuse.es_name}} to enable topic mirroring: + +* [MirrorMaker 2.0](../mirrormaker): MirrorMaker 2.0 is a part of Apache Kafka that is available (and supported) for use within {{site.data.reuse.es_name}}. +* [Geo-replication](../../georeplication/about): {{site.data.reuse.es_name}} provides a geo-replication feature based on MirrorMaker 2.0. Geo-replication is a simpler mechanism for configuring mirrored topics, with a slightly reduced feature set. + +The geo-replication feature, unique to {{site.data.reuse.es_name}}, is well suited to data distribution use cases. For advanced use cases such as disaster recovery and infrastructure maintenance, MirrorMaker 2.0 might be required. + + +## Common use cases + +The following sections discuss the various use cases in detail and the considerations in relation to topic mirroring. + +![Topic mirroring use case table]({{ 'images' | relative_url }}/mirroringtable.png "Diagram showing the various use cases for topic mirroring in Apache Kafka"){:height="50%" width="50%"} + +### Data distribution + +Topics are often mirrored to make their data more directly accessible in another region. One of the most common reasons for this is to reduce latency for consumers. Creating a copy of a topic in a data center that is local to the consumers reduces the network distance between the consumer and the Kafka broker. This can have a significant effect on the achievable throughput of event consumption. + +![Mirroring to reduce client latency]({{ 'images' | relative_url }}/reducing-latency.png "Diagram showing mirroring of topics between regions to reduce latency"){:height="75%" width="75%"} + +This pattern can also result in a reduction of data movement costs, especially if the second region has a significant number of consumers. Cloud vendors often charge for the transfer of data between their cloud regions (ingress and egress). If all consumers were to consume directly from the original region, this would create a very high volume of expensive cross-region traffic. If the consumers instead go to a local replica of the topic within their own region, there will only be a single data movement cost to copy the data through mirroring. + +Also, there might be multiple regions that would benefit from a copy of the data, which results in a broadcast pattern mirroring to multiple clusters. + +![Mirroring to broadcast to multiple regions]({{ 'images' | relative_url }}/broadcast.png "Diagram showing use of topic mirroring to broadcast to multiple consuming regions"){:height="60%" width="60%"} + +#### Data isolation + +There might be instances where the data on some topics in a Kafka cluster is sensitive (sometimes called sovereign data) and should never leave the region. It can be complex to validate that you have set up access control lists correctly, and ensure that consumers can only see the data they are allowed to. + +![Mirroring for data isolation]({{ 'images' | relative_url }}/data-isolation.png "Diagram showing use of topic mirroring to simplify data isolation between regions"){:height="75%" width="75%"} + +Sometimes a better option is to provide each region with a separate Kafka cluster (data isolation) and use mirroring to copy only topics with data that is appropriate for the other region. This makes the security model much simpler on the consumer side since the consumers in the second region do not even have a copy of topics that they should not see. + +#### Aggregation + +The aggregation type of topic mirroring is useful when the elements of data are produced in multiple different regions, but the data must be processed as a whole. For example, notifications of sales might be produced in multiple countries, but must be aggregated to analyze worldwide trends and manage overall inventory. + +![Mirroring to aggregate topics]({{ 'images' | relative_url }}/aggregation.png "Diagram showing aggregation of topics from multiple regions into a cluster in a central aggregation cluster"){:height="60%" width="60%"} + +When all the regions produce directly to the central cluster, there are heavy dependencies on the availability and performance of long network paths. Furthermore, the producers suffer higher latency when producing messages. + +To reduce latency and improve the availability and performance, you can use the aggregate type of mirroring as follows: + +1. Ensure that producers write to local clusters, and then pull topic data from the local cluster into a central cluster to process it in aggregate. +1. Use topic wildcards in your [subscription](https://kafka.apache.org/37/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html#subscribe(java.util.regex.Pattern,org.apache.kafka.clients.consumer.ConsumerRebalanceListener)){:target="_blank"} to consume from all the topics at once and perform your processing. + +#### Shared aggregation + +Another variant of the aggregation pattern is a shared aggregate topic. In our examples earlier, applications are specifically either producers or consumers, but there are circumstances where you might have both consumers and producers of a topic on both the local and remote locations. + +![Mirroring for a shared aggregate topic]({{ 'images' | relative_url }}/shared-aggregate-topic.png "Diagram showing how two physical topics can be combined to create one logical topic that enable production and consumption across two clusters"){:height="75%" width="75%"} + +To facilitate this, complete the following steps: + +1. Create a single logical topic from two physical topics. +2. Ensure that the producers always produce to a particular local topic which is then mirrored across to the other region. +3. Ensure that the consumers [subscribe](https://kafka.apache.org/37/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html#subscribe(java.util.regex.Pattern,org.apache.kafka.clients.consumer.ConsumerRebalanceListener)){:target="_blank"} by using topic-based wildcards to both the local topic and the remote topic that is mirrored, hence receiving events from producers in both regions. + +**Note:** With the shared aggregate pattern, the consumers in each region might receive locally produced events faster than remote produced events. This might be an issue if events with the same key are created on both local and remote locations. Sequence of events is often important and the normal sequence preservation from the Kafka partitioning strategy does not prevail across multiple separate topics. + +### Disaster recovery + +Topic mirroring can also be used as part of a disaster recovery plan to replicate the events to a separate cluster on the disaster recovery site. + +![Mirroring for disaster recovery]({{ 'images' | relative_url }}/disaster-recovery.png "Diagram showing how disaster recovery is achieve using mirroring"){:height="60%" width="60%"} + +**Important:** When planning for disaster recovery, review key business requirements such as the Recovery Point Objective (RPO) and Recovery Time Objective (RTO) for the topics that are being replicated. The topic data (the events) as well as the topic configuration (including partitioning information etc.) must be replicated to ensure that the target cluster looks largely identical to the consumers when they are switched over to it. + +**Note:** Ensure that you review how the consumer offsets are replicated. This can be done by topic mirroring if the consumers are storing offsets in Kafka. However, if the consumers are storing their own offsets, this needs to be separately handled as part of the disaster recovery plan. + +For more information, see [disaster recovery](../../installing/disaster-recovery/). + +### Infrastructure maintenance + +Mirroring is often employed to enable transition from one Kafka infrastructure to another. Mirroring in this situation is used as a transitory step, and only in place until the origin Kafka cluster is no longer required. + +This might be used to migrate to a cluster stationed locally to a new version of an application, or to move to an upgraded Kafka cluster when an in-place upgrade is for some reason untenable. + +![Mirroring for application migration]({{ 'images' | relative_url }}/migration.png "Diagram showing how mirroring is used to enable migration to a new version of an event driven application"){:height="75%" width="75%"} + +Topic mirroring is always asynchronous, so the origin and new applications will inevitably have consumed to a different offset in the event stream at any given time. Depending on the nature of the application, this might affect whether the origin and new applications can work in parallel alongside one another during the migration, or the origin application must be stopped before the new application can start. + +You also must consider at which point the producers must be migrated to the new cluster. To ensure that the consumers always have access to new messages until they are completely migrated, the producers would often be migrated after all the consumers are migrated. + +**Important:** Ensure that all the events that are produced to the origin cluster have been mirrored across, before the producers begin producing events on the new cluster. + +The other infrastructure upgrade use case is that of migrating applications to a new Kafka infrastructure, typically when performing a major upgrade to Kafka, or for example moving to a completely different hardware. The previous application migration considerations are applicable for this use case as well. + + diff --git a/_es_11.5/05-atopic-mirroring/02-mirrormaker.md b/_es_11.5/05-atopic-mirroring/02-mirrormaker.md new file mode 100644 index 00000000..bb975612 --- /dev/null +++ b/_es_11.5/05-atopic-mirroring/02-mirrormaker.md @@ -0,0 +1,44 @@ +--- +title: "About MirrorMaker 2.0" +excerpt: "Learn about how to use MirrorMaker 2.0 for topic replication across your clusters." +categories: mirroring +slug: mirrormaker +toc: true +--- + +MirrorMaker 2.0 is a capability within Apache Kafka that is used for copying (mirroring) topics from one Kafka cluster to another. MirrorMaker 2.0 can be used for a [wide range of use cases](../about) such as data distribution, to disaster recovery, and infrastructure maintenance. + +Since MirrorMaker 2.0 is a part of Apache Kafka, it is also part of {{site.data.reuse.es_name}}, and is fully supported. + +**Note:** Also see [geo-replication](../../georeplication/about) in {{site.data.reuse.es_name}}, which provides a simplified way of configuring MirrorMaker 2.0 for a subset of use cases. + +MirrorMaker 2.0 makes use of the Kafka Connect framework, and offers dynamic configuration changes, offset synchronization, bidirectional mirroring, and improved performance. + +## Offset synchronization + +An important feature of MirrorMaker 2.0 is the offset synchronization. + +When topics are mirrored to another cluster, the event offset for a given event in the origin topic will be different to the event in the destination topic. This can occur due to records that are deleted from the origin topic due to retention policies, or because of the control records that are created when using transactions. The issue is that if a consumer moves from the origin to the destination cluster without considering these offset differences, it will duplicate, or worse, loose events. + +MirrorMaker 2.0 handles these offset differences by storing a mapping between the offsets in the origin and destination clusters. It then uses this mapping to correctly populate the consumer offsets topic on the destination cluster. When consumers start up against the destination cluster, they will begin reading from the correct point in the topic's event stream. + +MirrorMaker 2.0 also synchronizes topic partitioning and access control lists, to ensure the destination cluster is as similar as possible to the origin cluster. + +## Reference + +The following blog posts explain how to configure MirrorMaker 2.0 for its most [common use cases](../about). + +Data distribution: + +- [Using MirrorMaker 2.0 to aggregate events from multiple regions](https://community.ibm.com/community/user/integration/blogs/dale-lane1/2024/03/29/mirrormaker-for-aggregating-across-regions){:target="_blank"} +- [Using MirrorMaker 2.0 to broadcast events to multiple regions](https://community.ibm.com/community/user/integration/blogs/dale-lane1/2024/04/02/mirrormaker-for-broadcasting-across-regions){:target="_blank"} +- [Using MirrorMaker 2.0 to share topics across multiple regions](https://community.ibm.com/community/user/integration/blogs/dale-lane1/2024/04/05/mirrormaker-for-shared-conceptual-topics){:target="_blank"} + +Disaster recovery: + +- [Using MirrorMaker 2.0 to create a failover cluster](https://community.ibm.com/community/user/integration/blogs/dale-lane1/2024/04/08/mirrormaker-for-failover){:target="_blank"} +- [Using MirrorMaker 2.0 to restore events from a backup cluster](https://community.ibm.com/community/user/integration/blogs/dale-lane1/2024/04/12/mirrormaker-for-backup-and-restore){:target="_blank"} + +Infrastructure maintenance: + + * [Using MirrorMaker 2.0 to migrate to a different region](https://community.ibm.com/community/user/integration/blogs/dale-lane1/2024/04/18/mirrormaker-for-migration){:target="_blank"} diff --git a/_es_11.5/05-atopic-mirroring/05-replication-failover.md b/_es_11.5/05-atopic-mirroring/05-replication-failover.md new file mode 100644 index 00000000..37bf9455 --- /dev/null +++ b/_es_11.5/05-atopic-mirroring/05-replication-failover.md @@ -0,0 +1,191 @@ +--- +title: "Switching clusters" +excerpt: "Using mirroring to recover when clusters become unavailable." +categories: mirroring +slug: failover +toc: true +--- + +When one of your origin {{site.data.reuse.es_name}} clusters experiences problems and becomes unavailable, you can switch your client applications over to use the mirrored topics on your destination {{site.data.reuse.es_name}} cluster. + +## Preparing clusters and applications for switching + +To make switching of clusters require a little intervention. Consider the following guidance when preparing your {{site.data.reuse.es_name}} clusters and applications. + +### Configure your applications for switching + +Set up your applications so that reconfiguring them to switch clusters is as easy as possible. Code your applications so that security credentials, certificates, bootstrap server addresses, and other configuration settings are not hard-coded, but can be set in configuration files, or otherwise injected into your applications. + +### Use the same certificates + +Consider using the same certificates for both the origin and destination clusters, by [providing your own certificates at installation](../../installing/configuring/#using-your-own-certificates). This allows applications to use a single certificate to access either cluster. + +**Note**: You must complete the process of providing your own certificates before installing an instance of Event Streams. + +**Note**: When providing your own certificates, ensure that certificate renewal processes are followed at both the origin and destination clusters, so that both clusters continue to use the same certificates. + +### Set up the same access to both clusters + +Consider providing your applications the same access to both the origin and destination clusters. For example, you can duplicate the application `KafkaUser` credentials from the origin cluster to the destination cluster. This allows applications to use a single set of credentials to access either cluster. Use the following commands to retrieve the `KafkaUser` credentials and custom resource from the origin cluster, and then create a new `KafkaUser` with these credentials on the destination cluster: +1. Log in to your origin cluster. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to retrieve the name of the secret for the `KafkaUser`: + + ```shell + kubectl get kafkauser --namespace -o jsonpath='{"username: "}{.status.username}{"\nsecret-name: "}{.status.secret}{"\n"}' + ``` + + The command provides the following output: + - The principal username + - The name of the Kubernetes `Secret`, which includes the namespace, containing the SCRAM password or the TLS certificates. + +3. Use the `secret-name` from the previous step to run the following command. The command retrieves the credentials from the Kubernetes `Secret` and and saves them to the `kafkauser-secret.yaml` file: + + ```shell + kubectl get secret --namespace -o yaml > kafkauser-secret.yaml + ``` + +4. Run the following command to retrieve the `KafkaUser` custom resource YAML and save it to the `kafkauser.yaml` file: + + ```shell + kubectl get kafkauser --namespace -o yaml > kafkauser.yaml + ``` + +5. Log in to your destination cluster. {{site.data.reuse.cncf_cli_login}} +6. Edit both the `kafkauser-secret.yaml` and `kafkauser.yaml` files to set the correct namespace and {{site.data.reuse.es_name}} cluster name for the following properties: + - `metadata.namespace`: provide the namespace of your destination cluster. + - `metadata.labels["eventstreams.ibm.com/cluster"]`: provide the name of your destination cluster. +7. Run the following command to create the Kubernetes `Secret` containing the `KafkaUser` credentials on the destination cluster: + + ```shell + kubectl apply -f kafkauser-secret.yaml + ``` + + **Note**: You must run this command before the creation of the `KafkaUser` to ensure the same credentials are available on both the origin and destination clusters. +8. Run the following command to create the `KafkaUser` on the destination cluster: + + ```shell + kubectl apply -f kafkauser.yaml + ``` + +**Note**: To duplicate `KafkaUser` credentials that use Mutual TLS authentication, the origin and destination cluster must be [configured with the same certificates for the client CA at installation](../../installing/configuring/#using-your-own-certificates). + +**Note**: When `KafkaUser` credentials or Access Control Lists (ACLs) are modified on the origin cluster, the changes will need to be duplicated to the destination cluster to ensure that you can still switch clusters. + +### Use regular expressions for consumer topic subscriptions + +Geo-replicated topics on the destination cluster will have a prefix added to the topic name. The prefix is the name of the {{site.data.reuse.es_name}} instance on the origin cluster, as defined in the `EventStreams` custom resource, for example `my_origin.`. Consider using regular expressions to define the topics that consuming applications are subscribed to, for example `.*`. Using a regular expression means that the topic subscription does not need to change when switching to the prefixed topic names on the destination cluster. + +### Plan to update consumer group offsets + +Consider how you will [update the consumer group offsets](#updating-consumer-group-offsets) in consuming applications when switching clusters. Mirroring includes consumer group checkpointing to store the mapping of consumer group offsets, allowing consuming applications to continue processing messages at the appropriate offset positions. + +### Produce to the same topic name + +When switching clusters, produce to the same topic name on the destination cluster as was used on the origin cluster. This will ensure mirrored messages and directly produced messages are stored in separate topics. If consuming applications use regular expressions to subscribe to both topics, then both sets of messages will be processed. + +### Consider message ordering + +If message ordering is required, configure your consuming applications to process all messages from the mirrored topic on the destination cluster before producing applications are restarted. + +## Updating existing applications to use mirroed topics on the destination cluster + +If you are not using the same certificates and credentials on the origin and destination clusters, use the following instructions to retrieve the information required to update your applications so that they can use the mirrored topics from the destination cluster. For example, if using the geo-replication feature: + +1. Log in to your destination {{site.data.reuse.es_name}} cluster as an administrator. +2. Click **Connect to this cluster**. +3. Go to the **Resources** tab, and use the information on the page to change your client application settings to use the geo-replicated topic on the destination cluster. You need the following information to do this: + + * **Bootstrap server**: In the **Kafka listener and credentials** section, select the listener from the list. + - Click the **External** tab for applications connecting from outside of the Kubernetes cluster. + - Click the **Internal** tab for applications connecting from inside the Kubernetes cluster. + + * **Credentials**: To connect securely to {{site.data.reuse.es_name}}, your application needs credentials with permission to access the cluster and resources such as topics. In the **Kafka listener and credentials** section, click the **Generate SCRAM credentials** or **Generate TLS credentials** button next to the listener you are using, and follow the instructions to select the level of access you want to grant to your resources with the credentials. + + * **Certificates**: A certificate is required by your client applications to connect securely to the destination cluster. In the **Certificates** section, download either the PKCS12 certificate or PEM certificate. If you use the PKCS12 certificate, make a copy of the **Certificate password** to use with the certificate in your application. + +After the connection is configured, your client application can continue to operate using the geo-replicated topics on the destination cluster. + +## Updating consumer group offsets + +The topic on the origin cluster and the mirrored topic on the destination cluster might have different offsets for the same messages, depending on when mirroring started. This means that a consuming application that is switched to use the destination cluster cannot use the consumer group offset from the origin cluster. + +### Updating consumer group offsets by using checkpoints + +Mirror Maker 2.0's `MirrorCheckpointConnector` automatically stores consumer group offset checkpoints for all origin cluster consumer groups. Each checkpoint maps the last committed offset for each consumer group in the origin cluster to the equivalent offset in the destination cluster. The checkpoints are stored in the `.checkpoints.internal` topic on the destination cluster. + +**Note**: Consumer offset checkpoint topics are internal topics that are not displayed in the UI and CLI. Run the following CLI command to include internal topics in the topic listing: + +```shell +kubectl es topics --internal +``` + +When processing messages from the destination cluster, select one of the following methods depending on your mirroring technology: +* If you are using MirrorMaker 2.0, set the `sync.group.offsets.enabled = true` to have MirrorMaker's `CheckPointConnector` update the consumer offsets table in the target cluster. When switched across to the new cluster, the consumers will then automatically continue from the correct offsets. +* If using the geo-replication feature, configure your consumer application to use the checkpoints and start consuming from an offset that is equivalent to the last committed offset on the origin cluster. + +If your application is written in Java, you can use Kafka's [RemoteClusterUtils](https://kafka.apache.org/26/javadoc/org/apache/kafka/connect/mirror/RemoteClusterUtils.html){:target="_blank"} class to provide the `translateOffsets()` utility method and retrieve the destination cluster offsets for a consumer group from the checkpoints topic. You can then use the `KafkaConsumer.seek()` method to override the offsets that the consumer will use on the next `poll`. + +For example, the following Java code snippet will update the `example-group` consumer group offset from the `origin-cluster` cluster to the destination cluster equivalent: + +```shell +// Retrieve the mapped offsets for the destination cluster topic-partitions +Map destinationOffsetsMap = RemoteClusterUtils.translateOffsets(properties, "origin-cluster", + "example-group", Duration.ofMillis(10000)); + +// Update the KafkaConsumer to start at the mapped offsets for every topic-partition +destinationOffsetsMap.forEach((topicPartition, offsetAndMetadata) -> kafkaConsumer.seek(topicPartition, offsetAndMetadata)); + +// Retrieve records from the destination cluster, starting from the mapped offsets +ConsumerRecords records = kafkaConsumer.poll(Duration.ofMillis(10000)) +``` + +**Note**: To configure how often checkpoints are stored and which consumer groups are stored in the checkpoints topic, you can edit the following properties in your Kafka MirrorMaker 2.0 custom resource: + - `spec.mirror.checkpointConnector.config` + - `spec.mirror.groupsPattern` + +### Updating consumer group offsets manually + +If you want your client application to continue processing messages on the destination cluster from the point they reached on the topic on the origin cluster, or if you want your client application to start processing messages from the beginning of the topic, you can use the `kubectl es group-reset` command. + +* To continue processing messages from the point they reached on the topic on the origin cluster, you can specify the offset for the consumer group that your client application is using: + + 1. {{site.data.reuse.es_cli_init_111}} + 2. Run the `kubectl es group-reset` command as follows: + + ```shell + kubectl es group-reset --group --topic --mode datetime --value + ``` + + For example, the following command instructs the applications in consumer group `consumer-group-1` to start consuming messages with timestamps from after midday on 28th September 2018: + + ```shell + kubectl es group-reset --group consumer-group-1 --topic GEOREPLICATED.TOPIC --mode datetime --value 2018-09-28T12:00:00+00:00 --execute + ``` + +* To start processing messages from the beginning of the topic, you can use the `--mode earliest` option, for example: + + ```shell + kubectl es group-reset --group consumer-group-1 --topic GEOREPLICATED.TOPIC --mode earliest --execute + ``` + +These methods also avoid the need to make code changes to your client application. + +## Reverting message production and consumption back to the origin cluster + +When the origin {{site.data.reuse.es_name}} cluster becomes available again, you can switch your client applications back to use the topics on your origin cluster. If messages have been produced directly to the destination cluster, use the following steps to replicate those messages to the origin cluster before switching back to using it. + + - [Create an `EventStreamsGeoReplicator` custom resource](../planning) configured to connect to the origin {{site.data.reuse.es_name}} cluster, and set up geo-replication in the reverse direction to the original geo-replication flow. This means there will be a geo-replicator running on the origin cluster which copies messages from non-geo-replicated topics on the destination cluster back to geo-replicated topics on the origin cluster. + - The geo-replicated topic named `.` on the destination cluster will not have new geo-replicated messages arriving, as the producing applications have been switched to produce messages directly to the topic without a prefix on the destination cluster. Ensure that the geo-replicated topic on the destination cluster is not geo-replicated back to the origin cluster as this will result in duplicate data on the origin cluster. + - Switch the producing and consuming applications back to the origin cluster again by following the [previous instructions](#preparing-clusters-and-applications-for-switching). Producing applications will continue to produce messages to the original topic name on the origin cluster, and consuming applications will read from both the geo-replicated topics and the original topics on the origin cluster. Consuming applications will need their [consumer group offsets to be correctly updated](#updating-consumer-group-offsets) for the offset positions on the origin cluster. + +**Note**: Due to the asynchronous nature of geo-replication, there might be messages in the original topics on the origin cluster that had not been geo-replicated over to the destination cluster when the origin cluster became unavailable. You will need to decide how to handle these messages. Consider setting consumer group offsets so that the messages are processed, or ignore the messages by setting consumer group offsets to the latest offset positions in the topic. + +For example, if the origin cluster is named `my_origin`, the destination cluster is named `my_destination`, and the topic on the `my_origin` cluster is named `my_topic`, then the geo-replicated topic on the `my_destination` cluster will be named `my_origin.my_topic`. + + - When the `my_origin` cluster becomes unavailable, producing applications are switched to the `my_destination` cluster. The `my_destination` cluster now has topics named `my_topic` and `my_origin.my_topic`. Consuming applications are also switched to the `my_destination` cluster and use the regular expression `.*my_topic` to consume from both topics. + - When the `my_origin` cluster becomes available again, reverse geo-replication is set up between the clusters. The `my_origin` cluster now has the topic named `my_topic` and a new geo-replicated topic named `my_destination.my_topic`. The topic named `my_destination.my_topic` contains the messages that were produced directly to the `my_destination` cluster. + - Producing applications are producing to the topic named `my_topic` on the `my_destination` cluster, so the geo-replicated topic named `my_origin.my_topic` on the `my_destination` cluster does not have any new messages arriving. Existing messages in the topic named `my_origin.my_topic` are consumed from the `my_destination` cluster until there is no more processing of the messages required. + + **Note:** The geo-replicated topic named `my_origin.my_topic` is not included in the reverse geo-replication back to the `my_origin` cluster, as that would create a geo-replicated topic named `my_destination.my_origin.my_topic` on the `my_origin` cluster containing the same messages as in the topic named `my_topic`. + - Producing applications are now switched back to the `my_origin` cluster, continuing to produce to the topic named `my_topic`. + - Consuming applications are also switched back to the `my_origin` cluster, with consumer group offsets updated for the offset positions at the `my_origin` cluster. Consuming applications continue to use the regular expression `.*my_topic` to consume from both the topic named `my_topic` and the geo-replicated topic named `my_destination.my_topic`. diff --git a/_es_11.5/05-replication/01-about-replication.md b/_es_11.5/05-replication/01-about-replication.md new file mode 100644 index 00000000..d67722e9 --- /dev/null +++ b/_es_11.5/05-replication/01-about-replication.md @@ -0,0 +1,43 @@ +--- +title: "About geo-replication" +excerpt: "Learn about setting up geo-replication for your clusters." +categories: georeplication +slug: about +toc: true +--- + +You can deploy multiple instances of {{site.data.reuse.es_name}} and use the included geo-replication feature to synchronize data between your clusters that are typically located in different geographical locations. + +![Event Streams geo-replication architecture]({{ 'images' | relative_url }}/architectures/ibm-event-automation-diagrams-es-georep.svg "Diagram showing the architecture of the Event Streams geo-replication as part of IBM Event Automation.") + +Geo-replication can help with various service availability scenarios, for example: + +* Making local copies of event topics available closer to applications in other locations in order to improve event consumption performance. +* Making mission-critical data safe: you might have mission-critical data that your applications depend on to provide services. Using the geo-replication feature, you can back up your topics to several destinations to ensure their safety and availability. +* Migrating data: you can ensure your topic data can be moved to another deployment, for example, when switching from a test to a production environment. + +**Note:** The geo-replication feature only provides access to a subset of the capabilities of the underlying [MirrorMaker 2.0](../../mirroring/mirrormaker) capability. For certain use cases, such as creating copies of topics for [disaster recovery](../../installing/disaster-recovery) purposes, it is best to use the underlying MirrorMaker 2.0 directly to access a wider set of features. + +## How it works + +The Kafka cluster where you have the topics that you want to make copies of is called the "origin cluster". + +The Kafka cluster where you want to copy the selected topics to is called the "destination cluster". + +So, one cluster is the origin where you want to copy the data from, while the other cluster is the destination where you want to copy the data to. + +Any of your {{site.data.reuse.es_name}} clusters can become a destination for geo-replication. At the same time, the origin cluster can also be a destination for topics from other sources. + +Geo-replication not only copies the messages of a topic, but also copies the topic configuration, the topic's metadata, its partitions, and even preserves the timestamps from the origin topic. + +After geo-replication starts, the topics are kept in sync. If you add a new partition to the origin topic, the geo-replicator adds a partition to the copy of the topic on the destination cluster. + +You can set up geo-replication by using the {{site.data.reuse.es_name}} UI or CLI. + +## What to replicate + +What topics you choose to replicate depend on your use case. + +For example, you might choose to replicate all topics to destination clusters closer to the consumers of those topics, so that the consumers can read the topics at lower latency and network cost. + +Alternatively, you can use geo-replication to only replicate a subset of topics to another geography that contain data specific to that country, but have other topics that contain more locationally sensitive data to be only present in the origin cluster. diff --git a/_es_11.5/05-replication/02-planning-replication.md b/_es_11.5/05-replication/02-planning-replication.md new file mode 100644 index 00000000..2f6761f9 --- /dev/null +++ b/_es_11.5/05-replication/02-planning-replication.md @@ -0,0 +1,155 @@ +--- +title: "Planning for geo-replication" +excerpt: "Plan geo-replication for your clusters." +categories: georeplication +slug: planning +toc: true +--- + +Consider the following when planning for geo-replication: + +- If you want to use the CLI to set up geo-replication, ensure you have the [Kubernetes CLI installed](https://kubernetes.io/docs/tasks/tools/){:target="_blank"}. +- Geo-replication requires both the origin and destination {{site.data.reuse.es_name}} instances to have client authentication enabled on the external listener and the internal TLS listener. +- If you are using geo-replication for disaster-recovery scenarios, see the guidance about [configuring your clusters and applications](../../mirroring/failover/#preparing-clusters-and-applications-for-switching) to ensure you can switch clusters if one becomes unavailable. +- [Prepare your destination cluster](#preparing-a-destination-cluster) by creating an EventStreamsGeoReplicator instance and defining the number of geo-replication workers. +- [Identify the topics](../about/#what-to-replicate) you want to create copies of. This depends on the data stored in the topics, its use, and how critical it is to your operations. +- Message history is included in geo-replication. The amount of history is determined by the message retention option set when the topics were created on the origin cluster. +- The replicated topics on the destination cluster will have a prefix added to the topic name. The prefix is the name of the {{site.data.reuse.es_name}} instance on the origin cluster, as defined in the EventStreams custom resource, for example `my_origin.`. +- Configuration of geo-replication is done by the UI, the CLI or by directly editing the associated KafkaMirrorMaker2 custom resource. + +## Preparing a destination cluster + +Before you can set up geo-replication and start replicating topics, you must create an EventStreamsGeoReplicator custom resource at the destination. The {{site.data.reuse.es_name}} operator uses the EventStreamsGeoReplicator custom resource to create a configured KafkaMirrorMaker2 custom resource. The KafkaMirrorMaker2 custom resource is used by the {{site.data.reuse.es_name}} operator to create geo-replication workers, which are instances of Kafka Connect running Kafka MirrorMaker 2.0 connectors. The number of geo-replication workers running at the destination cluster is configured in the EventStreamsGeoReplicator custom resource. + +The number of workers depend on the number of topics you want to replicate, and the throughput of the produced messages. + +For example, you can create a small number of workers at the time of installation. You can then increase the number later if you find that your geo-replication performance is not able to keep up with making copies of all the selected topics as required. Alternatively, you can start with a high number of workers, and then decrease the number if you find that the workers underperform. + +**Important:** For high availability reasons, ensure you have at least 2 workers on your destination cluster in case one of the workers encounters problems. + +You can configure the number of workers at the time of creating the EventStreamsGeoReplicator instance, or you can modify an existing EventStreamsGeoReplicator instance, even if you already have geo-replication set up and running on that cluster. + +### Configuring a new installation + +If you are installing a new EventStreamsGeoReplicator instance for geo-replication to a destination cluster, you must specify the existing destination {{site.data.reuse.es_name}} instance you are connecting to. You can also specify the number of workers as part of configuring the EventStreamsGeoReplicator instance. + +To configure the number of workers at the time of installation, see the following sections. + +#### Using the UI on {{site.data.reuse.openshift_short}} + +If you are using {{site.data.reuse.openshift_short}}, to create a new EventStreamsGeoReplicator instance for geo-replication by using the UI: +1. Go to where your destination cluster is installed. {{site.data.reuse.openshift_ui_login}} +2. From the navigation menu, click **Operators > Installed Operators**. +3. In the **Projects** drop-down list, select the project that contains the existing destination {{site.data.reuse.es_name}} instance. +4. Select the {{site.data.reuse.es_name}} operator in the list of installed operators. +5. In the **Operator Details > Overview** page, find the **Geo-Replicator** tile in the list of **Provided APIs** and click **Create Instance**. +6. In the **Create EventStreamsGeoReplicator** page, edit the provided YAML to set values for the following properties. + - In the **metadata.labels** section, set the **eventstreams.ibm.com/cluster** property value to the name of your destination {{site.data.reuse.es_name}} instance. + - Set the **metadata.name** property value to the name of your destination {{site.data.reuse.es_name}} instance. + - Set the **spec.replicas** property value to the number of geo-replication workers you want to run. + ![Create EventStreamsGeoReplicator]({{ 'images' | relative_url }}/georeplication_create_eventstreamsgeoreplicator_11.2.png "Screen capture showing the Create EventStreamsGeoReplicator page with completed YAML."){:height="80%" width="80%"} +7. Click **Create**. +8. The new EventStreamsGeoReplicator instance is listed in the **Operator Details > EventStreamsGeoReplicator** page. + +If the EventStreamsGeoReplicator instance is configured correctly, a KafkaMirrorMaker2 custom resource is created. You can see the details for the KafkaMirrorMaker2 custom resource in the **Kafka Mirror Maker 2** tab of the **Operator Details** page. + +#### Using the CLI + +To create a new EventStreamsGeoReplicator instance for geo-replication by using the CLI: +1. Go to where your destination cluster is installed. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to set the namespace that contains the existing destination cluster: + + ```shell + kubectl config set-context --current --namespace= + ``` + +3. Define an EventStreamsGeoReplicator instance in a file. For example, the following YAML defines an EventStreamsGeoReplicator instance in the `my-project` project that is connected to the {{site.data.reuse.es_name}} instance named `my-dest-cluster` and has 3 geo-replication workers. + + ```yaml + apiVersion: eventstreams.ibm.com/v1beta1 + kind: EventStreamsGeoReplicator + metadata: + labels: + eventstreams.ibm.com/cluster: my-dest-cluster + name: my-dest-cluster + namespace: my-project + spec: + version: 11.5.0 + replicas: 3 + ``` + + **Note:** The EventStreamsGeoReplicator `metadata.name` property and `eventstreams.ibm.com/cluster` label property must be set to the name of the destination {{site.data.reuse.es_name}} instance that you are geo-replicating to. +4. Run the following command to create the EventStreamsGeoReplicator instance: + + ```shell + kubectl create -f + ``` + +5. The new EventStreamsGeoReplicator instance is created. +6. Run the following command to list your EventStreamsGeoReplicator instances: + + ```shell + kubectl get eventstreamsgeoreplicators + ``` + +7. Run the following command to view the YAML for your EventStreamsGeoReplicator instance: + + ```shell + kubectl get eventstreamsgeoreplicator -o yaml + ``` + +When the EventStreamsGeoReplicator instance is ready, a KafkaMirrorMaker2 instance will be created. Run the following command to list your KafkaMirrorMaker2 instances: + +```shell +kubectl get kafkamirrormaker2s +``` + +Run the following command to view the YAML for your KafkaMirrorMaker2 instance: + +```shell +kubectl get kafkamirrormaker2 -o yaml +``` + +**Note:** If you have Strimzi installed, you might need to fully-qualify the resources you are requesting. The fully-qualified name for the KafkaMirrorMaker2 instances is `kafkamirrormaker2.eventstreams.ibm.com`. + +### Configuring an existing installation + +If you want to change the number of geo-replication workers at a destination cluster for scaling purposes, you can modify the number of workers by using the UI or CLI as follows. + +#### Using the UI on {{site.data.reuse.openshift_short}} + +If you are using {{site.data.reuse.openshift_short}}, to modify the number of workers by using the UI: +1. Go to where your destination cluster is installed. {{site.data.reuse.openshift_ui_login}} +2. From the navigation menu, click **Operators > Installed Operators**. +3. In the **Projects** drop-down list, select the project that contains the destination {{site.data.reuse.es_name}} instance. +4. Select the {{site.data.reuse.es_name}} operator in the list of installed operators. +5. Click the **Geo-Replicator** tab to see the list of EventStreamsGeoReplicator instances. +6. Click the EventStreamsGeoReplicator instance that you want to modify. +7. Click the **YAML** tab. +8. Update the **spec.replicas** property value to the number of geo-replication workers you want to run. +9. Click **Save**. + +#### Using the CLI + +To modify the number of workers by using the CLI: +1. Go to where your destination cluster is installed. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to set the namespace that contains the existing destination cluster: + + ```shell + kubectl config set-context --current --namespace= + ``` + +3. Run the following command to list your EventStreamsGeoReplicator instances: + + ```shell + kubectl get eventstreamsgeoreplicators + ``` + +4. Run the following command to edit the YAML for your EventStreamsGeoReplicator instance: + + ```shell + kubectl edit eventstreamsgeoreplicator + ``` + +5. Update the **spec.replicas** property value to the number of geo-replication workers you want to run. +6. Save your changes and close the editor. diff --git a/_es_11.5/05-replication/03-setting-up-replication.md b/_es_11.5/05-replication/03-setting-up-replication.md new file mode 100644 index 00000000..91e2d671 --- /dev/null +++ b/_es_11.5/05-replication/03-setting-up-replication.md @@ -0,0 +1,155 @@ +--- +title: "Setting up geo-replication" +excerpt: "Set up geo-replication for your clusters." +categories: georeplication +slug: setting-up +toc: true +--- + +You can set up geo-replication using the {{site.data.reuse.es_name}} UI or CLI. You can then switch your applications to use another cluster when needed. + +Ensure you [plan for geo-replication](../planning/) before setting it up. + +## Defining destination clusters + +To be able to replicate topics, you must define destination clusters. The process involves logging in to your intended destination cluster and copying its connection details to the clipboard. You then log in to the origin cluster and use the connection details to point to the intended destination cluster and define it as a possible target for your geo-replication. + +### Using the UI + +1. Log in to your destination {{site.data.reuse.es_name}} cluster as an administrator. +2. Click **Topics** in the primary navigation and then click **Geo-replication**. +3. Click the **Origin locations** tab, and click **Generate connection information for this cluster**. +4. Copy the connection information to the clipboard. This information is what you need to specify the cluster as a destination for replication when you log in to your origin cluster. + + **Note:** This information includes the security credentials for your destination cluster, which is then used by your origin cluster to authenticate with the destination. +5. Log in to your origin {{site.data.reuse.es_name}} cluster as an administrator. +6. Click **Topics** in the primary navigation and then click **Geo-replication**. +7. Click **Add destination cluster** on the **Destination location** tab. +8. Paste the information you copied in step 4, wait for the validation of your payload to complete and click **Connect cluster**.\\ + The cluster is added as a destination to where you can replicate topics to.\\ + {{site.data.reuse.replicator_origin_list}} + +Alternatively, you can also use the following steps: + +1. Log in to your destination {{site.data.reuse.es_name}} cluster as an administrator. +2. Click **Connect to this cluster** on the right, and then go to the **Geo-replication** tab. +3. Click the **I want this cluster to be able to receive topics from another cluster** tile. +4. Copy the connection information to the clipboard. This information is what you need to specify the cluster as a destination for replication when you log in to your origin cluster. + + **Note:** This step includes the security credentials for your destination cluster which is then used by your origin cluster to authenticate. +5. Log in to your origin {{site.data.reuse.es_name}} cluster as an administrator. +6. Click **Connect to this cluster** on the right, and then go to the **Geo-replication** tab. +7. Click **I want to replicate topics from this cluster to another cluster**. +8. Paste the information you copied in step 4, wait for the validation of your payload to complete and click **Connect cluster**.\\ + The cluster is added as a destination to where you can replicate topics to.\\ + {{site.data.reuse.replicator_origin_list}} + + +### Using the CLI + +1. Go to your destination cluster. {{site.data.reuse.cncf_cli_login}} +2. {{site.data.reuse.es_cli_init_111}} +3. Run the following command to display the connection details for your destination cluster: + + ```shell + kubectl es geo-cluster-connect + ``` + + The command returns a base64 encoded string consisting of the API URL and the security credentials required for creating a destination cluster that should be used to configure geo-replication using the CLI. If the connection details are to be used to configure geo-replication using the UI, add the `--json` option to return a JSON-formatted string. +4. Go to your origin cluster. {{site.data.reuse.cncf_cli_login}} +5. {{site.data.reuse.es_cli_init_111}} +6. Run the following command to add the cluster as a destination to where you can replicate your topics to: + + ```shell + kubectl es geo-cluster-add --cluster-connect + ``` + + Wait for all KafkaMirrorMaker2 pods to reconcile. + +## Specifying what and where to replicate + +To select the topics you want to replicate and set the destination cluster to replicate to, use the following steps. + +### Using the UI + +1. Log in to your origin {{site.data.reuse.es_name}} cluster as an administrator. +2. Click **Topics** in the primary navigation and then click **Geo-replication**. +3. Choose a destination cluster to replicate to by clicking the name of the cluster from the **Destination locations** list. +4. Choose the topics you want to replicate by selecting the checkbox next to each, and click **Geo-replicate to destination**. + + **Tip:** You can also click the ![Add topic to geo-replication icon]({{ 'images' | relative_url }}/add_to_georeplication_icon.png "Add to geo-replication icon that is displayed in each topic row.") icon in the topic's row to add it to the destination cluster. The icon turns into a **Remove** button, and the topic is added to the list of topics that are geo-replicated to the destination cluster. + + **Note:** A prefix of the origin cluster name will be added to the name of the new replicated topic that is created on the destination cluster, resulting in replicated topics named such as `.`. + + Message history is included in geo-replication. This means all available message data for the topic is copied. The amount of history is determined by the message retention options set when the topics were created on the origin cluster. +5. Click **Create** to create a geo-replicator for the selected topics on the chosen destination cluster. Geo-replication starts automatically when the geo-replicator for the selected topics is set up successfully. + + **Note:** After clicking **Create**, it might take up to 5 to 10 minutes before geo-replication becomes active. + +For each topic that has geo-replication set up, a visual indicator is shown in the topic's row. If topics are being replicated from the cluster you are logged in to, the **Geo-replication** column displays the number of clusters the topic is being replicated to. Clicking the column for the topic expands the row to show details about the geo-replication for the topic. You can then click **View** to see more details about the geo-replicated topic in the side panel:\\ + ![Geo-replication for topic on origin]({{ 'images' | relative_url }}/georeplication_onorigin_detail_201941.png "Screen capture showing geo-replication detail after expanding row by clicking icon for topics that have geo-replication set up from the cluster you are logged into.") + +### Using the CLI + +To set up replication by using the CLI: + +1. Go to your origin cluster. {{site.data.reuse.cncf_cli_login}} +2. {{site.data.reuse.es_cli_init_111}} +3. Choose a destination cluster to replicate to by listing all available destination clusters, making the ID of the clusters available to select and copy: + + ```shell + kubectl es geo-clusters + ``` + +4. Choose the topics you want to replicate by listing your topics, making their names available to select and copy: + + ```shell + kubectl es topics + ``` + +5. Specify the destination cluster to replicate to, and set the topics you want to replicate. Use the required destination cluster ID and topic names retrieved in the previous steps. List each topic you want to replicate by using a comma-separated list without spaces in between: + + ```shell + kubectl es geo-replicator-create --destination --topics + ``` + + Geo-replication starts automatically when the geo-replicator for the selected topics is set up successfully. + +**Note:** A prefix of the origin cluster name will be added to the name of the new replicated topic that is created on the destination cluster, resulting in replicated topics named such as `.`. + +Message history is included in geo-replication. This means all available message data for the topic is copied. The amount of history is determined by the message retention option set when the topics were created on the origin cluster. + +## Considerations + +{{site.data.reuse.es_name}} geo-replication uses Kafka's MirrorMaker 2.0 to replicate data from the origin cluster to the destination cluster. + +### Replication Factor + +{{site.data.reuse.es_name}} sets the number of replicas of geo-replicated topics to 3, or if there are fewer brokers available then to the number of brokers in the destination cluster. + +If a different number of replicas are required on the destination topic, edit the value of the sourceConnector configuration property replication.factor on the MirrorMaker2 instance that is created by {{site.data.reuse.es_name}} for the geo-replication pairing. The change will apply to all new topics created by the geo-replicator on the destination cluster after the changes is made. It will not be applied to topics already configured for geo-replication. + +### Topic configuration + +MirrorMaker 2.0 has a list of topic properties that are not copied from the source cluster topic: + +* `follower.replication.throttled.replicas` +* `leader.replication.throttled.replicas` +* `message.timestamp.difference.max.ms` +* `message.timestamp.type` +* `unclean.leader.election.enable` +* `min.insync.replicas` + +It is not possible to override the value of these properties using MirrorMaker 2.0 configuration, instead the values are taken from the settings of the destination cluster. + +To query the current values set on the destination cluster: + +1. Go to your destination cluster. {{site.data.reuse.cncf_cli_login}} +2. {{site.data.reuse.es_cli_init_111}} +3. List the broker configuration by using the following command: + + ```shell + kubectl es broker 0 + ``` + +4. Update the [broker configuration](../../installing/configuring/#applying-kafka-broker-configuration-settings) to set these properties to the values if required before configuring geo-replication. diff --git a/_es_11.5/05-replication/04-replication-health.md b/_es_11.5/05-replication/04-replication-health.md new file mode 100644 index 00000000..1d331009 --- /dev/null +++ b/_es_11.5/05-replication/04-replication-health.md @@ -0,0 +1,281 @@ +--- +title: "Monitoring and managing geo-replication" +excerpt: "Check the status of your geo-replication." +categories: georeplication +slug: health +toc: true +--- +When you have geo-replication set up, you can monitor and manage your geo-replication, such as checking the status of your geo-replicators, pausing and resuming geo-replication, removing replicated topics from destination clusters, and so on. + +## From a destination cluster + +You can check the status of your geo-replication and manage geo-replicators (such as pause and resume) on your destination cluster. + +You can view the following information for geo-replication on a destination cluster: + +* The total number of origin clusters that have topics being replicated to the destination cluster you are logged into. +* The total number of topics being geo-replicated to the destination cluster you are logged into. +* Information about each origin cluster that has geo-replication set up on the destination cluster you are logged into: + - The cluster name, which includes the release name. + - The health of the geo-replication for that origin cluster: **Creating**, **Running**, **Updating**, **Paused**, **Stopping**, **Assigning**, **Offline**, and **Error**. + - Number of topics replicated from each origin cluster. + +**Tip:** As your cluster can be used as a destination for more than one origin cluster and their replicated topics, this information is useful to understand the status of all geo-replicators running on the cluster. + +### Using the UI + +To view this information on the destination cluster by using the UI: +1. Log in to your destination {{site.data.reuse.es_name}} cluster as an administrator. +2. Click **Topics** in the primary navigation and then click **Geo-replication**. +3. Click the **Origin locations** tab for details. + +To manage geo-replication on the destination cluster by using the UI: +1. Log in to your destination {{site.data.reuse.es_name}} cluster as an administrator. +2. Click **Topics** in the primary navigation and then click **Geo-replication**. +3. Click the **Origin locations** tab for details. +4. Locate the name of the origin cluster for which you want to manage geo-replication for, and choose from one of the following options: + - ![More options icon]({{ 'images' | relative_url }}/more_options.png "Three vertical dots for the more options icon at end of each row."){:height="30px" width="15px"} **More options > Pause running replicators**: To pause geo-replication and suspend replication of data from the origin cluster. + - ![More options icon]({{ 'images' | relative_url }}/more_options.png "Three vertical dots for the more options icon at end of each row."){:height="30px" width="15px"} **More options > Resume paused replicators**: To resume geo-replication and restart replication of data from the origin cluster. + - ![More options icon]({{ 'images' | relative_url }}/more_options.png "Three vertical dots for the more options icon at end of each row."){:height="30px" width="15px"} **More options > Restart failed replicators**: To restart a geo-replicator that experienced problems. + - ![More options icon]({{ 'images' | relative_url }}/more_options.png "Three vertical dots for the more options icon at end of each row."){:height="30px" width="15px"} **More options > Stop replication**: To stop geo-replication from the origin cluster. + + **Important:** Stopping replication also removes the origin cluster from the list. + +**Note**: You cannot perform these actions on the destination cluster by using the CLI. + +## From an origin cluster + +On the origin cluster, you can check the status of all of your destination clusters, and drill down into more detail about each destination. + +You can also manage geo-replicators (such as pause and resume), and remove entire destination clusters as a target for geo-replication. You can also add topics to geo-replicate. + +You can view the following high-level information for geo-replication on an origin cluster: + +* The name of each destination cluster. +* The total number of topics being geo-replicated to all destination clusters from the origin cluster you are logged into. +* The total number of workers running for the destination cluster you are geo-replicating topics to. + +You can view more detailed information about each destination cluster after they are set up and running like: + +* The topics that are being geo-replicated to the destination cluster. +* The health status of the geo-replication on each destination cluster: **Awaiting creation**, **Pending**, **Running**, **Resume**, **Resuming**, **Pausing**, **Paused**, **Removing**, and **Error**. When the status is **Error**, the cause of the problem is also provided to aid resolution. + +### Using the UI + +To view this information on the origin cluster by using the UI: +1. Log in to your origin {{site.data.reuse.es_name}} cluster as an administrator. +2. Click **Topics** in the primary navigation and then click **Geo-replication**. +3. Click the **Destination locations** tab for details. + +To manage geo-replication on the origin cluster by using the UI: +1. Log in to your origin {{site.data.reuse.es_name}} cluster as an administrator. +2. Click **Topics** in the primary navigation and then click **Geo-replication**. +3. Click the name of the destination cluster for which you want to manage geo-replication. +4. Choose from one of the following options using the top right ![More options icon]({{ 'images' | relative_url }}/more_options.png "Three vertical dots for the more options icon at end of each row."){:height="30px" width="15px"} **More options** menu: + - ![More options icon]({{ 'images' | relative_url }}/more_options.png "Three vertical dots for the more options icon at end of each row."){:height="30px" width="15px"} **More options > Pause running geo-replicator**: To pause the geo-replicator for this destination and suspend replication of data to the destination cluster for all topics. + - ![More options icon]({{ 'images' | relative_url }}/more_options.png "Three vertical dots for the more options icon at end of each row."){:height="30px" width="15px"} **More options > Resume paused geo-replicator**: To resume the paused geo-replicator for this destination and resume replication of data to the destination cluster for all topics. + - ![More options icon]({{ 'images' | relative_url }}/more_options.png "Three vertical dots for the more options icon at end of each row."){:height="30px" width="15px"} **More options > Restart failed geo-replicator**: To restart a geo-replicator that experienced problems. + - ![More options icon]({{ 'images' | relative_url }}/more_options.png "Three vertical dots for the more options icon at end of each row."){:height="30px" width="15px"} **More options > Remove cluster as destination**: To remove the cluster as a destination for geo-replication. + +To stop an individual topic from being replicated and remove it from the geo-replicator, select ![More options icon]({{ 'images' | relative_url }}/more_options.png "Three vertical dots for the more options icon at end of each row."){:height="30px" width="15px"} **More options > Stop replicating topic**. + +### Using the CLI + +To view this information on the origin cluster by using the CLI: +1. Go to your origin cluster. {{site.data.reuse.cncf_cli_login}} +2. {{site.data.reuse.es_cli_init_111}} +3. Retrieve destination cluster IDs by using the following command: + + ```shell + kubectl es geo-clusters + ``` + +4. Retrieve information about a destination cluster by running the following command and copying the required destination cluster ID from the previous step: + + ```shell + kubectl es geo-cluster --destination + ``` + + For example: + + ```shell + kubectl es geo-cluster --destination destination_byl6x + ``` + + The command returns the following information: + + ```shell + Details of destination cluster destination_byl6x + Cluster ID Cluster name REST API URL Skip SSL validation? + destination_byl6x destination https://destination-ibm-es-admapi-external-myproject.apps.geodest.ibm.com:443 false + + Geo-replicator details + Geo-replicator name Status Origin bootstrap servers + origin_es->destination-mm2connector RUNNING origin_es-kafka-bootstrap-myproject.apps.geosource.ibm.com:443 + + Geo-replicated topics + Geo-replicator name Origin topic Destination topic + origin_es->destination-mm2connector topic1 origin_es.topic1 + origin_es->destination-mm2connector topic2 origin_es.topic2 + + ``` + +Each geo-replicator creates a MirrorSource connector and a MirrorCheckpoint connector. The MirrorSource connector replicates data from the origin to the destination cluster. You can use the MirrorCheckpoint connector during [failover](../../mirroring/failover/#updating-consumer-group-offsets-by-using-checkpoints) from the origin to the destination cluster. + +To manage geo-replication on the origin cluster by using the CLI: +1. Go to your origin cluster. {{site.data.reuse.cncf_cli_login}} +2. {{site.data.reuse.es_cli_init_111}} +3. Run the following commands as required: + + - ```shell + kubectl es geo-replicator-pause --destination --name "" + ``` + + For example: + + ```shell + kubectl es geo-replicator-pause --destination destination_byl6x --name "origin_es->destination-mm2connector" + ``` + + This will pause both the MirrorSource connector and the MirrorCheckpoint connector for this geo-replicator. Geo-replication for all topics that are part of this geo-replicator will be paused. + + - ```shell + kubectl es geo-replicator-resume --destination --name "" + ``` + + For example: + + ```shell + kubectl es geo-replicator-resume --destination destination_byl6x --name "origin_es->destination-mm2connector" + ``` + + This will resume both the MirrorSource connector and the MirrorCheckpoint connector for this geo-replicator after they have been paused. Geo-replication for all topics that are part of this geo-replicator will be resumed. + + - ```shell + kubectl es geo-replicator-restart --destination --name "" --connector + ``` + + For example: + + ```shell + kubectl es geo-replicator-restart --destination destination_byl6x --name "origin_es->destination-mm2connector" --connector MirrorSourceConnector + ``` + + This will restart a failed geo-replicator MirrorSource connector. + + - ```shell + kubectl es geo-replicator-topics-remove --destination --name "" --topics + ``` + + For example: + + ```shell + kubectl es geo-replicator-topics-remove --destination destination_byl6x --name "origin_es->destination-mm2connector " --topics topic1,topic2 + ``` + + This will remove the listed topics from this geo-replicator. + + - ```shell + kubectl es geo-replicator-delete --destination --name "" + ``` + + For example: + + ```shell + kubectl es geo-replicator-delete --destination destination_byl6x --name "origin_es->destination-mm2connector" + ``` + + This will remove all MirrorSource and MirrorCheckpoint connectors for this geo-replicator. + + - ```shell + kubectl es geo-cluster-remove --destination + ``` + + For example: + + ```shell + kubectl es geo-cluster-remove --destination destination_byl6x + ``` + + This will permanently remove a destination cluster. + + **Note:** If you are unable to remove a destination cluster due to technical issues, you can use the `--force` option with the `geo-cluster-remove` command to remove the cluster. + + +## Restarting a geo-replicator with Error status + +Running geo-replicators constantly consume from origin clusters and produce to destination clusters. If the geo-replicator receives an unexpected error from Kafka, it might stop replicating and report a status of **Error**. + +Monitor your geo-replication cluster to confirm that your geo-replicator is replicating data. + +To restart a geo-replicator that has an **Error** status from the UI: +1. Log in to your origin {{site.data.reuse.es_name}} cluster as an administrator. +2. Click **Topics** in the primary navigation and then click **Geo-replication**. +3. Locate the name of the destination cluster for the geo-replicator that has an **Error** status. +4. Locate the reason for the **Error** status under the entry for the geo-replicator. +5. Either fix the reported problem with the system or verify that the problem is no longer present. +6. Select ![More options icon]({{ 'images' | relative_url }}/more_options.png "Three vertical dots for the more options icon at the top right of the destination cluster window."){:height="30px" width="15px"} **More options > Restart failed replicator** to restart the geo-replicator. + +## Using Grafana dashboards to monitor geo-replication + +Metrics are useful indicators of the health of geo-replication. They can give warnings of potential problems as well as providing data that can be used to alert on outages. Monitor the health of your geo-replicator using the available metrics to ensure replication continues. + +Configure your {{site.data.reuse.es_name}} geo-replicator to export metrics, and then view them using the example [Grafana dashboard](http://ibm.biz/es-grafana-dashboards){:target="_blank"}. + + +### Configuring metrics + +Enable export of metrics in {{site.data.reuse.es_name}} geo-replication by editing the associated KafkaMirrorMaker2 custom resource. + +#### Using the {{site.data.reuse.openshift_short}} web console + +1. Go to where your destination cluster is installed. {{site.data.reuse.openshift_ui_login}} +2. From the navigation menu, click **Operators > Installed Operators**. +3. In the **Projects** dropdown list, select the project that contains the destination {{site.data.reuse.es_name}} instance. +4. Select the {{site.data.reuse.es_name}} operator in the list of installed operators. +5. Click the **Kafka Mirror Maker 2** tab to see the list of KafkaMirrorMaker2 instances. +6. Click the KafkaMirrorMaker2 instance with the name of the instance that you are adding metrics to. +7. Click the **YAML** tab. +8. Add the `spec.metrics` property. For example: + + ```yaml + # ... + spec: + metrics: {} + # ... + ``` + +9. Click **Save**. + + +#### Using the CLI + +To modify the number of geo-replicator workers run the following using the CLI: +1. Go to where your destination cluster is installed. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to select the namespace that contains the existing destination cluster: + + ```shell + kubectl namespace + ``` + +3. Run the following command to list your `KafkaMirrorMaker2` instances: + + ```shell + kubectl get kafkamirrormaker2s + ``` + +4. Run the following command to edit the custom resource for your `KafkaMirrorMaker2` instance: + + ```shell + kubectl edit kafkamirrormaker2 + ``` + +5. Add the `spec.metrics` property. For example: + + ```yaml + spec: + metrics: {} + ``` + +6. Save your changes and close the editor. + diff --git a/_es_11.5/06-connecting/00-rest-producer-api.md b/_es_11.5/06-connecting/00-rest-producer-api.md new file mode 100644 index 00000000..3b396913 --- /dev/null +++ b/_es_11.5/06-connecting/00-rest-producer-api.md @@ -0,0 +1,278 @@ +--- +title: "Event Streams producer API" +excerpt: "Use the Event Streams producer API to connect other systems to your Kafka cluster." +categories: connecting +slug: rest-api +toc: true +--- + +{{site.data.reuse.es_name}} provides a REST API to help connect your existing systems to your {{site.data.reuse.es_name}} Kafka cluster. Using the API, you can integrate {{site.data.reuse.es_name}} with any system that supports RESTful APIs. + +The REST producer API is a scalable REST interface for producing messages to {{site.data.reuse.es_name}} over a secure HTTP endpoint. Send event data to {{site.data.reuse.es_name}}, utilize Kafka technology to handle data feeds, and take advantage of {{site.data.reuse.es_name}} features to manage your data. + +Use the API to connect existing systems to {{site.data.reuse.es_name}}, such as IBM Z mainframe systems with IBM z/OS Connect, systems using IBM DataPower Gateway, and so on. + +![Event Streams Producer API architecture]({{ 'images' | relative_url }}/architectures/ibm-event-automation-diagrams-es-restproducer.svg "Diagram showing the architecture of the Event Streams producer API as part of IBM Event Automation.") + +## About authorization + +By default {{site.data.reuse.es_name}} requires clients to be authorized to write to topics. The available authentication mechanisms for use with the REST Producer are MutualTLS (`tls`) and SCRAM SHA 512 (`scram-sha-512`). For more information about these authentication mechanisms, see the information about [managing access](../../security/managing-access/). + +The REST producer API requires any authentication credentials be provided with each REST call to grant access to the requested topic. This can be done in one of the following ways: +1. **In an HTTP authorization header**: + + You can use this method when you have control over what HTTP headers are sent with each call to the REST producer API. For example, this is the case when the API calls are made by code you control. +2. **Mutual TLS authentication** (also referred to as **SSL client authentication** or **SSL mutual authentication**): + + You can use this method when you cannot control what HTTP headers are sent with each REST call. For example, this is the case when you are using third-party software or systems such as CICS events over HTTP. + +**Note:** You must have {{site.data.reuse.es_name}} version 2019.1.1 or later to use the REST API. In addition, you must have {{site.data.reuse.es_name}} version 2019.4.1 or later to use the REST API with SSL client authentication. + +## Content types + +The following are supported for the value of the `Content-Type` header: + + - `application/octet-stream` + - `text/plain` + - `application/json` + - `text/xml` + - `application/xml` + +For each content type the message body is copied "as is" into the Kafka record value. Both the `application/octet-stream` and `text/plain` types must specify a length header to avoid accidental truncation if the HTTP connection drops prematurely. The payload of a request that uses the `application/json` header must parse as valid JSON. Otherwise, the request will be rejected. + +The {{site.data.reuse.es_name}} REST Producer API also supports the following vendor content types: + + - `vnd.ibm-event-streams.json.v1` as a synonym for `application/json` + - `vnd.ibm-event-streams.binary.v1` as a synonym for `application/octet-stream` + - `vnd.ibm-event-streams.text.v1` as a synonym for `text/plain` + +These content types can be used to pin applications at the version 1 API level. + +## Prerequisites + +To be able to produce to a topic, ensure you have the following available: +- The URL of the {{site.data.reuse.es_name}} REST Producer API endpoint. +- The topic you want to produce to. +- If using a REST Producer API endpoint that requires HTTPS, the {{site.data.reuse.es_name}} certificate. + +To retrieve the full URL for the {{site.data.reuse.es_name}} API endpoint, you can use the Kubernetes command-line tool (`kubectl`) or {{site.data.reuse.es_name}} UI. + +Using the Kubernetes command-line tool (`kubectl`): +1. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to list available {{site.data.reuse.es_name}} REST Producer API endpoints: + + ```shell + kubectl get eventstreams -n -o=jsonpath='{.status.endpoints[?(@.name=="restproducer")].uri}{"\n"}' + ``` + +3. Copy the full URL of the required endpoint from the `HOST/PORT` section of the response. + +Using the UI: +1. {{site.data.reuse.es_ui_login_nonadmin}} +2. Click **Connect to this cluster** on the right. +3. Go to the **Resources** tab. +4. Scroll down to the **Producer endpoint and credentials** section. +5. Click **Copy Producer API endpoint**. + +By default the {{site.data.reuse.es_name}} REST Producer API endpoint requires a HTTPS connection. If this has not been disabled for the endpoint the {{site.data.reuse.es_name}} certificate must be retrieved. You can use the {{site.data.reuse.es_name}} CLI or UI. + +Using the CLI: +1. Ensure you have the {{site.data.reuse.es_name}} CLI [installed](../../installing/post-installation/#installing-the-event-streams-command-line-interface). + +2. {{site.data.reuse.es_cli_init_111}} + If you have more than one {{site.data.reuse.es_name}} instance installed, select the one where the topic you want to produce to is. + + Details of your {{site.data.reuse.es_name}} installation are displayed. +3. Download the server certificate for {{site.data.reuse.es_name}}: + + ```shell + kubectl es certificates --format pem + ``` + + By default, the certificate is written to a file called `es-cert.pem`. + +Using the UI: +1. {{site.data.reuse.es_ui_login_nonadmin}} +2. Click **Connect to this cluster** on the right. +3. Go to the **Resources** tab. +4. Scroll down to the **Certificates** section. +5. In the **PEM certificate** section, click **Download certificate**. + +For information on how to create a topic to produce to, see the information about [creating topics](../../getting-started/creating-topics/). + +### Key and message size limits + +The REST producer API has a configured limit for the key size (default is `4096` bytes) and the message size (default is `65536` bytes). If the request sent has a larger key or message size than the limits set, the request will be rejected. + +You can [configure](../../installing/configuring/) the key and message size limits at the time of [installation](({{ 'installpagedivert' | relative_url }})) or later as described in [modifying](../../administering/modifying-installation/) installation settings. The limits are configured by setting environment variables on the REST Producer component: + +```yaml +spec: + restProducer: + env: + - name: MAX_KEY_SIZE + value: "4096" + - name: MAX_MESSAGE_SIZE + value: "65536" +``` + +**Important:** Do not set the `MAX_MESSAGE_SIZE` to a higher value than the maximum message size that can be received by the Kafka broker or the individual topic (`max.message.bytes`). By default, the maximum message size for Kafka brokers is `1000012` bytes. If the limit is set for an individual topic, then that setting overrides the broker setting. Any message larger than the maximum limit will be rejected by Kafka. + +**Note:** Sending large requests to the REST producer increases latency, as it will take the REST producer longer to process the requests. + +## Producing messages using REST with HTTP authorization + +Ensure you have gathered all the details required to use the producer API, as described in the [prerequisites](#prerequisites). Before producing you must also create authentication credentials. + +To create authentication credentials to use in an HTTP authorization header, you can use the {{site.data.reuse.es_name}} CLI or UI. + +Using the CLI: +1. Ensure you have the {{site.data.reuse.es_name}} CLI [installed](../../installing/post-installation/#installing-the-event-streams-command-line-interface). + +2. {{site.data.reuse.es_cli_init_111}} + If you have more than one {{site.data.reuse.es_name}} instance installed, select the one where the topic you want to produce to is. + + Details of your {{site.data.reuse.es_name}} installation are displayed. +3. Use the `kafka-user-create` command to create a KafkaUser that can produce to your topic: + + ```shell + cloudctl es kafka-user-create --topic --name --producer --auth-type scram-sha-512 + ``` + + {{site.data.reuse.es_cli_kafkauser_note}} + You can also use the {{site.data.reuse.es_name}} UI to generate the Kafka users. +4. Follow the steps in [managing access](../../security/managing-access/#retrieving-credentials-later) to retrieve the SCRAM SHA 512 username and password. + +Using the UI: +1. {{site.data.reuse.es_ui_login_nonadmin}} +2. Click **Connect to this cluster** on the right. +3. Go to the **Resources** tab. +4. Scroll down to the **Producer endpoint and credentials** section, then click **Generate credentials**. +5. Select **SCRAM username and password**, then click **Next**. +6. Fill in a **Credential Name**, this name must be unique. +7. Select **Produce messages, consume messages and create topics and schemas**, then click **Next**. +8. Select **A specific topic** and fill in the topic name, then click **Next**. +9. Click **Next** on the consumer group panel, then **Generate credentials** on the transactional IDs panel using the default settings. +10. Take a copy of either the username and password or Basic authentication token. + +You can use the usual languages for making the API call. For example, to use cURL to produce messages to a topic with the producer API using a Basic authentication header, run the `curl` command as follows: + +```shell +curl -v -X POST -H "Authorization: Basic " -H "Content-Type: text/plain" -H "Accept: application/json" -d 'test message' --cacert es-cert.pem "https:///topics//records" +``` + +Where: +- `` is the Basic authentication token you generated earlier. +- `` is the full URL copied from the `Producer API endpoint` field earlier. Use `http` instead of `https` if the provided Producer API endpoint has TLS disabled. +- `` is the name of the topic you want to produce messages to. +- `--cacert es-cert.pem` can be ommitted if the provided Producer API endpoint has TLS disabled + +To use cURL to produce messages to a topic with the producer API using a SCRAM username and password, run the `curl` command as follows: + +```shell +curl -v -X POST -u : -H "Content-Type: text/plain" -H "Accept: application/json" -d 'test message' --cacert es-cert.pem "https:///topics//records" +``` + +Where: +- `` is the SCRAM username provided when generating credentials. +- `` is the SCRAM password retrieved earlier. +- `` is the full URL copied from the `Producer API endpoint` field earlier. Use `http` instead of `https` if the provided Producer API endpoint has TLS disabled. +- `` is the name of the topic you want to produce messages to. +- `--cacert es-cert.pem` can be omitted if the provided Producer API endpoint has TLS disabled. + +For full details of the API, see the [API reference]({{ 'api' | relative_url }}){:target="_blank"}. + +## Producing messages using REST with Mutual TLS authentication + +Ensure you have gathered all the details required to use the producer API, as described in the [prerequisites](#prerequisites). Before producing you must also create TLS credentials. + +To create authentication credentials to use with Mutual TLS authentication, you can use the {{site.data.reuse.es_name}} CLI or UI. + +Using the CLI: +1. Ensure you have the {{site.data.reuse.es_name}} CLI [installed](../../installing/post-installation/#installing-the-event-streams-command-line-interface). +2. {{site.data.reuse.es_cli_init_111}} + If you have more than one {{site.data.reuse.es_name}} instance installed, select the one where the topic you want to produce to is. + + Details of your {{site.data.reuse.es_name}} installation are displayed. +3. Use the `kafka-user-create` command to create a KafkaUser that can produce to your topic: + + ```shell + cloudctl es kafka-user-create --topic --name --producer --auth-type tls + ``` + + {{site.data.reuse.es_cli_kafkauser_note}} + You can also use the {{site.data.reuse.es_name}} UI to generate the Kafka users. +4. Follow the steps in [managing access](../../security/managing-access/#retrieving-credentials-later) to TLS certificates and keys. + +Using the UI: +1. {{site.data.reuse.es_ui_login_nonadmin}} +2. Click **Connect to this cluster** on the right. +3. Go to the **Resources** tab. +4. Scroll down to the **Producer endpoint and credentials** section, then click **Generate credentials**. +5. Select **Mutual TLS certificate**, then click **Next**. +6. Fill in a **Credential Name**, this name must be unique. +7. Select **Produce messages, consume messages and create topics and schemas**, then click **Next**. +8. Select **A specific topic** and fill in the topic name, then click **Next**. +9. Click **Next** on the consumer group panel, then **Generate credentials** on the transactional IDs panel using the default settings. +10. Click **Download certificates** and unzip the downloaded ZIP archive containing the TLS certificates and keys. + +Each KafkaUser has a related secret that stores all credentials needed for the mutual TLS authentication. These secrets are: + +- `ca.crt` - the client CA certificate +- `user.key` - the client private key +- `user.crt` - the client certificate +- `user.password` - password for accessing the client private key +- `user.p12` - PKCS #12 archive containing the client certificate and private key + +By default, the name of the secret is same as the name of the KafkaUser. For more information about secrets, see the [Strimzi documentation](https://strimzi.io/docs/operators/0.42.0/deploying.html#user_secrets_generated_by_the_user_operator){:target="_blank"}. + +For some systems, for example CICS, you need to download and import the client CA certificate into your truststore. The client CA certificate can be downloaded using the Kubernetes command-line tool (`kubectl`) or the {{site.data.reuse.openshift_short}} web console. + +Using the Kubernetes command-line tool (`kubectl`): +1. {{site.data.reuse.es_ui_login_nonadmin}} +2. Run the following command to view details of the KafkaUser you want the client CA certificate for: + + ```shell + kubectl get ku/ -o jsonpath='{.status.secret}' + ``` + +3. Note down the name of the secret associated with the KafkaUser. +4. Run the following `kubectl` command to get the client CA certificate from the secret found in the previous command: + + ```shell + kubectl get secret -o jsonpath='{.data.ca\.crt}' | base64 -d > ca.crt + ``` + + Where `` is the name of your KafkaUser. + + Using the OpenShift web console: + 1. {{site.data.reuse.openshift_ui_login}} + 2. Expand the **Workloads** dropdown and select **Secrets** to open the **Secrets** dashboard. + 3. Find and click the secret created for your Kafka User. + 4. Go to **Data** section in the **Secret details** which lists the available certificates. + 5. Click on **Reveal values** and get the ca.crt certificate. + +If you are using Java keystores, the client certificate can be imported by using the `keytool -importcert ...` command as described in the [IBM SDK, Java Technology Edition documentation](https://www.ibm.com/docs/en/sdk-java-technology/8?topic=keystore-importcert){:target="_blank"}. + +Some systems require the client certificate and private key to be combined into one PKCS12 file (with extension `.p12` or `.pfx`). A PKCS12 file and associated password file is included in the KafkaUser secret and the ZIP file downloaded from the {{site.data.reuse.es_name}} UI. + +You can use the usual languages for making the API call. Consult the documentation for your system to understand how to specify the client certificate and private key for the outgoing REST calls to {{site.data.reuse.es_name}}. For example, to use cURL to produce messages to a topic with the producer API, run the `curl` command as follows: + +```shell +curl -v -X POST -H "Content-Type: text/plain" -H "Accept: application/json" -d 'test message' --cacert es-cert.pem --key user.key --cert user.crt "https:///topics//records" +``` + +Where: +- `` is the full URL copied from the `Producer API endpoint` field earlier. +- `` is the name of the topic you want to produce messages to. +- `es-cert.pem` is the {{site.data.reuse.es_name}} server certificate downloaded as part of the [prerequisites](#prerequisites) +- `user.key` is the private key of the user downloaded from the UI or read from the KafkaUser secret +- `user.crt` is the user certificate that contains the public key of the user downloaded from the UI or read from the KafkaUser secret + + For example, the steps to configure a CICS URIMAP as an HTTP client is described in the [CICS Transaction Server documentation](https://www.ibm.com/docs/en/cics-ts/5.4?topic=application-creating-urimap-resource-cics-as-http-client){:target="_blank"}. In this case, load the client certificate and private key, together with the {{site.data.reuse.es_name}} server certificate into your RACF key ring. When defining the URIMAP: + +- `Host` is the client authentication API endpoint obtained as part of the [prerequisites](#prerequisites), without the leading `https://` +- `Path` is `/topics//records` +- `Certificate` is the label given to the client certificate when it was loaded into the key ring. + +For full details of the API, see the [API reference]({{ 'api' | relative_url }}){:target="_blank"}. diff --git a/_es_11.5/06-connecting/00a-exposing-services.md b/_es_11.5/06-connecting/00a-exposing-services.md new file mode 100644 index 00000000..f7d31477 --- /dev/null +++ b/_es_11.5/06-connecting/00a-exposing-services.md @@ -0,0 +1,86 @@ +--- +title: "Exposing services for external access" +excerpt: "Learn how to expose a service outside your cluster." +categories: connecting +slug: expose-service +toc: true +--- + +If you want to expose a service associated with {{site.data.reuse.es_name}} outside your Kubernetes cluster, create and apply a custom resource that defines the external listener as described in the following sections. + +**Note:** This topic is about exposing services that are associated with your {{site.data.reuse.es_name}} deployment, such as Kafka Bridge and Kafka Connect. For configuring access to services representing {{site.data.reuse.es_name}} components, see [configuring access](../../installing/configuring/#configuring-access). + +## Prerequisites + +- Ensure the service to be exposed is available and ready (for example, you have [Kafka Bridge enabled](../../installing/configuring/#enabling-and-configuring-kafka-bridge)). +- Ensure the exposing resource is available. For example, the {{site.data.reuse.openshift_short}} uses routes by default, but when using ingress as the connection type, ensure an [ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/){:target="_blank"} is deployed and running in your Kubernetes cluster to enable the ingress resource to work. The SSL passthrough must be enabled in the ingress controller and your ingresses. Refer to your ingress controller documentation for more information. + +**Important:** Exposing a service to external access means any external client will be able to access the service. For example, writing to or reading from Kafka topics through Kafka Bridge, or accessing and changing connector configuration settings in the case of Kafka Connect. This creates a potential security risk due to the unsecured nature of these services. Ensure you consider the risks before exposing these services to external clients. + +## Configuring external access + +To configure the external access: + +1. Extract the Service Name of the service to be exposed. For example, run the following command to list the services: + + ```shell + kubectl get service + ``` + + This will provide a list of service names, for example, `Service Name=MyKafkaBridge` + +2. Create the custom resource YAML file that defines how your service is exposed. + + - For example, if you have an {{site.data.reuse.openshift_short}} cluster, create a `Route` custom resource for the service to expose: + + ```yaml + kind: Route + apiVersion: route.openshift.io/v1 + metadata: + name: + namespace: + spec: + host: -.apps..example.com + to: + kind: Service + name: + weight: 100 + port: + targetPort: rest-api + wildcardPolicy: None + ``` + + Where `` is the name of the OpenShift cluster you are using. If you do not provide a hostname, it is automatically generated when the route custom resource is applied. + + **Note:** In case of Kafka Bridge, ensure you expose the Kafka Bridge service by running: `kubectl expose service `. + + - To expose a service by using ingress and NGINX as the [ingress controller](https://kubernetes.github.io/ingress-nginx/deploy/){:target="_blank"}, create an `Ingress` custom resource for the service to expose: + + ```yaml + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: + spec: + ingressClassName: nginx + rules: + - host: + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: + port: + number: + ``` + + For information about the schema for REST endpoints, see the table in [configuring access](../../installing/configuring/#rest-services-access). + + +3. Apply the custom resource by running the following command: + + ```shell + kubectl apply -f .yaml + ``` diff --git a/_es_11.5/06-connecting/01-kafka-bridge.md b/_es_11.5/06-connecting/01-kafka-bridge.md new file mode 100644 index 00000000..1f71a624 --- /dev/null +++ b/_es_11.5/06-connecting/01-kafka-bridge.md @@ -0,0 +1,347 @@ +--- +title: "Connecting over HTTP with Kafka Bridge" +excerpt: "Use Kafka Bridge to connect to your Event Streams Kafka cluster over HTTP." +categories: connecting +slug: kafka-bridge +toc: true +--- + +Connect client applications to your {{site.data.reuse.es_name}} Kafka cluster over HTTP by making HTTP requests. + +## Overview + +With [Kafka Bridge](https://strimzi.io/docs/bridge/latest/){:target="_blank"}, you can connect client applications to your {{site.data.reuse.es_name}} Kafka cluster over HTTP, providing a standard web API connection to {{site.data.reuse.es_name}} rather than the custom Kafka protocol. + +![Event Streams - Kafka Bridge architecture]({{ 'images' | relative_url }}/architectures/ibm-event-automation-diagrams-es-bridge.svg "Diagram showing the architecture of the Kafka bridge in Event Streams as part of IBM Event Automation.") + +Apache Kafka uses a custom protocol on top of TCP/IP for communication between applications and the Kafka cluster. With Kafka Bridge, clients can communicate with your {{site.data.reuse.es_name}} Kafka cluster over the HTTP/1.1 protocol. You can manage consumers and send and receive records over HTTP. + +## Prerequisites + +Ensure you enable Kafka Bridge for {{site.data.reuse.es_name}} as described in [configuring](../../installing/configuring/#enabling-and-configuring-kafka-bridge). + +## Security + +Authentication and encryption between client applications and Kafka Bridge is not supported. Requests sent from clients to Kafka Bridge are not encrypted and must use HTTP (not HTTPS), and are sent without authentication. + +You can configure TLS (`tls`) or SASL-based (`scram-sha-512`) user authentication between Kafka Bridge and your {{site.data.reuse.es_name}} Kafka cluster. + +To configure authentication between Kafka Bridge and your Kafka cluster, use the `KafkaBridge` custom resource. + +- The following example includes an {{site.data.reuse.es_name}} cluster that requires TLS authentication for user access, and the user `` was [created](../../security/managing-access/#managing-access-to-kafka-resources) for Kafka Bridge earlier. In addition, it includes TLS authentication for the connection between the {{site.data.reuse.es_name}} instance (called `development`) and the Kafka Bridge. + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: KafkaBridge +metadata: + name: my-bridge + namespace: + labels: + eventstreams.ibm.com/cluster: + backup.eventstreams.ibm.com/component: kafkabridge +spec: + replicas: 1 + bootstrapServers: '-kafka-bootstrap:9093' + http: + port: 8080 + authentication: + type: tls + certificateAndKey: + certificate: user.crt + key: user.key + secretName: + tls: + trustedCertificates: + - certificate: ca.crt + secretName: -cluster-ca-cert + template: + bridgeContainer: + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - s390x + - ppc64le +``` + +- The following example includes an {{site.data.reuse.es_name}} cluster that requires SCRAM-SHA-512 authentication for user access, and the user `` was [created](../../security/managing-access/#managing-access-to-kafka-resources) for Kafka Bridge earlier. In addition, it includes TLS authentication for the connection between the {{site.data.reuse.es_name}} instance (called `development`) and the Kafka Bridge. + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: KafkaBridge +metadata: + name: my-bridge + namespace: + labels: + eventstreams.ibm.com/cluster: + backup.eventstreams.ibm.com/component: kafkabridge +spec: + replicas: 1 + bootstrapServers: '-kafka-bootstrap:9093' + http: + port: 8080 + authentication: + type: scram-sha-512 + username: + passwordSecret: + secretName: + password: password + tls: + trustedCertificates: + - certificate: ca.crt + secretName: -cluster-ca-cert + template: + bridgeContainer: + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - s390x + - ppc64le +``` + + +## Client access + +### Internal access + +Client applications running within the same Kubernetes cluster as Kafka Bridge can access Kafka Bridge on the port configured in the `KafkaBridge` custom resource (the default is `8080`). + +### External access + +Client applications running outside of the Kubernetes cluster can access the Kafka Bridge through an ingress or an OpenShift route, as described in [enabling Kafka Bridge](../../installing/configuring/#enabling-and-configuring-kafka-bridge). + +## Using Kafka Bridge + + +After enabling and configuring the Kafka Bridge, use the bridge to connect applications to your Kafka cluster over HTTP. + + + +### Available HTTP requests + +With Kafka Bridge, you can make the following HTTP requests to your Kafka cluster: +- Send messages to a topic. +- Retrieve messages from topics. +- Retrieve a list of partitions for a topic. +- Create and delete consumers. +- Subscribe consumers to topics, so that they start receiving messages from those topics. +- Retrieve a list of topics that a consumer is subscribed to. +- Unsubscribe consumers from topics. +- Assign partitions to consumers. +- Commit a list of consumer offsets. +- Seek on a partition, so that a consumer starts receiving messages from the first or last offset position, or a given offset position. + +The related request operations are the following: +- `GET /` +- `POST /consumers/{groupid}` +- `DELETE /consumers/{groupid}/instances/{name}` +- `POST /consumers/{groupid}/instances/{name}/assignments` +- `POST /consumers/{groupid}/instances/{name}/offsets` +- `POST /consumers/{groupid}/instances/{name}/positions` +- `POST /consumers/{groupid}/instances/{name}/positions/beginning` +- `POST /consumers/{groupid}/instances/{name}/positions/end` +- `GET /consumers/{groupid}/instances/{name}/records` +- `POST /consumers/{groupid}/instances/{name}/subscription` +- `GET /consumers/{groupid}/instances/{name}/subscription` +- `DELETE /consumers/{groupid}/instances/{name}/subscription` +- `GET /healthy` +- `GET /openapi` +- `GET /ready` +- `GET /topics` +- `POST /topics/{topicname}` +- `GET /topics/{topicname}` +- `GET /topics/{topicname}/partitions` +- `POST /topics/{topicname}/partitions/{partitionid}` +- `GET /topics/{topicname}/partitions/{partitionid}` +- `GET /topics/{topicname}/partitions/{partitionid}/offsets` + +For more detailed information about the request paths, see the [Strimzi documentation](https://strimzi.io/docs/bridge/latest/#_paths){:target="_blank"}. + +### Content types + +The following are supported for the value of the `Content-Type` header. + +- For consumer operations, `POST` requests must provide the following `Content-Type` header: + ```shell + Content-Type: application/vnd.kafka.v2+json + ``` +- For producer operations, `POST` requests must provide `Content-Type` headers specifying the embedded data format of the messages produced (JSON or binary: + + Embedded data format | `Content-Type` header + -------------------------|----------------------------------------------------- + JSON | `Content-Type: application/vnd.kafka.json.v2+json` + Binary | `Content-Type: application/vnd.kafka.binary.v2+json` + + +### Producing + +After setting up Kafka Bridge, you can produce to specified topics over HTTP. To produce to a topic, [create a topic](../../getting-started/creating-topics/), and run the following curl command. + +**Note:** If automatic topic creation is enabled (set by `auto.create.topics.enable` in the broker configuration, default is `true`), then you can specify a new topic in the following command and it will be created before messages are written to the topic. + +```shell +curl -X POST \ + http://.apps...com/topics/ \ + -H 'content-type: application/vnd.kafka.json.v2+json' \ + -d '{ + "records": [ + { + "key": "key-1", + "value": "value-1" + }, + { + "key": "key-2", + "value": "value-2" + } + ] +}' +``` + +For example, to produce two messages about temperature readings to a topic called `my-topic`, you can use the following command: + +```shell +curl -X POST \ + http://my-bridge-route-es-kafka-bridge.apps.example.com/topics/my-topic \ + -H 'content-type: application/vnd.kafka.json.v2+json' \ + -d '{ + "records": [ + { + "key": "temperature-1", + "value": "20" + }, + { + "key": "temperature-2", + "value": "25" + } + ] +}' + +``` + +If the request to produce to the topic is successful, the Kafka Bridge returns an HTTP status code `200 OK` and a JSON payload describing for each message the partition that the message was sent to and the offset the messages are written to. + +```json +# ... +{ + "offsets":[ + { + "partition":0, + "offset":0 + }, + { + "partition":2, + "offset":0 + } + ] +} +``` + +### Consuming + +To consume messages over HTTP with Kafka Bridge: +1. [Create a Kafka Bridge consumer](#creating-a-kafka-bridge-consumer) +2. [Subscribe to a topic](#subscribing-to-topics) +3. [Retrieve messages from the topic](#retrieving-messages-from-topics) + +#### Creating a Kafka Bridge consumer + +To interact with your Kafka cluster, the Kafka Bridge requires a consumer. To create the Kafka Bridge [consumer endpoint](https://strimzi.io/docs/bridge/latest/#_createconsumer){:target="_blank"}, create a consumer within a consumer group. For example, the following command creates a Kafka Bridge consumer called `my-consumer` in a new consumer group called `my-group`: + +```shell +curl -X POST http://my-bridge-route-es-kafka-bridge.apps.example.com/consumers/my-group \ +-H 'content-type: application/vnd.kafka.json.v2+json' \ +-d '{ + "name": "my-consumer", + "format": "json", + "auto.offset.reset": "earliest", + "enable.auto.commit": false +}' +``` + +If the request is successful, the Kafka Bridge returns an HTTP status code `200 OK` and a JSON payload containing the consumer ID (`instance_id`) and the base URL (`base_uri`). The base URL is used by the HTTP client to interact with the consumer and receive messages from topics. The following is an example response: + +```json +{ + "instance_id":"my-consumer", + "base_uri":"http://my-bridge-bridge-service:80/consumers/my-group/instances/my-consumer" +} +``` + +#### Subscribing to topics + +After creating a Kafka Bridge consumer, you can subscribe the Kafka Bridge consumer to topics by creating a [subscription endpoint](https://strimzi.io/docs/bridge/latest/#_subscribe){:target="_blank"}. For example, the following command subscribes the consumer to the topic called `my-topic`: + +```shell +curl -X POST http://my-bridge-route-es-kafka-bridge.apps.example.com/consumers/my-group/instances/my-consumer/subscription \ +-H 'content-type: application/vnd.kafka.json.v2+json' \ +-d '{ + "topics": [ + "my-topic" + ] +}' +``` +If the request is successful, the Kafka Bridge returns an HTTP status code `200 OK` with an empty body. + +After subscribing, the consumer receives all messages that are produced to the topic. + +#### Retrieving messages from topics + +After subscribing a Kafka Bridge consumer to a topic, your client applications can retrieve the messages on the topic from the Kafka Bridge consumer. To retrieve the latest messages, request data from the [records endpoint](https://strimzi.io/docs/bridge/latest/#_poll){:target="_blank"}. The following is an example command to retrieve messages from `my-topic`: + +```shell +curl -X GET http://my-bridge-route-es-kafka-bridge.apps.example.com/consumers/my-group/instances/my-consumer/records \ +-H 'accept: application/vnd.kafka.json.v2+json' +``` + +If the request is successful, the Kafka Bridge returns an HTTP status code `200 OK` and the JSON body containing the messages from the topic. Messages are retrieved from the latest offset by default. + +**Note:** If you receive an empty response, [produce more records](#producing) to the consumer, and then try retrieving messages again. + +In production, HTTP clients can call this endpoint repeatedly (in a loop), for example: + +```shell +while true; do sleep 1; curl -X GET http://my-bridge-route-es-kafka-bridge.apps.example.com/consumers/my-group/instances/my-consumer/records \ +-H 'accept: application/vnd.kafka.json.v2+json'; echo ; done +``` + +#### Deleting a Kafka Bridge consumer + +If you no longer need a Kafka Bridge consumer, delete it to free up resources on the bridge. + +Use the [delete consumer endpoint](https://strimzi.io/docs/bridge/latest/#_deleteconsumer){:target="_blank"} to delete a consumer instance. For example, to delete the consumer [created earlier](#creating-a-kafka-bridge-consumer), run the following command: + +```shell +curl -X DELETE http://my-bridge-bridge-service:80/consumers/my-group/instances/my-consumer/ +``` + + +For more information about using Kafka Bridge to produce to and consume from your Kafka cluster, including committing offsets, see the following [blog post](https://strimzi.io/blog/2019/11/05/exposing-http-bridge/){:target="_blank"}. diff --git a/_es_11.5/06-connecting/02-connectors.md b/_es_11.5/06-connecting/02-connectors.md new file mode 100644 index 00000000..a9edac43 --- /dev/null +++ b/_es_11.5/06-connecting/02-connectors.md @@ -0,0 +1,88 @@ +--- +title: "Kafka Connect and connectors" +excerpt: "Read about Kafka Connect and connectors." +categories: connecting +slug: connectors +toc: true +--- + +You can integrate external systems with {{site.data.reuse.es_name}} by using the Kafka Connect framework and connectors. + +![Event Streams connectors architecture]({{ 'images' | relative_url }}/architectures/ibm-event-automation-diagrams-es-kafkaconnect.svg "Diagram showing the architecture of the Event Streams connectors as part of IBM Event Automation.") + + +## What is Kafka Connect? + +When connecting Apache Kafka and other systems, the technology of choice is the [Kafka Connect framework](https://kafka.apache.org/37/documentation/#connect){:target="_blank"}. + +Use Kafka Connect to reliably move large amounts of data between your Kafka cluster and external systems. For example, it can ingest data from sources such as databases and make the data available for stream processing. + + +### Source and sink connectors + +Kafka Connect uses connectors for moving data into and out of Kafka. **Source connectors** import data from external systems into Kafka topics, and **sink connectors** export data from Kafka topics into external systems. A wide range of connectors exists, some of which are commercially supported. In addition, you can write your own connectors. + +A number of source and sink connectors are available to use with {{site.data.reuse.es_name}}. See the [connector catalog](#connector-catalog) section for more information. + +![Kafka Connect: source and sink connectors]({{ 'images' | relative_url }}/Kafka_Connect_Basics_2.svg "Diagram showing a representation of how source and sink connectors connect Event Streams and external systems.") + +![Kafka Connect: introduction]({{ 'images' | relative_url }}/Kafka_Connect_Basics_1.svg "Diagram showing a representation of external systems connecting to Event Streams.") + +### Workers + +Kafka Connect connectors run inside a Java process called a worker. Kafka Connect can run in either stand-alone or distributed mode. Stand-alone mode is intended for testing and temporary connections between systems, and all work is performed in a single process. Distributed mode is more appropriate for production use, as it benefits from additional features such as automatic balancing of work, dynamic scaling up or down, and fault tolerance. + +![Kafka Connect: workers]({{ 'images' | relative_url }}/Kafka_Connect_Basics_3.svg "Diagram showing a representation of how workers and tasks are distributed in stand-alone and distributed modes.") + +When you run Kafka Connect with a stand-alone worker, there are two configuration files: + +* The worker configuration file contains the properties that are required to connect to Kafka. This is where you provide the details for connecting to Kafka. +* The connector configuration file contains the properties that are required for the connector. This is where you provide the details for connecting to the external systems (for example, IBM MQ). + +When you run Kafka Connect with the distributed worker, you still use a worker configuration file but the connector configuration is supplied by using a REST API. Refer to the [Kafka Connect documentation](https://kafka.apache.org/documentation/#connect){:target="_blank"} for more details about the distributed worker. + +For getting started and problem diagnosis, the simplest setup is to run only one connector in each stand-alone worker. Kafka Connect workers print a lot of information and it's easier to understand if the messages from multiple connectors are not interleaved. + + +### Kafka Connect topics + +When running in distributed mode, Kafka Connect uses three topics to store configuration, current offsets and status. Kafka Connect can create these topics automatically as it is started by the {{site.data.reuse.es_name}} operator. By default, the topics are: + +- **connect-configs**: This topic stores the connector and task configurations. +- **connect-offsets**: This topic stores offsets for Kafka Connect. +- **connect-status**: This topic stores status updates of connectors and tasks. + +**Note:** If you want to run multiple Kafka Connect environments on the same cluster, you can override the default names of the topics in the configuration. + +### Authentication and authorization + +Kafka Connect uses an Apache Kafka client just like a regular application, and the usual authentication and authorization rules apply. + +Kafka Connect will need authorization to: + +* Produce and consume to the internal Kafka Connect topics and, if you want the topics to be created automatically, to create these topics. +* Produce to the target topics of any [source connectors](#source-and-sink-connectors) that you are using. +* Consume from the source topics of any [sink connectors](#source-and-sink-connectors) that you are using. + +**Note:** For more information about authentication and the credentials and certificates required, see the information about [managing access](../../security/managing-access/). + + +## Connector catalog + +The connector catalog contains a list of connectors that are supported either by IBM or the community. + +Community supported connectors are supported through the community that maintains them. IBM supported connectors are fully supported as part of the official {{site.data.reuse.es_name}} support entitlement if you have a license for {{site.data.reuse.ea_long}} or {{site.data.reuse.cp4i}}. + +See the [connector catalog]({{ 'connectors' | relative_url }}){:target="_blank"} for a list of connectors that work with {{site.data.reuse.es_name}}. + +![Kafka Connect: connector catalog]({{ 'images' | relative_url }}/Kafka_Connect_Basics_4.svg "Diagram showing how you can choose connectors from the catalog to facilitate communication between Event Streams and external systems.") + +## Setting up connectors + +{{site.data.reuse.es_name}} provides help with setting up your Kafka Connect environment, adding connectors to that environment, and starting the connectors. See the instructions about [setting up and running connectors](../setting-up-connectors/). + +## Connectors for IBM MQ + +Connectors are available for copying data between IBM MQ and {{site.data.reuse.es_name}}. There is a {{site.data.reuse.kafka-connect-mq-source-short}} for copying data from IBM MQ into {{site.data.reuse.es_name}} or Apache Kafka, and a {{site.data.reuse.kafka-connect-mq-sink-short}} for copying data from {{site.data.reuse.es_name}} or Apache Kafka into IBM MQ. + +For more information about MQ connectors, see the topic about [connecting to IBM MQ](../mq/). diff --git a/_es_11.5/06-connecting/02-setting-up-connectors.md b/_es_11.5/06-connecting/02-setting-up-connectors.md new file mode 100644 index 00000000..b9e12213 --- /dev/null +++ b/_es_11.5/06-connecting/02-setting-up-connectors.md @@ -0,0 +1,472 @@ +--- +title: "Setting up and running connectors" +excerpt: "Event Streams helps you set up a Kafka Connect environment, add connectors to it, and run the connectors to help integrate external systems." +categories: connecting +slug: setting-up-connectors +toc: true +--- + +{{site.data.reuse.es_name}} helps you integrate Kafka with external systems by setting up and managing [Kafka Connect and connectors](../connectors/#what-is-kafka-connect) as custom resources. + +## Using Kafka Connect + +To start Kafka Connect, define a `KafkaConnect` custom resource and configure it for your environment. A `KafkaConnect` custom resource represents a Kafka Connect distributed cluster and contains information about your Kafka cluster, the connectors that you want to use, and how you want to distribute the workload. Each connector is started and managed by configuring and creating a `KafkaConnector` custom resource. + +Complete the following sections to set up your Kafka Connect environment. + +## Configure connection to Kafka + +Configure the `KafkaConnect` custom resource with information about your Kafka cluster: + +- `bootstrapservers`: Kafka bootstrap servers to connect to. Use a comma-separated list for `:` pairs. For example: + + ```yaml + bootstrapServers: 'my-es-kafka-bootstrap-es.hostname:443' + ``` + + To get the `bootstrapServers` for your {{site.data.reuse.es_name}} instance, run the following command: + + ```shell + kubectl get eventstreams -n -o=jsonpath='{.status.kafkaListeners[?(@.name=="")].bootstrapServers}{"\n"}' + ``` + + Where `` is the name of your external Kafka listener. + +- `authentication`: If {{site.data.reuse.es_name}} instance is secured with authentication, configure the `spec.authentication` field of `KafkaConnect` custom resource with credentials, which allow the connectors to read and write to Kafka Connect topics. For more information, see [authentication and authorization](../connectors/#authentication-and-authorization). + + If {{site.data.reuse.es_name}} instance is secured with SCRAM authentication, use the following configuration: + + ```yaml + authentication: + type: scram-sha-512 + username: my-user + passwordSecret: + password: password + secretName: my-kafka-user + ``` + + If {{site.data.reuse.es_name}} instance is secured with Transport Layer Security (TLS) authentication, use the following configuration: + + ```yaml + authentication: + type: tls + certificateAndKey: + certificate: user.crt + secretName: kafka-connect-user + key: user.key + ``` + +- `tls`: If {{site.data.reuse.es_name}} instance is secured with TLS encryption, configure the `spec.tls` field of `KafkaConnect` custom resource with reference to required certificates for connecting securely to Kafka listeners. + + ```yaml + tls: + trustedCertificates: + - secretName: -cluster-ca-cert + certificate: ca.crt + ``` + +## Add connectors you want to use + +Prepare Kafka Connect for connecting to external systems by adding the required connectors using one of the methods described below: + +- [Use the {{site.data.reuse.es_name}} operator](#use-the-sitedatareusees_name-operator) to add the connectors by specifying the connectors within a `KafkaConnect` custom resource. The {{site.data.reuse.es_name}} operator will create a container image with Kafka Connect and all the specified connector artifacts, and use it to start Kafka Connect. + +- [Manually add required connectors](#manually-add-required-connectors) and Kafka Connect to a container image and specify the image in `KafkaConnect` custom resource. The {{site.data.reuse.es_name}} operator will use the specified image to start Kafka Connect. + +### Use the {{site.data.reuse.es_name}} operator + +The {{site.data.reuse.es_name}} operator can create a container image with Kafka Connect and all the specified connector artifacts, tag and push the image to the specified container registry, and use the built image to start Kafka Connect. + +To build the connector image by using the {{site.data.reuse.es_name}} operator, configure the `spec.build` section of `KafkaConnect` custom resource as follows: + +- `spec.build.plugins`: Specify the connectors you want to use in `spec.build.plugins` section. + + Each connector can be specified as a plugin with a name and a list of artifacts that represent the connector and any other dependencies you want to use with that connector. + + Each artifact has a type, and additional fields that define how the artifact can be downloaded and used. {{site.data. reuse.es_name}} supports the following types of artifacts: + + - JAR files, which are downloaded and used directly + - TGZ and ZIP archives, which are downloaded and unpacked + - Maven artifacts, which uses Maven coordinates + - Other artifacts, which are downloaded and used directly + + If the artifact is of type `jar`, `tgz`, `zip`, or other, then the plugin can be defined as follows: + + ```yaml + plugins: + - name: plugin-1 + artifacts: + - type: jar + url: + sha512sum: + ``` + + Where: + + - `type` is the file format for the connector image that you will download. + - `` defines the location accessible from your cluster to download the connector from. + - `` is the checksum that you use to verify that the downloaded artifact is correct before it adds it to your Kafka Connect environment. + + If the artifact type is `maven`, then the plugin can be defined as follows: + + ```yaml + plugins: + - name: mq-sink + artifacts: + - type: maven + group: + artifact: + version: + ``` + + **Note:** If you encounter issues while retrieving the maven artifacts, consider encoding the configuration values. For example, to retrieve the `com.ibm.mq.allclient` artifact, configure your value as `artifact: com%2Eibm%2Emq%2Eallclient`. + +- `spec.build.output`: Configure this section to specify the output that you want from the `KafkaConnect` build process. + + For example: + + ```yaml + output: + image: my-image-registry.my-kafka-connect-image:latest + type: docker + pushSecret: my-registry-credentials + ``` + + Where: + + - `type` can be `docker` if you want to create a container image or `imagestream` if you are using `OpenShift ImageStream`. + - `image` is the full name of the new container image including registry address, port number (if listening to a non-standard port), and tag. For example, `my-registry.io/my-connect-cluster-image:my-tag` where: + - `my-registry.io/my-connect-cluster-image` is the address of the registry. + - `my-tag` is the tag of the new container image. + - `pushSecret` if the image registry is secured, you can optionally provide the name of the secret that contains credentials with write permission. + +If the build process needs to pull or push images from or to a secured container registry, specify the secret containing the credentials in the `spec.template` section. For example, to retrieve the {{site.data.reuse.es_name}} Kafka Connect image, which is used as the base image for the build, provide your `ibm-entitlement-key` secret: + +- If running on OpenShift, specify the secret in `spec.template.buildConfig.pullSecret` section: + + ```yaml + template: + buildConfig: + pullSecret: ibm-entitlement-key + ``` + +- If running on other Kubernetes platforms, specify the secret in `spec.template.builPod.imagePullSecrets` section: + + ```yaml + template: + buildPod: + imagePullSecrets: + - name: ibm-entitlement-key + ``` + +- To provide the secret for pulling any of the images that are used by the specific pod where Kafka Connect is running, specify the secret in `spec.template.pod.imagePullSecrets` section:. + + ```yaml + template: + pod: + imagePullSecrets: + - name: default-dockercfg-abcde + ``` + + **Important:** The secrets that are referenced in the `KafkaConnect` custom resource must be present in the same namespace as the `KafkaConnect` instance. + +#### Rebuild the Kafka Connect image + +Rebuild the Kafka Connect image regularly to ensure that your Kafka Connect environment is up-to-date with changes to Kafka Connect and any new releases of connectors. + +When you change the image (`spec.image`) or the connector plugin artifacts configuration (`spec.build.plugins`) in the `KafkaConnect` custom resource, container image is built automatically. + +To pull an upgraded base image or to download the latest connector plugin artifacts without changing the `KafkaConnect` resource, you can trigger a rebuild of the container image that is used by your Kafka Connect cluster by applying the following annotation to the `StrimziPodSet` resource named `-connect`: + +```shell +eventstreams.ibm.com/force-rebuild: true +``` + +### Manually add required connectors + +You can create a container image with Kafka Connect and all the required connector artifacts yourself and use it to start Kafka Connect. This approach is useful when the required connectors cannot be included using the operator, for example, when authentication is required to access to connector artifacts. + +- {{site.data.reuse.es_name}} Kafka image is a convenient starting point as it includes everything you need to start Kafka Connect. You can prepare a directory containing all the required connectors and use the sample `Dockerfile` to add them to the {{site.data.reuse.es_name}} Kafka image: + + ```yaml + FROM cp.icr.io/cp/ibm-eventstreams-kafka: + COPY ./my-plugins/ /opt/kafka/plugins/ + USER 1001 + ``` + + Where: + - `` is the version of {{site.data.reuse.es_name}} that you are using. + - `my-plugins` is the folder containing all connector artifacts. + + **Note:** Both components must be in a single directory. For example, the following snippet shows the directory structure: + + ```shell + +-- KafkaConnectComponents + | +-- Dockerfile + | +-- my-plugins + ``` + +- If your connector consists of just a single JAR file, you can copy the JAR file directly into the `my-plugins` directory. If your connector consists of multiple JAR files or requires additional dependencies, create a directory for the connector inside the `my-plugins` directory and copy all the JAR files of your connector into `my-plugins` directory. For example, the following snippet shows the directory structure with three connectors: + + ```shell + +-- my-plugins + | +-- connector1.jar + | +-- connector2 + | | +-- connector2.jar + | | +-- connector2-lib.jar + | +-- connector3.jar + ``` + +- Run the following commands to build the container image and push it to an image registry that is accessible to your {{site.data.reuse.es_name}} instance: + + ```bash + docker build -t /: + docker push /: + ``` + + **Note:** You might need to log in to the [IBM Container software library](https://myibm.ibm.com/products-services/containerlibrary){:target="_blank"} before building the image to allow the base image that is specified in the `Dockerfile` to be pulled successfully. + + +- Specify the image in the `spec.image` field of `KafkaConnect` resource to start Kafka Connect with the image you have built with your connectors. + +#### Rebuild the Kafka Connect image + +Rebuild the Kafka Connect image regularly with a new unique tag and update the `KafkaConnect` resource to use the new image. This ensures that your Kafka Connect environment is up-to-date with changes to Kafka Connect and any new releases of connectors. + +## Configure workers + +{{site.data.reuse.es_name}} runs Kafka Connect in distributed mode, distributing data streaming tasks across one or more worker pods. + +- You can configure the number of workers in the `replicas` section of the `KafkaConnect` resource: + + ```yaml + replicas: 1 + ``` + +- Worker configuration can be optionally specified in the `spec.config` section of `KafkaConnect` resource. For example: + + ```yaml + config: + group.id: connect-cluster + offset.storage.topic: connect-cluster-offsets + config.storage.topic: connect-cluster-configs + status.storage.topic: connect-cluster-status + config.storage.replication.factor: 1 + offset.storage.replication.factor: 1 + status.storage.replication.factor: 1 + ``` + + Where: + + - `group.id` is the unique name used by the Kafka Connect cluster to identify the workers in the cluster. + - `offset.storage.topic` is the topic that Kafka Connect uses to store source connector offsets. + - `config.storage.topic` is the topic that Kafka Connect uses to store connector configurations. + - `status.storage.topic` is the topic that Kafka Connect uses to store the status of connectors. + + **Note:** Set the following factors to `1` if you have less than three brokers in your {{site.data.reuse.es_name}} cluster: + + - `config.storage.replication.factor` + - `offset.storage.replication.factor` + - `status.storage.replication.factor` + +## Starting Kafka Connect + +- Start Kafka Connect by creating the `KafkaConnect` custom resource with the required configuration. For example, if you are using the MQ source and sink connectors, the `KafkaConnect` custom resource might be similar to the following YAML: + + ```yaml + apiVersion: eventstreams.ibm.com/v1beta2 + kind: KafkaConnect + metadata: + name: mq-connectors + namespace: es + annotations: + eventstreams.ibm.com/use-connector-resources: true + labels: + backup.eventstreams.ibm.com/component: kafkaconnect + spec: + authentication: + certificateAndKey: + certificate: user.crt + key: user.key + secretName: my-kafka-user + type: tls + bootstrapServers: mtls-listener.my-cluster:443 + build: + output: + image: my-image-registry.my-kafka-connect-image:latest + type: docker + plugins: + - artifacts: + - type: jar + url: https://github.com/ibm-messaging/kafka-connect-mq-source/releases/download/v2.1.0/kafka-connect-mq-source-2.1.0-jar-with-dependencies.jar + name: mq-source + - artifacts: + - type: jar + url: https://github.com/ibm-messaging/kafka-connect-mq-sink/releases/download/v2.2.0/kafka-connect-mq-sink-2.2.0-jar-with-dependencies.jar + name: mq-sink + template: + buildConfig: + pullSecret: ibm-entitlement-key + pod: + imagePullSecrets: + - name: default-dockercfg-abcde + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - s390x + - ppc64le + connectContainer: + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + tls: + trustedCertificates: + - certificate: ca.crt + secretName: -cluster-ca-cert + ``` + +- {{site.data.reuse.es_name}} operator will populate the `status` section of the `KafkaConect` custom resource. Use the following command to verify that your Kafka Connect cluster is running and connectors configured are available for use: + + ```shell + kubectl describe kafkaconnect my-connect-cluster + ``` + + The following snippet is an example output for the previous command: + + ```output + Status: + Conditions: + Last Transition Time: 2024-06-25T07:56:40.943007974Z + Status: True + Type: Ready + Connector Plugins: + Class: com.ibm.eventstreams.connect.mqsource.MQSourceConnector + Type: source + Version: 1.3.2 + Class: com.ibm.eventstreams.connect.mqsink.MQSinkConnector + Type: sink + Version 1.5.0 + ``` + +**Note:** If Kafka Connect fails to connect to Kafka with timeout errors, then ensure that all the connection details are correct. If the problem persists, try duplicating the following connection properties in your `KafkaConnect` custom resource, adding the `producer` prefix for source connectors, the `consumer` prefix for sink connectors, or both if both sink and source connectors are in use. + +| Connection property | Required for TLS | Required for SCRAM | Source Connector property | Sink Connector Property | +| ------------------------- | ---------------- | ------------------ | ---------------------------------- | ----------------------------------- | +| `bootstrap.servers` | Yes | Yes | `producer.bootstrap.servers` | `consumer.bootstrap.server` | +| `ssl.protocol` | Yes | No | `producer.ssl.protocol` | `consumer.ssl.protocol` | +| `ssl.truststore.location` | Yes | No | `producer.ssl.truststore.location` | `consumer.ssl.truststore.location` | +| `ssl.truststore.password` | Yes | No | `producer.ssl.truststore.password` | `consumer.ssl.truststore.password` | +| `ssl.truststore.type` | Yes | No | `producer.ssl.truststore.type` | `consumer.ssl.truststore.type` | +| `security.protocol` | No | Yes | `producer.security.protocol` | `consumer.security.protocol` | +| `sasl.mechanism` | No | Yes | `producer.sasl.mechanism` | `consumer.sasl.mechanism` | +| `sasl.jaas.config` | No | Yes | `producer.sasl.jaas.config` | `consumer.sasl.jaas.config` | + + +These values can also be set on a per connector level using the `producer.override` and `consumer.override` prefixes. + +## Set up a Kafka connector + +Set up a connector by defining a `KafkaConnector` custom resource with the required connector configuration. + +**Note:** To use `KafkaConnector` resources for managing each connector rather than using Kafka Connect REST API directly, set `eventstreams.ibm.com/use-connector-resources` to `true` in the `metadata.annotations` section of the `KafkaConnect` custom resource. + +### Configure the connector + +- Kafka Connect cluster: Add the label `eventstreams.ibm.com/cluster: ` to `KafkaConnector` custom resource to specify the Kafka Connect cluster where the connector must be started. The value of this label must be set to the name of the corresponding Kafka Connect instance. + + +- `class`: Specifies the complete class name for starting the connector. For example, to set up a MQ sink connector, the name of the connector class is as follows: + + ```yaml + spec: + class: com.ibm.eventstreams.connect.mqsink.MQSinkConnector + ``` + +- `tasksMax`: Specify the maximum number of tasks that must be used to run this connector. + +- `autorestart`: Specify if the Kafka Connect must automatically restart the connector to recover from failures, and optionally specify the maximum number of attempts that must be made before stopping the connector. + + ```yaml + autoRestart: + enabled: true + maxRestarts: 10 + ``` + +- `config`: Each connector documents the supported configuration options that allow users to specify details about the external system and control the connector behavior during the data transfer. These configuration properties can be specified as a set of key-value pairs in the `spec.config` section of the `KafkaConnector` custom resource. For example, if you are trying to connect to a database by using this connector, then the configurations might include parameters such as the database URL, credentials to connect to the database, and table names. See the documentation of your connector to view the full list of supported configuration properties. + +- `state`: Optionally, specify the state you want the connector to be in. Valid states are: + - To start or resume the connector: `running` (default) + - To pause the connector: `paused` + - To stop the connector: `stopped` + +### Start the connector + +Start the connector by creating `KafkaConnector` custom resource with the required configuration. For example, if you are using the MQ source connector, the `KafkaConnector` custom resource might be similar to the following snippet: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: KafkaConnector +metadata: + name: mq-source + labels: + # The eventstreams.ibm.com/cluster label identifies the KafkaConnect instance + # in which to create this connector. That KafkaConnect instance + # must have the eventstreams.ibm.com/use-connector-resources annotation + # set to true. + eventstreams.ibm.com/cluster: + backup.eventstreams.ibm.com/component: kafkaconnector +spec: + class: com.ibm.eventstreams.connect.mqsource.MQSourceConnector + tasksMax: 1 + config: + topic: TSOURCE + mq.queue.manager: QM1 + mq.connection.name.list: localhost(1414) + mq.channel.name: MYSVRCONN + mq.queue: MYQSOURCE + mq.user.name: alice + mq.password: passw0rd + key.converter: org.apache.kafka.connect.storage.StringConverter + value.converter: org.apache.kafka.connect.storage.StringConverter + mq.record.builder: com.ibm.eventstreams.connect.mqsource.builders.DefaultRecordBuilder +``` + +{{site.data.reuse.es_name}} operator will populate the `status` section of the `KafkaConnector` resource. Use the following command to verify that your connector is running as expected: + +```shell +kubectl describe kafkaconnector +``` + +This command provides the complete description of the connector that you created. You can verify the current status of the connector from the `status` section. For example: + +```output +Status: + Conditions: + Last Transition Time: 2024-07-13T07:56:40.943007974Z + Status: True + Type: Ready + Connector Status: + Connector: + State: RUNNING + worker_id: mq-connectors-connect-0.mq-connectors-connect-0.es.svc:8083 + Name: mq-sink + Tasks: + Id: 0 + State: RUNNING + worker_id: mq-connectors-connect-0.mq-connectors-connect-0.es.svc:8083 + Type: sink + Observerd Generation: 1 + Tasks Max: 1 +``` + diff --git a/_es_11.5/06-connecting/03-connecting-mq.md b/_es_11.5/06-connecting/03-connecting-mq.md new file mode 100644 index 00000000..8507c181 --- /dev/null +++ b/_es_11.5/06-connecting/03-connecting-mq.md @@ -0,0 +1,109 @@ +--- +title: "Connecting to IBM MQ" +excerpt: "Connecting to MQ." +categories: connecting +slug: mq +toc: true +--- + +You can set up connections between IBM MQ and Apache Kafka or {{site.data.reuse.es_name}} systems. + + +## Available connectors + +Connectors are available for copying data in both directions. + + - [{{site.data.reuse.kafka-connect-mq-source}}](../mq/source/): + + You can use the {{site.data.reuse.kafka-connect-mq-source-short}} to copy data from IBM MQ into {{site.data.reuse.es_name}} or Apache Kafka. The connector copies messages from a source MQ queue to a target Kafka topic. + + **Note:** The [IBM MQ source connector]({{ 'connectors/kc-source-ibm-mq/installation' | relative_url}}), offers [exactly-once delivery](../mq/source/#exactly-once-message-delivery-semantics-in-ibm-mq-source-connector) of data from IBM MQ into {{site.data.reuse.es_name}} or Apache Kafka. + - [{{site.data.reuse.kafka-connect-mq-sink}}](../mq/sink/): + + You can use the {{site.data.reuse.kafka-connect-mq-sink-short}} to copy data from {{site.data.reuse.es_name}} or Apache Kafka into IBM MQ. The connector copies messages from a Kafka topic into a MQ queue. + + **Note:** The [IBM MQ sink connector]({{ 'connectors/kc-sink-ibm-mq/installation' | relative_url}}) offers [exactly-once delivery](../mq/sink/#exactly-once-message-delivery-semantics-in-ibm-mq-sink-connector) of data from Apache Kafka into IBM MQ. + +![Kafka Connect: MQ source and sink connectors]({{ 'images' | relative_url }}/mq_sink_and_source.png "Diagram showing a representation of how Event Streams and MQ can be connected by using the MQ source and MQ sink connectors.") + + **Important:** If you want to use IBM MQ connectors on IBM z/OS, you must [prepare your setup first](../mq/zos/). + +## When to use + +Many organizations use both IBM MQ and Apache Kafka for their messaging needs. Although they're generally used to solve different kinds of messaging problems, users often want to connect them together for various reasons. For example, IBM MQ can be integrated with systems of record while Apache Kafka is commonly used for streaming events from web applications. The ability to connect the two systems together enables scenarios in which these two environments intersect. + +**Note:** You can use an existing IBM MQ or Kafka installation, either locally or on the cloud. For convenience, it is recommended to run the Kafka Connect worker on the same Kubernetes cluster as {{site.data.reuse.es_name}}. If the network latency between MQ and {{site.data.reuse.es_name}} is significant, you might prefer to run the Kafka Connect worker close to the queue manager to minimize the effect of network latency. For example, if you have a queue manager in your datacenter and Kafka in the cloud, it's best to run the Kafka Connect worker in your datacenter. + + +## Configuration options + +Find out the available configuration options for the IBM MQ source connector and the IBM MQ sink connector. + +### IBM MQ source connector + +The configuration options for the Kafka Connect source connector for IBM MQ are as follows: + +| Name | Description | Type | Default | Valid values | +| --------------------------------------- | ---------------------------------------------------------------------- | ------- | -------------- | ------------------------------------------------------- | +| topic | The name of the target Kafka topic | string | | Topic name | +| mq.queue.manager | The name of the MQ queue manager | string | | MQ queue manager name | +| mq.connection.mode | The connection mode - bindings or client | string | client | client, bindings | +| mq.connection.name.list | List of connection names for queue manager | string | | host(port)[,host(port),...] | +| mq.channel.name | The name of the server-connection channel | string | | MQ channel name | +| mq.queue | The name of the source MQ queue | string | | MQ queue name | +| exactly.once.source.support | Whether to enable exactly-once support for source connectors in the cluster by using transactions to write source records and their source offsets, and by proactively fencing out old task generations before bringing up new ones. | string | disabled | [disabled, enabled, preparing] | +| mq.exactly.once.state.queue | The name of the MQ queue used to store state when running with exactly-once semantics | string | | MQ state queue name | +| mq.user.name | The user name for authenticating with the queue manager | string | | User name | +| mq.password | The password for authenticating with the queue manager | string | | Password | +| mq.user.authentication.mqcsp | Whether to use MQ connection security parameters (MQCSP) | boolean | true | | +| mq.ccdt.url | The URL for the CCDT file containing MQ connection details | string | | URL for obtaining a CCDT file | +| mq.record.builder | The class used to build the Kafka Connect record | string | | Class implementing RecordBuilder | +| mq.message.body.jms | Whether to interpret the message body as a JMS message type | boolean | false | | +| mq.record.builder.key.header | The JMS message header to use as the Kafka record key | string | | JMSMessageID, JMSCorrelationID, JMSCorrelationIDAsBytes, JMSDestination | +| mq.jms.properties.copy.to.kafka.headers | Whether to copy JMS message properties to Kafka headers | boolean | false | | +| mq.ssl.cipher.suite | The name of the cipher suite for TLS (SSL) connection | string | | Blank or valid cipher suite | +| mq.ssl.peer.name | The distinguished name pattern of the TLS (SSL) peer | string | | Blank or DN pattern | +| mq.ssl.keystore.location | The path to the JKS keystore to use for SSL (TLS) connections | string | JVM keystore | Local path to a JKS file | +| mq.ssl.keystore.password | The password of the JKS keystore to use for SSL (TLS) connections | string | | | +| mq.ssl.truststore.location | The path to the JKS truststore to use for SSL (TLS) connections | string | JVM truststore | Local path to a JKS file | +| mq.ssl.truststore.password | The password of the JKS truststore to use for SSL (TLS) connections | string | | | +| mq.ssl.use.ibm.cipher.mappings | Whether to set system property to control use of IBM cipher mappings | boolean | | | +| mq.batch.size | The maximum number of messages in a batch (unit of work) | integer | 250 | 1 or greater | +| mq.message.mqmd.read | Whether to enable reading of all MQMD fields | boolean | false | | + +### IBM MQ sink connector + +The configuration options for the Kafka Connect sink connector for IBM MQ are as follows: + +| Name | Description | Type | Default | Valid values | +| --------------------------------------- | ---------------------------------------------------------------------- | ------- | -------------- | --------------------------------- | +| topics or topics.regex | List of Kafka source topics | string | | topic1[,topic2,...] | +| mq.queue.manager | The name of the MQ queue manager | string | | MQ queue manager name | +| mq.connection.mode | The connection mode - bindings or client | string | client | client, bindings | +| mq.connection.name.list | List of connection names for queue manager | string | | host(port)[,host(port),...] | +| mq.channel.name | The name of the server-connection channel | string | | MQ channel name | +| mq.exactly.once.state.queue | The name of the MQ queue used to store state when running with exactly-once semantics | string | | MQ state queue name | +| mq.queue | The name of the target MQ queue | string | | MQ queue name | +| mq.user.name | The user name for authenticating with the queue manager | string | | User name | +| mq.password | The password for authenticating with the queue manager | string | | Password | +| mq.user.authentication.mqcsp | Whether to use MQ connection security parameters (MQCSP) | boolean | true | | +| mq.ccdt.url | The URL for the CCDT file containing MQ connection details | string | | URL for obtaining a CCDT file | +| mq.message.builder | The class used to build the MQ message | string | | Class implementing MessageBuilder | +| mq.message.body.jms | Whether to generate the message body as a JMS message type | boolean | false | | +| mq.time.to.live | Time-to-live in milliseconds for messages sent to MQ | long | 0 (unlimited) | [0,...] | +| mq.persistent | Send persistent or non-persistent messages to MQ | boolean | true | | +| mq.ssl.cipher.suite | The name of the cipher suite for TLS (SSL) connection | string | | Blank or valid cipher suite | +| mq.ssl.peer.name | The distinguished name pattern of the TLS (SSL) peer | string | | Blank or DN pattern | +| mq.ssl.keystore.location | The path to the JKS keystore to use for SSL (TLS) connections | string | JVM keystore | Local path to a JKS file | +| mq.ssl.keystore.password | The password of the JKS keystore to use for SSL (TLS) connections | string | | | +| mq.ssl.truststore.location | The path to the JKS truststore to use for SSL (TLS) connections | string | JVM truststore | Local path to a JKS file | +| mq.ssl.truststore.password | The password of the JKS truststore to use for SSL (TLS) connections | string | | | +| mq.ssl.use.ibm.cipher.mappings | Whether to set system property to control use of IBM cipher mappings | boolean | | | +| mq.message.builder.key.header | The JMS message header to set from the Kafka record key | string | | JMSCorrelationID | +| mq.kafka.headers.copy.to.jms.properties | Whether to copy Kafka headers to JMS message properties | boolean | false | | +| mq.message.builder.value.converter | The class and prefix for message builder's value converter | string | | Class implementing Converter | +| mq.message.builder.topic.property | The JMS message property to set from the Kafka topic | string | | Blank or valid JMS property name | +| mq.message.builder.partition.property | The JMS message property to set from the Kafka partition | string | | Blank or valid JMS property name | +| mq.message.builder.offset.property | The JMS message property to set from the Kafka offset | string | | Blank or valid JMS property name | +| mq.reply.queue | The name of the reply-to queue | string | | MQ queue name or queue URI | +| mq.retry.backoff.ms | Wait time, in milliseconds, before retrying after retriable exceptions | long | 60000 | [0,...] | \ No newline at end of file diff --git a/_es_11.5/06-connecting/04-connecting-mq-source.md b/_es_11.5/06-connecting/04-connecting-mq-source.md new file mode 100644 index 00000000..3218492e --- /dev/null +++ b/_es_11.5/06-connecting/04-connecting-mq-source.md @@ -0,0 +1,298 @@ +--- +title: "Running the MQ source connector" +# permalink: /connecting/mq/source/ +excerpt: "Running MQ source connector" +categories: connecting/mq +slug: source +toc: true +--- + +You can use the {{site.data.reuse.kafka-connect-mq-source-short}} to copy data from IBM MQ into Apache Kafka. The connector copies messages from a source MQ queue to a target Kafka topic. + +Kafka Connect can be run in standalone or distributed mode. This document contains steps for running the connector in distributed mode on a Kubernetes platform. In this mode, work balancing is automatic, scaling is dynamic, and tasks and data are fault-tolerant. For more details on the difference between standalone and distributed mode see the [explanation of Kafka Connect workers](../../connectors/#workers). + +## Prerequisites + +To follow these instructions, ensure you have [IBM MQ](https://www.ibm.com/docs/en/ibm-mq/8.0){:target="_blank"} v8 or later installed. + +**Note:** These instructions are for [IBM MQ](https://www.ibm.com/docs/en/ibm-mq/9.3){:target="_blank"} v9 running on Linux. If you are using a different version or platform, you might have to adjust some steps slightly. + +## Setting up the queue manager + +You can set up a queue manager by using the local operating system to authenticate, or by using the IBM MQ Operator. + +### By using local operating system to authenticate + +These sample instructions set up an IBM MQ queue manager that uses its local operating system to authenticate the user ID and password. The user ID and password you provide must already be created on the operating system where IBM MQ is running. + +1. Log in as a user authorized to administer IBM MQ, and ensure the MQ commands are on the path. +2. Create a queue manager with a TCP/IP listener on port 1414: + ```crtmqm -p 1414 ``` + + For example, to create a queue manager called `QM1`, use: `crtmqm -p 1414 QM1` +3. Start the queue manager: + ```strmqm ``` +4. Start the `runmqsc` tool to configure the queue manager: + ```runmqsc ``` +5. In `runmqsc`, create a server-connection channel: + ```DEFINE CHANNEL() CHLTYPE(SVRCONN)``` +6. Set the channel authentication rules to accept connections requiring userid and password: + 1. `SET CHLAUTH() TYPE(BLOCKUSER) USERLIST('nobody')` + 1. `SET CHLAUTH('*') TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(NOACCESS)` + 1. `SET CHLAUTH() TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(CHANNEL) CHCKCLNT(REQUIRED)` + +7. Set the identity of the client connections based on the supplied context (the user ID): + ```ALTER AUTHINFO(SYSTEM.DEFAULT.AUTHINFO.IDPWOS) AUTHTYPE(IDPWOS) ADOPTCTX(YES)``` +8. Refresh the connection authentication information: + ```REFRESH SECURITY TYPE(CONNAUTH)``` +9. Create a queue for the Kafka Connect connector to use: + ```DEFINE QLOCAL()``` +10. Authorize the IBM MQ user ID to connect to and inquire the queue manager: + ```SET AUTHREC OBJTYPE(QMGR) PRINCIPAL('') AUTHADD(CONNECT,INQ)``` +11. Authorize the IBM MQ user ID to use the queue: + ```SET AUTHREC PROFILE() OBJTYPE(QUEUE) PRINCIPAL('') AUTHADD(ALLMQI)``` +12. Stop the `runmqsc` tool by typing `END`. + +For example, for a queue manager called `QM1`, with user ID `alice`, creating a server-connection channel called `MYSVRCONN` and a queue called `MYQSOURCE`, you run the following commands in `runmqsc`: + +```bash +DEFINE CHANNEL(MYSVRCONN) CHLTYPE(SVRCONN) +SET CHLAUTH(MYSVRCONN) TYPE(BLOCKUSER) USERLIST('nobody') +SET CHLAUTH('*') TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(NOACCESS) +SET CHLAUTH(MYSVRCONN) TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(CHANNEL) CHCKCLNT(REQUIRED) +ALTER AUTHINFO(SYSTEM.DEFAULT.AUTHINFO.IDPWOS) AUTHTYPE(IDPWOS) ADOPTCTX(YES) +REFRESH SECURITY TYPE(CONNAUTH) +DEFINE QLOCAL(MYQSOURCE) +SET AUTHREC OBJTYPE(QMGR) PRINCIPAL('alice') AUTHADD(CONNECT,INQ) +SET AUTHREC PROFILE(MYQSOURCE) OBJTYPE(QUEUE) PRINCIPAL('alice') AUTHADD(ALLMQI) +END +``` + +The queue manager is now ready to accept connection from the connector and get messages from a queue. + +### By using the IBM MQ Operator + +You can also use the IBM MQ Operator to set up a queue manager. For more information about installing the IBM MQ Operator and setting up a queue manager, see the [IBM MQ documentation](https://www.ibm.com/docs/en/ibm-mq/9.3?topic=miccpi-using-mq-in-cloud-pak-integration-red-hat-openshift){:target="_blank"}. + +If you are using the IBM MQ Operator to set up a queue manager, you can use the following YAML file to create a queue manager with the required configuration: + +1. Create a file called `custom-source-mqsc-configmap.yaml` and copy the following YAML content into the file to create the ConfigMap that has the details for creating a server connection channel called `MYSVRCONN` and a queue called `MYQSOURCE`: + + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: custom-source-mqsc + data: + source.mqsc: | + DEFINE CHANNEL(MYSVRCONN) CHLTYPE(SVRCONN) + SET CHLAUTH(MYSVRCONN) TYPE(BLOCKUSER) USERLIST('nobody') + SET CHLAUTH('*') TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(NOACCESS) + SET CHLAUTH(MYSVRCONN) TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(CHANNEL) CHCKCLNT(REQUIRED) + ALTER AUTHINFO(SYSTEM.DEFAULT.AUTHINFO.IDPWOS) AUTHTYPE(IDPWOS) ADOPTCTX(YES) + REFRESH SECURITY TYPE(CONNAUTH) + DEFINE QLOCAL(MYQSOURCE) + SET AUTHREC OBJTYPE(QMGR) PRINCIPAL('alice') AUTHADD(CONNECT,INQ) + SET AUTHREC PROFILE(MYQSOURCE) OBJTYPE(QUEUE) PRINCIPAL('alice') AUTHADD(ALLMQI) + ``` + +1. Create the ConfigMap by using the following command: + + ```shell + oc apply -f custom-source-mqsc-configmap.yaml + ``` + +1. To create a queue manager with the required configuration, update the `spec.queueManager` section of the `QueueManager` custom resource YAML file: + + ```yaml + # ... + queueManager: + # ... + mqsc: + - configMap: + name: custom-source-mqsc + items: + - source.mqsc + ``` + +The queue manager is now ready to accept connection from the connector and get messages from a queue. + +## Configuring the connector to connect to MQ + +To connect to IBM MQ and to your {{site.data.reuse.es_name}} or Apache Kafka cluster, the connector requires configuration settings added to a `KafkaConnector` custom resource that represents the connector. + +For IBM MQ connectors, you can generate the `KafkaConnector` custom resource YAML file from either the {{site.data.reuse.es_name}} UI or the CLI. You can also use the CLI to generate a JSON file, which you can use in distributed mode where you supply the connector configuration through REST API calls. + +The connector connects to IBM MQ using a client connection. You must provide the following connection information for your queue manager (these configuration settings are added to the `spec.config` section of the `KafkaConnector` custom resource YAML): + +* The name of the target Kafka topic. +* The name of the IBM MQ queue manager. +* The connection name (one or more host and port pairs). +* The channel name. +* The name of the source IBM MQ queue. +* The user name and password if the queue manager is configured to require them for client connections. + +### Using the UI + +Use the {{site.data.reuse.es_name}} UI to generate and download the `KafkaConnector` custom resource YAML file for your IBM MQ source connector. + +1. {{site.data.reuse.es_ui_login_nonadmin}} +2. Click **Toolbox** in the primary navigation, and scroll to the **Connectors** section. +3. Go to the **Set Up IBM MQ Connector** tile, and click **{{site.data.reuse.kafka-connect-connecting-to-mq}}**. +4. Ensure the **MQ Source** tab is selected, and click **Generate**. +5. In the dialog, enter the configuration of the `MQ Source` connector. +6. Click **Download** to generate and download the configuration file with the supplied fields. +7. Open the downloaded configuration file and change the values of `mq.user.name` and `mq.password` to the username and password that you used to configure your instance of MQ. Also set the label `eventstreams.ibm.com/cluster` to the name of your Kafka Connect instance. + +### Using the CLI + +Use the {{site.data.reuse.es_name}} CLI to generate and download the `KafkaConnector` custom resource YAML file for your IBM MQ source connector. You can also use the CLI to generate a JSON file for distributed mode. + +1. {{site.data.reuse.es_cli_init_111}} +2. Run the `connector-config-mq-source` command to generate the configuration file for the `MQ Source` connector.\\ + For example, to generate a configuration file for an instance of `MQ` with the following information: a queue manager called `QM1`, with a connection point of `localhost(1414)`, a channel name of `MYSVRCONN`, a queue of `MYQSOURCE` and connecting to the topic `TSOURCE`, run the following command: + + ```shell + kubectl es connector-config-mq-source --mq-queue-manager="QM1" --mq-connection-name-list="localhost(1414)" --mq-channel="MYSVRCONN" --mq-queue="MYQSOURCE" --topic="TSOURCE" --file="mq-source" --format yaml + ``` + + **Note**: Omitting the `--format yaml` flag will generate a `mq-source.properties` file which can be used for standalone mode. Specifying `--format json` will generate a `mq-source.json` file which can be used for distributed mode outside the Kubernetes platform. + +3. Change the values of `mq.user.name` and `mq.password` to the username and password that you used to configure your instance of MQ. Also set the label `eventstreams.ibm.com/cluster` to the name of your Kafka Connect instance. + +The final configuration file will resemble the following: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: KafkaConnector +metadata: + name: mq-source + labels: + # The eventstreams.ibm.com/cluster label identifies the KafkaConnect instance + # in which to create this connector. That KafkaConnect instance + # must have the eventstreams.ibm.com/use-connector-resources annotation + # set to true. + eventstreams.ibm.com/cluster: + backup.eventstreams.ibm.com/component: kafkaconnector +spec: + class: com.ibm.eventstreams.connect.mqsource.MQSourceConnector + tasksMax: 1 + config: + topic: TSOURCE + mq.queue.manager: QM1 + mq.connection.name.list: localhost(1414) + mq.channel.name: MYSVRCONN + mq.queue: MYQSOURCE + mq.user.name: alice + mq.password: passw0rd + key.converter: org.apache.kafka.connect.storage.StringConverter + value.converter: org.apache.kafka.connect.storage.StringConverter + mq.record.builder: com.ibm.eventstreams.connect.mqsource.builders.DefaultRecordBuilder +``` + +Run the following command to find a list of all the possible flags: + `kubectl es connector-config-mq-source --help`. + For all available configuration options for IBM MQ source connector, see [connecting to IBM MQ](../#configuration-options). + + + +## Configuring Kafka Connect + +Set up your Kafka Connect environment with the MQ source connector as described in [setting up connectors](../../setting-up-connectors/). When adding connectors, add the MQ connector JAR you downloaded, [add connector dependencies](#adding-connector-dependencies), and when starting the connector, use the Kafka Connect [YAML file](../../setting-up-connectors/#sample-file) you created earlier. + + + +### Verifying the log output + +Verify the log output of Kafka Connect includes the following messages that indicate the connector task has started and successfully connected to IBM MQ: + +``` shell +$ kubectl logs +... +INFO Created connector mq-source +... +INFO Connection to MQ established +... +``` + +## Send a test message + +1. To add messages to the IBM MQ queue, run the `amqsput` sample and type in some messages: + + ```shell + /opt/mqm/samp/bin/amqsput + ``` + +2. {{site.data.reuse.es_ui_login_nonadmin}} + +3. Click **Topics** in the primary navigation and select the connected topic. Messages will appear in the message browser of that topic. + +## Exactly-once message delivery semantics in IBM MQ source connector + +IBM MQ source connector offers exactly-once message delivery semantics. An additional IBM MQ queue is used to store the state of message deliveries. When exactly-once delivery is enabled, all IBM MQ messages are delivered to Kafka with no duplicated messages. + +**Note**: + +- Exactly-once support for source connectors is only available in distributed mode; standalone Kafka Connect workers cannot provide exactly-once delivery semantics. +- Enabling exactly-once delivery in the IBM MQ source connector, results in extra interactions with IBM MQ and {{site.data.reuse.es_name}}, which reduces the throughput. + + +### Prerequisites + +Ensure the following values are set in your environment before you enable the exactly-once behavior: + +* When the MQ source connector has been configured to deliver messages to Kafka with exactly-once semantics, ensure that the downstream consumers are only consuming transactionally committed messages. You can do this by setting the [isolation.level](https://kafka.apache.org/37/documentation/#consumerconfigs_isolation.level){:target="_blank"} configuration property to `read_committed`. +* The IBM MQ source connector must run on Kafka Connect version 3.3.0 or later and the `exactly.once.source.support` property must be set to `enabled` in the Kafka Connect worker configuration. For more information about the `exactly.once.source.support` setting, and the Access Control List (ACL) requirements for the worker nodes, see the [Apache Kafka documentation](https://kafka.apache.org/37/documentation/#connect_exactlyoncesource){:target="_blank"}. +* On the server-connection channel (SVRCONN) used for Kafka Connect, set `HBINT` to 30 seconds to allow IBM MQ transaction rollbacks to occur more quickly in failure scenarios. +* On the [state queue](#creating-a-state-queue-in-ibm-mq-by-using-the-runmqsc-tool) (the queue where the messages are stored), set `DEFSOPT` to `EXCL` to ensure the state queue share option is exclusive. +* Ensure that the messages that are sent through the MQ source connector do not expire and are persistent. + +### Enabling exactly-once delivery + +Ensure you configure the following properties to enable exactly-once delivery: + +* The IBM MQ source connector must have the state queue name configured in the `mq.exactly.once.state.queue` property. The value of the `mq.exactly.once.state.queue` property is the name of a pre-configured IBM MQ queue on the same queue manager as the source IBM MQ queue. + +* If you are configuring the [transaction.boundary](https://kafka.apache.org/37/documentation/#sourceconnectorconfigs_transaction.boundary){:target="_blank"} property, the only permitted property value for the IBM MQ source connector is `poll` (the default value). The `poll` value in the `transaction.boundary` property ensures that the Kafka producer transactions are started and committed for each batch of records that are provided to Kafka by the IBM MQ source connector. + +* Only a single connector task can run in the Kafka Connect instance. As a consequence, the `tasks.max` property must be set to `1` to ensure that failure scenarios do not cause duplicated messages to be delivered. + +* Ensure that the IBM MQ source connector principal has a specific set of ACLs to be able to write transactionally to Kafka. See the [Kafka documentation](https://kafka.apache.org/37/documentation/#connect_exactlyoncesource){:target="_blank"} for the ACL requirements. + +* Ensure that the state queue is empty each time exactly-once delivery is enabled (especially when re-enabling the exactly-once feature). Otherwise, the connector behaves in the same way as when recovering from a failure state, and attempts to get undelivered messages recorded in the out-of-date state message. + +#### Creating a state queue in IBM MQ by using the `runmqsc` tool + +A state message is the message stored in the state queue, and contains the last offset information. +A state queue is a queue in IBM MQ that stores the last offset of the message transferred from Kafka to MQ. The last offset information is used to resume the transfer in case of a failure. When a failure occurs, the connector collects the information and continues to transfer messages from the point where the connector is interrupted. + +Create a state queue as follows. + +**Important:** Each connector instance must have its own state queue. + +1. Start the `runmqsc` tool by running the following command: + + ```shell + runmqsc + ``` + +2. Enter the following command to create a queue: + + ```shell + DEFINE QLOCAL () DEFSOPT (EXCL) + ``` + + Wait for the queue to be created. + +3. Stop the `runmqsc` tool by entering the command`END`. + +**Note:** If the source connector is consuming messages from an IBM MQ for z/OS shared queue, then for performance reasons, the state queue should be placed on the same coupling facility structure. + +### Exactly-once failure scenarios + +The IBM MQ source connector is designed to fail on start-up in certain cases to ensure that exactly-once delivery is not compromised. +In some of these failure scenarios, it will be necessary for an IBM MQ administrator to remove messages from the exactly-once state queue before the IBM MQ source connector can start up and deliver messages from the source queue again. In these cases, the IBM MQ source connector will have the `FAILED` status and the Kafka Connect logs will describe any required administrative action. + +## Advanced configuration + +For all available configuration options for IBM MQ source connector, see [connecting to IBM MQ](../#configuration-options). \ No newline at end of file diff --git a/_es_11.5/06-connecting/05-connecting-mq-sink.md b/_es_11.5/06-connecting/05-connecting-mq-sink.md new file mode 100644 index 00000000..a6fb32a7 --- /dev/null +++ b/_es_11.5/06-connecting/05-connecting-mq-sink.md @@ -0,0 +1,420 @@ +--- +title: "Running the MQ sink connector" +# permalink: /connecting/mq/sink/ +excerpt: "Running MQ sink connector" +categories: connecting/mq +slug: sink +toc: true +--- + +You can use the {{site.data.reuse.kafka-connect-mq-sink-short}} to copy data from {{site.data.reuse.es_name}} or Apache Kafka into IBM MQ. The connector copies messages from a Kafka topic into a target MQ queue. + +Kafka Connect can be run in standalone or distributed mode. This document contains steps for running the connector in distributed mode on a Kubernetes platform. In this mode, work balancing is automatic, scaling is dynamic, and tasks and data are fault-tolerant. For more details on the difference between standalone and distributed mode see the [explanation of Kafka Connect workers](../../connectors/#workers). + +## Prerequisites + +To follow these instructions, ensure you have [IBM MQ](https://www.ibm.com/docs/en/ibm-mq/8.0){:target="_blank"} v8 or later installed. + +**Note:** These instructions are for [IBM MQ](https://www.ibm.com/docs/en/ibm-mq/9.3){:target="_blank"} v9 running on Linux. If you are using a different version or platform, you might have to adjust some steps slightly. + +## Setting up the queue manager + +You can set up a queue manager by using the local operating system to authenticate, or by using the IBM MQ Operator. + +### By using local operating system to authenticate + +These sample instructions set up an IBM MQ queue manager that uses its local operating system to authenticate the user ID and password. The user ID and password you provide must already be created on the operating system where IBM MQ is running. + +1. Log in as a user authorized to administer IBM MQ, and ensure the MQ commands are on the path. +2. Create a queue manager with a TCP/IP listener on port 1414: + ```crtmqm -p 1414 ``` + + For example, to create a queue manager called `QM1`, use: `crtmqm -p 1414 QM1` +3. Start the queue manager: + ```strmqm ``` +4. Start the `runmqsc` tool to configure the queue manager: + ```runmqsc ``` +5. In `runmqsc`, create a server-connection channel: + ```DEFINE CHANNEL() CHLTYPE(SVRCONN)``` +6. Set the channel authentication rules to accept connections requiring userid and password: + 1. `SET CHLAUTH() TYPE(BLOCKUSER) USERLIST('nobody')` + 1. `SET CHLAUTH('*') TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(NOACCESS)` + 1. `SET CHLAUTH() TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(CHANNEL) CHCKCLNT(REQUIRED)` + +7. Set the identity of the client connections based on the supplied context (the user ID): + ```ALTER AUTHINFO(SYSTEM.DEFAULT.AUTHINFO.IDPWOS) AUTHTYPE(IDPWOS) ADOPTCTX(YES)``` +8. Refresh the connection authentication information: + ```REFRESH SECURITY TYPE(CONNAUTH)``` +9. Create a queue for the Kafka Connect connector to use: + ```DEFINE QLOCAL()``` +10. Authorize the IBM MQ user ID to connect to and inquire the queue manager: + ```SET AUTHREC OBJTYPE(QMGR) PRINCIPAL('') AUTHADD(CONNECT,INQ)``` +11. Authorize the IBM MQ user ID to use the queue: + ```SET AUTHREC PROFILE() OBJTYPE(QUEUE) PRINCIPAL('') AUTHADD(ALLMQI)``` +12. Stop the `runmqsc` tool by typing `END`. + +For example, for a queue manager called `QM1`, with user ID `alice`, creating a server-connection channel called `MYSVRCONN` and a queue called `MYQSINK`, you run the following commands in `runmqsc`: + +```bash +DEFINE CHANNEL(MYSVRCONN) CHLTYPE(SVRCONN) +SET CHLAUTH(MYSVRCONN) TYPE(BLOCKUSER) USERLIST('nobody') +SET CHLAUTH('*') TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(NOACCESS) +SET CHLAUTH(MYSVRCONN) TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(CHANNEL) CHCKCLNT(REQUIRED) +ALTER AUTHINFO(SYSTEM.DEFAULT.AUTHINFO.IDPWOS) AUTHTYPE(IDPWOS) ADOPTCTX(YES) +REFRESH SECURITY TYPE(CONNAUTH) +DEFINE QLOCAL(MYQSINK) +SET AUTHREC OBJTYPE(QMGR) PRINCIPAL('alice') AUTHADD(CONNECT,INQ) +SET AUTHREC PROFILE(MYQSINK) OBJTYPE(QUEUE) PRINCIPAL('alice') AUTHADD(ALLMQI) +END +``` + +The queue manager is now ready to accept connection from the connector and put messages on a queue. + +### By using the IBM MQ Operator + +You can also use IBM MQ Operator to set up a queue manager. For more information about installing the IBM MQ Operator and setting up a queue manager, see the [IBM MQ documentation](https://www.ibm.com/docs/en/ibm-mq/9.3?topic=miccpi-using-mq-in-cloud-pak-integration-red-hat-openshift){:target="_blank"}. + +If you are using IBM MQ Operator to set up a queue manager, you can use the following yaml to create a queue manager with the required configuration: + +1. Create a file called `custom-sink-mqsc-configmap.yaml` and copy the following YAML content to create the ConfigMap that has the details for creates a server-connection channel called `MYSVRCONN` and a queue called `MYQSINK`: + + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: custom-sink-mqsc + data: + sink.mqsc: | + DEFINE CHANNEL(MYSVRCONN) CHLTYPE(SVRCONN) + SET CHLAUTH(MYSVRCONN) TYPE(BLOCKUSER) USERLIST('nobody') + SET CHLAUTH('*') TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(NOACCESS) + SET CHLAUTH(MYSVRCONN) TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(CHANNEL) CHCKCLNT(REQUIRED) + ALTER AUTHINFO(SYSTEM.DEFAULT.AUTHINFO.IDPWOS) AUTHTYPE(IDPWOS) ADOPTCTX(YES) + REFRESH SECURITY TYPE(CONNAUTH) + DEFINE QLOCAL(MYQSINK) + SET AUTHREC OBJTYPE(QMGR) PRINCIPAL('alice') AUTHADD(CONNECT,INQ) + SET AUTHREC PROFILE(MYQSINK) OBJTYPE(QUEUE) PRINCIPAL('alice') AUTHADD(ALLMQI) + ``` + +1. Create the ConfigMap by using the following command: + + ```shell + oc apply -f custom-sink-mqsc-configmap.yaml + ``` + +1. To create a queue manager with the required configuration, update the `spec.queueManager` section of the `QueueManager` custom resource YAML file: + + ```yaml + # ... + queueManager: + # ... + mqsc: + - configMap: + name: custom-sink-mqsc + items: + - sink.mqsc + ``` + +The queue manager is now ready to accept connection from the connector and put messages on a queue. + +## Configuring the connector to connect to MQ + +To connect to IBM MQ and to your {{site.data.reuse.es_name}} or Apache Kafka cluster, the connector requires configuration settings added to a `KafkaConnector` custom resource that represents the connector. + +For IBM MQ connectors, you can generate the `KafkaConnector` custom resource YAML file from either the {{site.data.reuse.es_name}} UI or the CLI. You can also use the CLI to generate a JSON file, which you can use in distributed mode where you supply the connector configuration through REST API calls. + +The connector connects to IBM MQ using a client connection. You must provide the following connection information for your queue manager (these configuration settings are added to the `spec.config` section of the `KafkaConnector` custom resource YAML): + +* Comma-separated list of Kafka topics to pull events from. +* The name of the IBM MQ queue manager. +* The connection name (one or more host and port pairs). +* The channel name. +* The name of the sink IBM MQ queue. +* The user name and password if the queue manager is configured to require them for client connections. + +### Using the UI + +Use the {{site.data.reuse.es_name}} UI to generate and download the `KafkaConnector` custom resource YAML file for your IBM MQ sink connector. + +1. {{site.data.reuse.es_ui_login_nonadmin}} +2. Click **Toolbox** in the primary navigation, and scroll to the **Connectors** section. +3. Go to the **Set Up IBM MQ Connector** tile, and click **{{site.data.reuse.kafka-connect-connecting-to-mq}}**. +4. Ensure the **MQ Sink** tab is selected, and click **Generate**. +5. In the dialog, enter the configuration of the `MQ Sink` connector. +6. Click **Download** to generate and download the configuration file with the supplied fields. +7. Open the downloaded configuration file and change the values of `mq.user.name` and `mq.password` to the username and password that you used to configure your instance of MQ. Also set the label `eventstreams.ibm.com/cluster` to the name of your Kafka Connect instance. + +### Using the CLI + +Use the {{site.data.reuse.es_name}} CLI to generate and download the `KafkaConnector` custom resource YAML file for your IBM MQ sink connector. You can also use the CLI to generate a JSON file for distributed mode. + +1. {{site.data.reuse.es_cli_init_111}} +2. Run the following command to initialize the {{site.data.reuse.es_name}} CLI on the cluster: + + ```shell + kubectl es init + ``` + +3. Run the `connector-config-mq-sink` command to generate the configuration file for the `MQ Sink` connector. + + For example, to generate a configuration file for an instance of `MQ` with the following information: a queue manager called `QM1`, with a connection point of `localhost(1414)`, a channel name of `MYSVRCONN`, a queue of `MYQSINK` and connecting to the topics `TSINK`, run the following command: + + ```shell + kubectl es connector-config-mq-sink --mq-queue-manager="QM1" --mq-connection-name-list="localhost(1414)" --mq-channel="MYSVRCONN" --mq-queue="MYQSINK" --topics="TSINK" --file="mq-sink" --format yaml + ``` + + **Note**: Omitting the `--format yaml` flag will generate a `mq-sink.properties` file which can be used for standalone mode. Specifying `--format json` will generate a `mq-sink.json` file which can be used for distributed mode outside {{site.data.reuse.openshift_short}}. +4. Change the values of `mq.user.name` and `mq.password` to the username and password that you used to configure your instance of MQ. Also set the label `eventstreams.ibm.com/cluster` to the name of your Kafka Connect instance. + +The final configuration file will resemble the following: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: KafkaConnector +metadata: + name: mq-sink + labels: + # The 'eventstreams.ibm.com/cluster' label identifies the KafkaConnect instance in which to create this connector. + # That KafkaConnect instance must have the 'eventstreams.ibm.com/use-connector-resources' annotation set to 'true'. + eventstreams.ibm.com/cluster: + backup.eventstreams.ibm.com/component: kafkaconnector +spec: + class: com.ibm.eventstreams.connect.mqsink.MQSinkConnector + tasksMax: 1 + config: + topics: TSINK + mq.queue.manager: QM1 + mq.connection.name.list: localhost(1414) + mq.channel.name: MYSVRCONN + mq.queue: MYQSINK + mq.user.name: alice + mq.password: passw0rd + key.converter: org.apache.kafka.connect.storage.StringConverter + value.converter: org.apache.kafka.connect.storage.StringConverter + mq.message.builder: com.ibm.eventstreams.connect.mqsink.builders.DefaultMessageBuilder +``` + +Run the following command to find a list of all the possible flags: `kubectl es connector-config-mq-sink --help`. +For all available configuration options for IBM MQ sink connector, see [connecting to IBM MQ](../#configuration-options). + +## Downloading the MQ sink connector + +Follow the instructions to download the MQ sink connector from [IBM Fix Central](https://ibm.biz/ea-fix-central){:target="_blank"}. + +1. Go to [IBM Fix Central](https://ibm.biz/ea-fix-central){:target="_blank"}. If you are on the **Select fixes** page, you can skip to step 6. +2. In the **Find Product > Product selector**, enter **IBM Event Automation**. +3. In the **Installed version**, select a specific version of {{site.data.reuse.ea_long}} or select **All** to list products in all the versions. +4. In the **Platform**, select **All**, and then click **Continue**. +5. In the **Identify Fixes** page, select **Browse for fixes**, and then click **Continue**. The available connectors are listed in the **Select fixes** page. +6. Select the MQ sink connector version you want to download and click **Continue**. For example, `kafka-connect-mq-sink-2.0.0`. The download page opens with the default download option. +7. In your preferred download option, click the connector (for example, `kafka-connect-mq-sink-2.0.0.jar`) to download the connector. + +The connector JAR file is downloaded. + +**Important:** To use the Kafka Connect Build capability for setting this connector, you must upload the JAR to a location that is accessible from the cluster, and provide the URL in the Kafka Connect custom resource. + +## Configuring Kafka Connect + +Set up your Kafka Connect environment as described in [setting up connectors](../../setting-up-connectors/). When adding connectors, add the MQ connector JAR you downloaded, [add connector dependencies](#adding-connector-dependencies), and when starting the connector, use the Kafka Connect [YAML file](../../setting-up-connectors/#sample-file) you created earlier. + + +### Verifying the log output + +Verify the log output of Kafka Connect includes the following messages that indicate the connector task has started and successfully connected to IBM MQ: + +``` shell +$ kubectl logs +... +INFO Created connector mq-sink +... +INFO Connection to MQ established +... +``` + +## Send a test message + +To test the connector you will need an application to produce events to your topic. + +1. {{site.data.reuse.es_ui_login_nonadmin}} +2. Click **Toolbox** in the primary navigation. +3. Go to the **Starter application** tile under **Applications**, and click **Get started**. +4. Click **Download JAR from GitHUb**. Download the JAR file from the list of assets for the latest release. +5. Click **Generate properties**. +6. Enter a name for the application. +7. Go to the **Existing topic** tab and select the topic you provided in the MQ connector configuration. +8. Click **Generate and download .zip**. +9. Follow the instructions in the UI to get the application running. + +Verify the message is on the queue: + +1. Navigate to the UI of the [sample application](../../../getting-started/generating-starter-app/) you generated earlier and start producing messages to {{site.data.reuse.es_name}}. +2. Use the `amqsget` sample to get messages from the MQ queue: + + ```shell + /opt/mqm/samp/bin/amqsget + ``` + + After a short delay, the messages are printed. + +## Exactly-once message delivery semantics in IBM MQ sink connector + +The IBM MQ sink connector offers exactly-once message delivery semantics. An additional IBM MQ queue is used to store the state of message deliveries. When exactly-once delivery is enabled, Kafka messages are delivered to IBM MQ with no duplicated messages. + +Follow the instructions to enable exactly-once delivery in the IBM MQ sink connector. + +**Important**: Exactly-once support for sink connectors is only available in distributed mode. + +### Prerequisites + +Ensure the following values are set in your environment before you enable the exactly-once behavior: + +* Configure the consumer group of the sink connector to ignore records in aborted transactions. You can find detailed instructions in the [Kafka documentation](https://kafka.apache.org/37/documentation/#connect_exactlyoncesink){:target="_blank"}. Notably, this configuration does not have any additional Access Control List (ACL) requirements. +* The IBM MQ sink connector is only supported on Kafka Connect version 2.6.0 or later. +* On the server-connection channel (SVRCONN) used for Kafka Connect, set `HBINT` to 30 seconds to allow IBM MQ transaction rollbacks to occur more quickly in failure scenarios. +* On the [state queue](#creating-a-state-queue-in-ibm-mq-by-using-the-runmqsc-tool) (the queue where the state messages are stored), set `DEFSOPT` to `EXCL` to ensure the state queue share option is exclusive. +* Ensure that the messages sent through the MQ sink connector do not expire and that all the messages on the state queue are persistent. + +### Enabling exactly-once delivery + +Configure the following properties to enable exactly-once delivery: + +* The IBM MQ sink connector must be configured with the `mq.exactly.once.state.queue` property set to the name of a pre-configured IBM MQ queue on the same queue manager as the sink IBM MQ queue. + +* Only a single connector task can be run. As a consequence, the `tasks.max` property must be left unset, or set to `1`. + +* Ensure that the state queue is empty each time exactly-once delivery is enabled (especially when re-enabling the exactly-once feature). Otherwise, the connector behaves in the same way as when recovering from a failure state, and attempts to get undelivered messages recorded in the out-of-date state message. + +After enabling exactly-once delivery, Kafka messages are delivered to IBM MQ with no duplicated messages. + +#### Creating a state queue in IBM MQ by using the `runmqsc` tool + +A state message is the message stored in the state queue, and contains the last offset information. +A state queue is a queue in IBM MQ that stores the last offset of the message transferred from Kafka to MQ. The last offset information is used to resume the transfer in case of a failure. When a failure occurs, the connector collects the information and continues to transfer messages from the point where the connector is interrupted. + +Create a state queue as follows. + +**Important:** Each connector instance must have its own state queue. + +1. Start the `runmqsc` tool by running the following command: + + ```shell + runmqsc + ``` + + Replace `` with the name of the queue manager you want to work with. + +1. Enter the following command to create a queue: + + ```shell + DEFINE QLOCAL () DEFSOPT (EXCL) + ``` + + Where: + + * `` is the name of the state queue. + * Set `DEFSOPT` to `EXCL` to ensure the state queue share option is exclusive. + + Wait for the queue to be created. + +1. Stop the `runmqsc` tool by entering the command`END`. + +**Note:** If the sink connector is delivering messages to an IBM MQ for z/OS shared queue, then for performance reasons, the state queue should be placed on the same coupling facility structure. + +## Enabling MQMD in IBM MQ sink connector + +You can configure the IBM MQ sink connector to add values to the [MQ message descriptor (MQMD)](https://www.ibm.com/docs/en/ibm-mq/9.3?topic=mqi-mqmd-message-descriptor){:target="_blank"}. The MQMD is used to add additional control information to accompany the message data before sending to MQ. + +### Prerequisites + +Adding values to the MQMD is only supported in the IBM MQ sink connector, which is only supported on Kafka Connect version 2.6.0 or later. + +### Enabling MQMD + +Configure the following properties in the `KafkaConnector` custom resource to enable adding values to the MQMD: + +1. Set the `mq.message.mqmd.write` property to `true` in the IBM MQ sink connector. By default, it is set to `false`. + +2. Configure the `mq.message.mqmd.context` property according to the message context. Options include: + + * `ALL`, which corresponds to `WMQ_MDCTX_SET_ALL_CONTEXT` + * `IDENTITY`, mapped to `WMQ_MDCTX_SET_IDENTITY_CONTEXT` + + **Important:** If your message contains any of the following properties, you must ensure that `WMQ_MQMD_MESSAGE_CONTEXT` is set to either `WMQ_MDCTX_SET_IDENTITY_CONTEXT` or `WMQ_MDCTX_SET_ALL_CONTEXT`: + + * JMS_IBM_MQMD_UserIdentifier + * JMS_IBM_MQMD_AccountingToken + * JMS_IBM_MQMD_ApplIdentityData + + Similarly, if your message includes any of the following properties, set the `WMQ_MQMD_MESSAGE_CONTEXT` field to `WMQ_MDCTX_SET_ALL_CONTEXT`: + + * JMS_IBM_MQMD_PutApplType + * JMS_IBM_MQMD_PutApplName + * JMS_IBM_MQMD_PutDate + * JMS_IBM_MQMD_PutTime + * JMS_IBM_MQMD_ApplOriginData + + Other message properties do not require the `mq.message.mqmd.context` property. + +### Creating a custom message builder + +The MQ sink connector uses the default message builder to write messages to MQ. You can also create a custom message builder to tailor message properties based on your specific requirements. + +For example, complete the following steps to create a custom message builder that overrides values of MQMD fields such as `JMS_IBM_MQMD_USERIDENTIFIER`, `JMS_IBM_MQMD_APPLIDENTITYDATA`, and `JMS_IBM_MQMD_PERSISTENCE` for outgoing messages: + +1. Create a Java class called `CustomMessageDescriptorBuilder` that extends the `DefaultMessageBuilder` provided by the MQ sink connector. The `CustomMessageDescriptorBuilder` class is the custom message builder. + + ```java + // CustomMessageDescriptorBuilder.java + import javax.jms.JMSContext; + import javax.jms.JMSException; + import javax.jms.Message; + import org.apache.kafka.connect.errors.ConnectException; + import org.apache.kafka.connect.sink.SinkRecord; + import com.ibm.eventstreams.connect.mqsink.builders.DefaultMessageBuilder; + import com.ibm.msg.client.jms.JmsConstants; + + public class CustomMessageDescriptorBuilder extends DefaultMessageBuilder { + + @Override + public Message getJMSMessage(final JMSContext jmsCtxt, final SinkRecord record) { + final Message message = super.getJMSMessage(jmsCtxt, record); + try { + // Override properties + message.setStringProperty(JmsConstants.JMS_IBM_MQMD_USERIDENTIFIER, "Sample User"); + message.setStringProperty(JmsConstants.JMS_IBM_MQMD_APPLIDENTITYDATA, String.format("{}-{}-{}", record.topic(), record.kafkaOffset(), record.kafkaPartition())); + // Set persistence based on message content + if (message.getBody(String.class).startsWith("PRIORITY")) { + message.setIntProperty(JmsConstants.JMS_IBM_MQMD_PERSISTENCE, MQConstants.MQPER_PERSISTENT); + } else { + message.setIntProperty(JmsConstants.JMS_IBM_MQMD_PERSISTENCE, MQConstants.MQPER_NOT_PERSISTENT); + } + } catch (final JMSException e) { + throw new ConnectException("Failed to write property", e); + } + return message; + } + } + ``` + +1. In the `getJMSMessage()` method, override the values of MQMD properties based on your requirements. In the earlier example, `JMS_IBM_MQMD_USERIDENTIFIER` and `JMS_IBM_MQMD_APPLIDENTITYDATA` properties are set with sample values, and the `JMS_IBM_MQMD_PERSISTENCE` property is set based on the message content. + +1. Package your custom message builder into a JAR file along with any dependencies it might have. Ensure that this JAR file is placed in a folder alongside the connector JAR file: + + ```shell + mq-sink-connector/ + |-- ibm-mq-sink-connector.jar + |-- custom-message-builder.jar + ``` + +1. Configure the IBM MQ sink connector to use your custom message builder class. Specify the class name of your custom message builder in the connector configuration: + + ```yaml + mq.message.builder= + mq.message.body.jms=true + mq.message.mqmd.write=true + mq.message.mqmd.context=ALL + # ... + ``` + + Where `` is the name of your custom message builder. + diff --git a/_es_11.5/06-connecting/06-running-connectors-on-zos.md b/_es_11.5/06-connecting/06-running-connectors-on-zos.md new file mode 100644 index 00000000..a0a29b4c --- /dev/null +++ b/_es_11.5/06-connecting/06-running-connectors-on-zos.md @@ -0,0 +1,279 @@ +--- +title: "Running connectors on IBM z/OS" +# permalink: /connecting/mq/zos/ +excerpt: "Set up your Kafka Connect workers to run on IBM z/OS." +categories: connecting/mq +slug: zos +toc: true +--- + +You can use the IBM MQ connectors to connect into IBM MQ for z/OS, and you can run the connectors on z/OS as well, connecting into the queue manager using bindings mode. These instructions explain how to run Kafka Connect in both standalone and distributed mode. For more information and to help decide which mode to use see the [explanation of Kafka Connect workers](../../connectors/#workers). + +Before you can run IBM MQ connectors on IBM z/OS, you must prepare your Kafka files and your system as follows. + +## Setting up Kafka to run on IBM z/OS + +You can run Kafka Connect workers on [IBM z/OS Unix System Services](https://www.ibm.com/docs/en/zos/2.3.0?topic=descriptions-zos-unix-system-services){:target="_blank"}. To do so, you must ensure that the Kafka Connect shell scripts and the Kafka Connect configuration files are converted to EBCDIC encoding. + +### Download the Kafka Connect files + +Download Apache Kafka to a non-z/OS system to retrieve the `.tar` file that includes the Kafka Connect shell scripts and JAR files. + +To download Kafka Connect and make it available to your z/OS system: + +1. Log in to a system that is not running IBM z/OS, for example, a Linux system. +2. [Download](https://kafka.apache.org/downloads){:target="_blank"} Apache Kafka 3.4.0 or later to the system. {{site.data.reuse.es_name}} provides support for Kafka Connect if you are using a Kafka version listed in the **Kafka version shipped** column of the [support matrix]({{ 'support/matrix/#event-streams' | relative_url }}). +3. Extract the downloaded `.tgz` file, for example: + + ```shell + gunzip -k kafka_2.13-3.4.0.tgz + ``` + +4. Copy the resulting `.tar` file to a directory on the z/OS Unix System Services. + +### Download IBM MQ connectors and configuration + +Depending on the connector you want to use: + + - Download the [source connector JAR](../source/#downloading-the-mq-source-connector) and [source configuration file](../source/#configuring-the-connector-to-connect-to-mq) + - Download the [sink connector JAR](../sink/#downloading-the-mq-sink-connector) and [sink configuration file](../sink/#configuring-the-connector-to-connect-to-mq) + +If you want to run a standalone Kafka Connect worker, you need a `.properties` file. To run a distributed Kafka Connect worker, you need a `.json` file. + +Copy the connector JAR file(s) and the required configuration file to a directory on the z/OS Unix System Services. + +### Convert the files + +If you want to run a standalone Kafka Connect worker, convert the following shell scripts and configuration files from ISO8859-1 to EBCDIC encoding: +- `bin/kafka-run-class.sh` +- `bin/connect-standalone.sh` +- `config/connect-standalone.properties` +- `mq-source.properties` or `mq-sink.properties` + +If you want to run a distributed Kafka Connect worker, convert the following shell scripts and configuration files from ISO8859-1 to EBCDIC encoding: +- `bin/kafka-run-class.sh` +- `bin/connect-distributed.sh` +- `config/connect-distributed.sh` + +Extract the Apache Kafka distribution: +1. Log in to the IBM z/OS system and [access the Unix System Services](https://www.ibm.com/docs/en/zos/2.3.0?topic=zos-unix-system-services-procedures){:target="_blank"}. +2. Change to an empty directory that you want to use for the Apache Kafka distribution, and copy the `.tar` file to the new directory. +3. Extract the `.tar` file, for example: + + ```shell + tar -xvf kafka_2.13-2.8.1.tar + ``` + +4. Change to the resulting `kafka_` directory. + +Convert the shell scripts: +1. Copy the `connect-standalone.sh` shell script (or `connect-distributed.sh` for a distributed setup) into the current directory, for example: + + ```shell + cp bin/connect-standalone.sh ./connect-standalone.sh.orig + ``` + +2. Determine the codeset on the IBM z/OS system by running: + + ```shell + locale -k codeset + ``` + +3. Convert the script to EBCDIC encoding and replace the original, for example for codeset IBM-1047: + + ```shell + iconv -f ISO8859-1 -t IBM-1047 ./connect-standalone.sh.orig > bin/connect-standalone.sh + ``` + +4. Ensure the file permissions are set so that the script is executable, for example: + + ```shell + chmod +x bin/connect-standalone.sh + ``` + +5. Copy the `kafka-run-class.sh` shell script into the current directory, for example: + + ```shell + cp bin/kafka-run-class.sh ./kafka-run-class.sh.orig + ``` + +6. Convert the script to EBCDIC encoding and replace the original, for example for codeset IBM-1047: + + ```shell + iconv -f ISO8859-1 -t IBM-1047 ./kafka-run-class.sh.orig > bin/kafka-run-class.sh + ``` + +7. Ensure the file permissions are set so that the script is executable, for example: + + ```shell + chmod +x bin/kafka-run-class.sh + ``` + +Convert the configuration files: +1. Copy the `connect-standalone.properties` file (or `connect-distributed.properties` for a distributed setup) into the current directory, for example: + + ```shell + cp config/connect-standalone.properties ./connect-standalone.properties.orig + ``` + +2. Determine the codeset on the IBM z/OS system by running: + + ```shell + locale -k codeset + ``` + +3. Convert the script to EBCDIC encoding and replace the original, for example for codeset IBM-1047: + + ```shell + iconv -f ISO8859-1 -t IBM-1047 ./connect-standalone.properties.orig > config/connect-standalone.properties + ``` + +If running in **standalone mode**: +1. Copy the MQ `.properties` file into the current directory, for example: + + ```shell + cp ./mq-source.properties ./mq-source.properties.orig + ``` + +2. Determine the codeset on the IBM z/OS system by running: + + ```shell + locale -k codeset + ``` + +3. Convert the script to EBCDIC encoding and replace the original, for example for codeset IBM-1047: + + ```shell + iconv -f ISO8859-1 -t IBM-1047 ./mq-source.properties.orig > ./mq-source.properties + ``` + +**Note:** For distributed mode the `.json` file must remain in ASCII format. + +## Update the Kafka Connect configuration + +The `connect-standalone.properties` (or `connect-distributed.properties` for distributed mode) file must include the correct `bootstrap.servers` and SASL/SSL configuration for your Apache Kafka or {{site.data.reuse.es_name}} install. + +For example, if running against {{site.data.reuse.es_name}}, download the certificate for your install to your IBM z/OS system. Generate credentials that can produce, consume and create topics and update the `connect-standalone.properties` (or `connect-distributed.properties`) file to include: + +```shell +bootstrap.servers= +security.protocol=SASL_SSL +ssl.protocol=TLSv1.3 +ssl.truststore.location=/opt/kafka/es-cert.p12 +ssl.truststore.password= +ssl.truststore.type=PKCS12 +sasl.mechanism=SCRAM-SHA-512 +sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password=""; +producer.security.protocol=SASL_SSL +producer.ssl.protocol=TLSv1.3 +producer.ssl.truststore.location=/opt/kafka/es-cert.p12 +producer.ssl.truststore.password= +producer.ssl.truststore.type=PKCS12 +producer.sasl.mechanism=SCRAM-SHA-512 +producer.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="userName" password=""; +consumer.security.protocol=SASL_SSL +consumer.ssl.protocol=TLSv1.3 +consumer.ssl.truststore.location=/opt/kafka/es-cert.p12 +consumer.ssl.truststore.password= +consumer.ssl.truststore.type=PKCS12 +consumer.sasl.mechanism=SCRAM-SHA-512 +consumer.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="userName" password=""; +plugin.path=/opt/connectors +``` + +## Configuring the environment + +The IBM MQ connectors use the JMS API to connect to MQ. You must set the environment variables required for JMS applications before running the connectors on IBM z/OS. + +Ensure you set `CLASSPATH` to include `com.ibm.mq.allclient.jar`, and also set the JAR file for the connector you are using - this is the connector JAR file you downloaded from the {{site.data.reuse.es_name}} UI or built after cloning the GitHub project, for example, `kafka-connect-mq-source-1.3.0-jar-with-dependencies.jar`. + +As you are using the bindings connection mode for the connector to connect to the queue manager, also set the following environment variables: + +1. The `STEPLIB` used at run time must contain the IBM MQ `SCSQAUTH` and `SCSQANLE` libraries. Specify this library in the startup JCL, or specify it by using the `.profile` file. + + From UNIX and Linux System Services, you can add these using a line in your `.profile` file as shown in the following code snippet, replacing `thlqual` with the high-level data set qualifier that you chose when installing IBM MQ: + + ```shell + export STEPLIB=thlqual.SCSQAUTH:thlqual.SCSQANLE:$STEPLIB + ``` + +2. The connector needs to load a native library. Set `LIBPATH` to include the following directory of your MQ installation: + + ```shell + /mqm//java/lib + ``` + +The bindings connection mode is a configuration option for the connector as described in the [source connector GitHub README](https://github.com/ibm-messaging/kafka-connect-mq-source/blob/master/README.md#configuration){:target="_blank"} and in the [sink connector GitHub README](https://github.com/ibm-messaging/kafka-connect-mq-sink/blob/master/README.md#configuration){:target="_blank"}. + +## Starting Kafka Connect on z/OS + +Kafka Connect is started using a bash script. If you do not already have bash installed on your z/OS system install it now. + +To install bash version 4.2.53 or later: + +1. Download the bash archive file from [Bash Version 4.2.53](https://www.rocketsoftware.com/product-categories/mainframe/bash-zos){:target="_blank"}. + + **Note:** You must [register](https://my.rocketsoftware.com/RocketCommunity/s/login/?ec=302&startURL=%2FRocketCommunity%2Fs%2F){:target="_blank"} before downloading the bash archive. +2. Extract the archive file to get the .tar file: + + ```shell + gzip -d bash.tar.gz + ``` + +3. FTP the .tar file to your z/OS USS directory such as `/bin` +4. Extract the .tar file to install bash: + + ```shell + tar -cvfo bash.tar + ``` + +If bash on your z/OS system is not in /bin, you need to update the `kafka-run-class.sh` file. For example, if bash is located in `/usr/local/bin` update the first line of `kafka-run-class.sh` to have `#!/usr/local/bin/bash` + +### Starting Kafka Connect in standalone mode + +To start Kafka Connect in standalone mode navigate to your Kafka directory and run the `connect-standalone.sh` script, passing in your `connect-standalone.properties` and `mq-source.properties` or `mq-sink.properties`. For example: + +```shell +cd kafka +./bin/connect-standalone.sh connect-standalone.properties mq-source.properties +``` + +For more details on creating the properties files see the [connecting MQ documentation](../../../connecting/mq). Make sure connection type is set to bindings mode. + +### Starting Kafka Connect in distributed mode + +To start Kafka Connect in distributed mode navigate to your Kafka directory and run the `connect-distributed.sh` script, passing in your `connect-distributed.properties`. Unlike in standalone mode, MQ properties are not passed in on startup. For example: + +```shell +cd kafka +./bin/connect-distributed.sh connect-distributed.properties +``` + +To start an individual connector use the [Kafka Connect REST API](https://kafka.apache.org/37/documentation/#connect_rest){:target="_blank"}. For example, given a configuration file `mq-source.json` with the following contents: + +```json +{ + "name":"mq-source", + "config" : { + "connector.class":"com.ibm.eventstreams.connect.mqsource.MQSourceConnector", + "tasks.max":"1", + "mq.queue.manager":"QM1", + "mq.connection.mode":"bindings", + "mq.queue":"MYQSOURCE", + "mq.record.builder":"com.ibm.eventstreams.connect.mqsource.builders.DefaultRecordBuilder", + "topic":"test", + "key.converter":"org.apache.kafka.connect.storage.StringConverter", + "value.converter":"org.apache.kafka.connect.converters.ByteArrayConverter" + } + } +``` + +start the connector using: + +```shell +curl -X POST http://localhost:8083/connectors -H "Content-Type: application/json" -d @mq-source.json +``` + +## Advanced configuration + +For more details about the connectors and to see all configuration options, see the [source connector GitHub README](https://github.com/ibm-messaging/kafka-connect-mq-source#readme){:target="_blank"} or [sink connector GitHub README](https://github.com/ibm-messaging/kafka-connect-mq-sink#readme){:target="_blank"}. diff --git a/_es_11.5/07-administering/01-deployment-health.md b/_es_11.5/07-administering/01-deployment-health.md new file mode 100644 index 00000000..452bef65 --- /dev/null +++ b/_es_11.5/07-administering/01-deployment-health.md @@ -0,0 +1,70 @@ +--- +title: "Monitoring deployment health" +excerpt: "Understand the health of your deployment at a glance, and learn how to find information about problems." +categories: administering +slug: deployment-health +toc: true +--- + +Understand the health of your {{site.data.reuse.es_name}} deployment at a glance, and learn how to find information about problems. + +![Event Streams metrics architecture]({{ 'images' | relative_url }}/architectures/ibm-event-automation-diagrams-es-metrics.svg "Diagram showing the architecture of the Event Streams metrics as part of IBM Event Automation.") + +## Checking {{site.data.reuse.es_name}} health + +### Using the {{site.data.reuse.es_name}} UI + +The {{site.data.reuse.es_name}} UI provides information about the health of your environment at a glance. In the bottom right corner of the UI, a message shows a summary status of the system health. If there are no issues, the message states **System is healthy**. + +If any of the {{site.data.reuse.es_name}} resources experience problems, the message states **component isn't ready**. +If any of the components are not ready for an extended period of time, see how you can troubleshoot as described in [debugging](#debugging). + +### Using the CLI + +You can check the health of your {{site.data.reuse.es_name}} environment using the Kubernetes command-line tool (`kubectl`). + +1. {{site.data.reuse.cncf_cli_login}} +2. To check the status and readiness of the pods, run the following command, where `` is where your {{site.data.reuse.es_name}} instance is installed: + + ```shell + kubectl -n get pods + ``` + + The command lists the pods together with simple status information for each pod. + +If any of the components are not ready for an extended period of time, check the [debugging topic](#debugging). + +## Debugging + +### Using the {{site.data.reuse.openshift_short}} UI + +1. {{site.data.reuse.openshift_ui_login}} +2. {{site.data.reuse.task_openshift_navigate_installed_operators}} +3. {{site.data.reuse.task_openshift_select_operator}} +4. {{site.data.reuse.task_openshift_select_instance}} +5. Click on the **Resources** tab. +6. To filter only pods, deselect all resource types with the exception of **Pod**. +7. Click the pod _not_ in the **Running** state to display information regarding the pod. +8. In the **Overview**, resource usage, such as CPU and memory, can be viewed. + ![Example pod overview]({{ 'images' | relative_url }}/pod_overview.png "Example screen capture showing pod details with graphs for memory and CPU usage"){:height="100%" width="100%"} +9. Click on the **Logs** tab to search logs. + +**Tip:** You can also use the [cluster logging](https://docs.openshift.com/container-platform/4.16/observability/logging/cluster-logging-deploying.html){:target="_blank"} provided by the {{site.data.reuse.openshift_short}} to collect, store, and visualize logs. The cluster logging components are based upon Elasticsearch, Fluentd, and Kibana (EFK). You can download and install the pre-configured {{site.data.reuse.es_name}} Kibana dashboards by following the instructions in [monitoring cluster health](../cluster-health/). + +### Using the CLI + +1. To retrieve further details about the pods, including events affecting them, use the following command: + + ```shell + kubectl -n describe pod + ``` + +2. To retrieve detailed log data for a pod to help analyze problems, use the following command: + + ```shell + kubectl -n logs -c + ``` + +For more information about debugging, see the [Kubernetes documentation](https://kubernetes.io/docs/tasks/debug-application-cluster/debug-application-introspection/#using-kubectl-describe-pod-to-fetch-details-about-pods){:target="_blank"}. + +**Note:** After a component restarts, the `kubectl` command retrieves the logs for the new instance of the container. To retrieve the logs for a previous instance of the container, add the `-–previous` option to the `kubectl logs` command. diff --git a/_es_11.5/07-administering/02-cluster-health.md b/_es_11.5/07-administering/02-cluster-health.md new file mode 100644 index 00000000..f516fda9 --- /dev/null +++ b/_es_11.5/07-administering/02-cluster-health.md @@ -0,0 +1,101 @@ +--- +title: "Monitoring Kafka cluster health" +excerpt: "Understand the health of your Kafka cluster at a glance." +categories: administering +slug: cluster-health +toc: true +--- + +Monitoring the health of your Kafka cluster helps to verify that your operations are running smoothly. The {{site.data.reuse.es_name}} UI includes a [preconfigured dashboard](#viewing-the-preconfigured-dashboard) that monitors Kafka data. + +{{site.data.reuse.es_name}} also provides a number of ways to export metrics from your Kafka brokers to external monitoring and logging applications. These metrics are useful indicators of the health of the cluster, and can provide warnings of potential problems. You can use any monitoring solution that is compatible with Prometheus and JMX formats to collect, store, visualize, and set up alerts based on metrics provided by {{site.data.reuse.es_name}}. The following sections provide an overview of the available options. + +For information about the health of your topics, check the [producer activity](../topic-health/) dashboard. + +## JMX Exporter + +You can use {{site.data.reuse.es_name}} to collect JMX metrics from Kafka brokers, ZooKeeper nodes, and Kafka Connect nodes, and export them to Prometheus. + +For an example of how to configure the JMX exporter, see [configuring the JMX Exporter](../../installing/configuring#configuring-the-jmx-exporter). + +## Kafka Exporter + +You can use {{site.data.reuse.es_name}} to export Kafka metrics to Prometheus. These metrics are otherwise only accessible through the Kafka command-line tools. This allows topic metrics such as consumer group lag to be collected. + +For an example of how to configure a Kafka Exporter, see [configuring the Kafka Exporter](../../installing/configuring#configuring-the-kafka-exporter). + +## Grafana + +You can use [Grafana](https://grafana.com/docs/grafana/latest/){:target="_blank"} and configure the example [Grafana dashboards](http://ibm.biz/es-grafana-dashboards){:target="_blank"} to monitor the health and performance of your Kafka clusters by collecting and displaying metrics from Prometheus. + +## Kibana + +You can use the Kibana service that is provided by the {{site.data.reuse.openshift_short}} [cluster logging](https://docs.openshift.com/container-platform/4.16/observability/logging/cluster-logging.html){:target="_blank"}, and use the example [Kibana dashboards](https://github.com/IBM/ibm-event-automation/tree/master/event-streams/kibana-dashboards){:target="_blank"} to monitor for specific errors in the logs and set up alerts for when a number of errors occur over a period of time in your {{site.data.reuse.es_name}} instance. + +To install the {{site.data.reuse.es_name}} Kibana dashboards, follow these steps: + +1. Ensure you have [cluster logging](https://docs.openshift.com/container-platform/4.16/observability/logging/cluster-logging-deploying.html){:target="_blank"} installed. +2. Download the JSON file that includes the example Kibana dashboards for {{site.data.reuse.es_name}} from [GitHub](https://github.com/IBM/ibm-event-automation/tree/master/event-streams/kibana-dashboards){:target="_blank"}. + +3. Navigate to the Kibana homepage on your cluster. + + {{site.data.reuse.openshift_ui_login}} Then follow the instructions to navigate to [the cluster logging's Kibana homepage](https://docs.openshift.com/container-platform/4.16/observability/logging/log_visualization/logging-kibana.html#cluster-logging-visualizer-kibana_logging-kibana){:target="_blank"}. +4. Click **Management** in the navigation on the left. +5. Click **Index patterns**. +6. Click **Create index pattern**. +7. Enter `app*` in the **Index pattern** field, and click **Next step**. +8. Select `@timestamp` from the **Time Filter field name** list, and click **Create index pattern**. +9. Click **Saved Objects**. +10. Click the **Import** icon and navigate to the JSON file you downloaded earlier that includes the example Kibana dashboards for {{site.data.reuse.es_name}}. +11. If an `Index Pattern Conflicts` warning is displayed, select the `app*` index pattern from the **New index pattern** list for each conflict, then click **Confirm all changes**. +12. Click **Dashboard** in the navigation on the left to view the downloaded dashboards. + +You can also use Kibana on other Kubernetes platforms, where it might also be included, or you can install Kibana externally. You can use the example [Kibana dashboards](https://github.com/IBM/ibm-event-automation/tree/master/event-streams/kibana-dashboards){:target="_blank"} to monitor for specific errors in the logs and set up alerts for when a number of errors occur over a period of time in your {{site.data.reuse.es_name}} instance. + +## IBM Instana + +Instana is an observability tool that can be used to monitor your {{site.data.reuse.es_name}} deployment. + +Instana also offers [Kafka-centric monitoring](https://www.instana.com/supported-technologies/apache-kafka-observability/){:target="_blank"} that can provide useful insights into the performance and the health of your Kafka cluster. + +For information about installing and configuring an Instana host agent on the {{site.data.reuse.openshift}}, see the [Instana documentation for OpenShift](https://www.ibm.com/docs/en/instana-observability/current?topic=requirements-installing-host-agent-openshift){:target="_blank"}. To use Instana on other Kubernetes platforms, see the [Instana documentation for Kubernetes](https://www.ibm.com/docs/en/instana-observability/current?topic=requirements-installing-host-agent-kubernetes){:target="_blank"}. + +After installing, Instana can monitor all aspects of an {{site.data.reuse.es_name}} instance with no extra configuration required. + +**Note**: You might receive the following error message in the Instana dashboards when you check monitoring metrics for the {{site.data.reuse.es_name}} UI container: + +```shell +Monitoring issue: nodejs_collector_not_installed + +The @instana/collector package is not installed in this Node.js application, or the @instana/collector package cannot announce itself to the host agent, for example due to networking issues. +``` + +If you require monitoring of the {{site.data.reuse.es_name}} UI, you can enable Instana to monitor the UI by setting the following in the `EventStreams` custom resource: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: +# ... +adminUI: + env: + - name: INSTANA_AGENT_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP +``` + +## Other Monitoring Tools + +You can also use [external monitoring tools](../external-monitoring/) to monitor the deployed {{site.data.reuse.es_name}} Kafka cluster. + +## Viewing the preconfigured dashboard + +To get an overview of the cluster health, you can view a selection of metrics on the {{site.data.reuse.es_name}} **Monitoring** dashboard. + +**Important:** Ensure that you enable the **Monitoring** dashboard by following the instructions in the [post-upgrade tasks](../../installing/upgrading/#enable-metrics-for-monitoring) before accessing the dashboard. + +1. {{site.data.reuse.es_ui_login}} +2. Click **Monitoring** in the primary navigation. A dashboard is displayed with overview charts for messages, partitions, and replicas. +3. Select **1 hour**, **1 day**, **1 week**, or **1 month** to view data for different time periods. diff --git a/_es_11.5/07-administering/03-topic-health.md b/_es_11.5/07-administering/03-topic-health.md new file mode 100644 index 00000000..d2ce663b --- /dev/null +++ b/_es_11.5/07-administering/03-topic-health.md @@ -0,0 +1,32 @@ +--- +title: "Monitoring topic health" +excerpt: "Understand the health of your topics at a glance." +categories: administering +slug: topic-health +toc: false +--- + +To gain an insight into the overall health of topics, and highlight potential performance issues with systems producing to {{site.data.reuse.es_name}}, you can use the **Producers** dashboard provided for each topic. + +The dashboard displays aggregated information about producer activity for the selected topic through metrics such as active producer count, message size, and message produce rates. The dashboard also displays information about each producer that has been producing to the topic. + +You can expand an individual producer identified by its **Kafka producer ID** to gain insight into its performance through metrics such as messages produced, message size and rates, failed produce requests and any occurrences where a producer has exceeded a broker quota. + +The information displayed on the dashboard can also be used to provide insight into potential causes when applications experience issues such as delays or omissions when consuming messages from the topic. For example, highlighting that a particular producer has stopped producing messages, or has a lower message production rate than expected. + +**Important:** The **Producers** dashboard is intended to help highlight producers that might be experiencing issues producing to the topic. You might need to investigate the producer applications themselves to identify an underlying problem. + +To access the dashboard: + +**Note:** To gather producer metrics and display them in the **Producers** dashboard, you must [enable the Kafka Proxy](../../installing/configuring/#enabling-collection-of-producer-metrics). + +1. {{site.data.reuse.es_ui_login}} +2. Click **Topics** in the primary navigation. +3. Select the topic name from the list you want to view information about.\\ + The **Producers** tab is displayed with the dashboard and details about each producer. You can refine the time period for which information is displayed. You can expand each producer to view details about their activity. + + **Note:** When a new client starts producing messages to a topic, it might take up to 5 to 10 minutes before information about the producer's activity appears in the dashboard. In the meantime, you can go to the **Messages** tab to check whether messages are being produced. + +{{site.data.reuse.monitor_metrics_retention}} + +For information on how to monitor consumer groups for a particular topic, see [monitoring Kafka consumer group lag](../consumer-lag). diff --git a/_es_11.5/07-administering/04-consumer-group-lag.md b/_es_11.5/07-administering/04-consumer-group-lag.md new file mode 100644 index 00000000..57756ec8 --- /dev/null +++ b/_es_11.5/07-administering/04-consumer-group-lag.md @@ -0,0 +1,73 @@ +--- +title: "Monitoring Kafka consumer group lag" +excerpt: "Understand the health of your Kafka consumer clients through monitoring heuristics such as lag." +categories: administering +slug: consumer-lag +toc: true +--- + +You can monitor the consumer lag for Kafka clients connecting to {{site.data.reuse.es_name}}. This information is available through both the {{site.data.reuse.es_name}} UI and CLI. + +## Consumer lag + +Each partition will have a consumer within a consumer group with information relating to its consumption as follows: + +- **Client ID** and **Consumer ID**: each partition will have exactly one consumer in the consumer group, identifiable by a **Client ID** and **Consumer ID**. +- **Current offset**: the last committed offset of the consumer. +- **Log end offset**: the highest offset of the partition. +- **Offset lag**: the difference between the current consumer offset and the highest offset, which shows how far behind the consumer is. + +### Using the {{site.data.reuse.es_name}} UI + +To access the consumer groups side panel in the {{site.data.reuse.es_name}} UI, do the following: + +1. {{site.data.reuse.es_ui_login_nonadmin}} +2. Click **Topics** in the primary navigation. +3. Locate your topic using the **Name** column and click the row for the topic. +4. Click the **Consumer groups** tab. +5. The **Consumer groups** dashboard will display all consumer groups for the selected topic. + Click the consumer group of interest from the **Consumer group ID** column. + The consumer group side panel should now be displayed on screen. + +This side panel will display a table containing [consumer group information](#consumer-lag) for each partition of the topic. + +### Using the {{site.data.reuse.es_name}} CLI + +To access information about a consumer group in the {{site.data.reuse.es_name}} CLI, do the following: + +1. {{site.data.reuse.es_cli_init_111}} +2. To list all consumer groups on a cluster, run: + + ```shell + kubectl es groups + ``` + +3. To list information about a consumer group, run: + + ```shell + kubectl es group --group + ``` + + Where `` is the name of the consumer group of interest. + +The CLI will print a table containing [consumer group information](#consumer-lag) for each partition of the topic. + +The following example shows the information the command returns for a consumer group called `my-consumer`: + +```shell +$ kubectl es group --group my-consumer +> +Details for consumer group my-consumer +Group ID State +my-consumer Stable + +Topic Partition Current offset End offset Offset lag Client Consumer Host +my-topic 2 999 1001 2 some-client some-consumer some-host +my-topic 0 1000 1001 1 some-client some-consumer some-host +my-topic 1 1000 1000 0 some-client some-consumer some-host +OK +``` + +To view other Kafka-related metrics, consider configuring a [Kafka Exporter](../../installing/configuring/#configuring-the-kafka-exporter). + +For information on how to monitor the general health of a particular topic, see [monitoring topic health](../topic-health). diff --git a/_es_11.5/07-administering/05-distributed-tracing.md b/_es_11.5/07-administering/05-distributed-tracing.md new file mode 100644 index 00000000..0839512e --- /dev/null +++ b/_es_11.5/07-administering/05-distributed-tracing.md @@ -0,0 +1,93 @@ +--- +title: "Monitoring applications with distributed tracing" +excerpt: "Use the built-in tracing mechanism to monitor the flow of events, find performance issues, and pinpoint problems with applications using Event Streams." +categories: administering +slug: tracing +toc: true +--- + +{{site.data.reuse.es_name}} 10.3.0 and later versions are built on [Strimzi](https://strimzi.io/){:target="_blank"}. Strimzi 0.14.0 and later support distributed tracing based on the open-source [OpenTracing](https://opentracing.io/){:target="_blank"} and [Jaeger](https://www.jaegertracing.io/){:target="_blank"} projects. + +Distributed tracing provides a way to understand how applications work across processes, services, or networks. Tracing uses the trail they leave behind based on requests made, carrying tracing information in the requests as they pass from system to system. Through tracing, you can monitor applications that work with {{site.data.reuse.es_name}}, helping you to understand the shape of the generated traffic, and pinpoint the causes of any potential problems. + +To use distributed tracing for your Kafka client applications, you add code to your applications that gets called on the client's send and receive operations. This code adds headers to the client's messages to carry tracing information. + + +## OpenTelemetry tracing with Instana + +[OpenTelemetry (OTel)](https://opentelemetry.io/docs/){:target="_blank"} aims to simplify and automate the collection and analysis of telemetry data by providing a unified and extensible instrumentation framework that can be integrated into any application or infrastructure component. + + +Integrating OpenTelemetry with {{site.data.reuse.es_name}} can provide a powerful way to gain real-time insights into the performance and behavior of your event-driven applications. By streaming telemetry data to {{site.data.reuse.es_name}}, you can monitor the health of your applications, troubleshoot issues quickly, and optimize your applications for better performance. + + +[IBM Instana](https://www.ibm.com/docs/en/instana-observability/current){:target="_blank"} supports OpenTelemetry for collecting distributed traces, metrics, and logs from various sources. Instana can seamlessly integrate with OpenTelemetry and can also enhance the data from OpenTelemetry. As an observability backend for OpenTelemetry, use Instana to monitor and analyze the performance and health of your applications in real-time. + +There are two ways to ingest the generated OpenTelemetry traces and metrics with Instana: by using the Otel collector or by using an Instana host agent. + +**Note:** The OpenTelemetry operator available for the {{site.data.reuse.openshift_short}} does not support Instana as an exporter while using the Otel collector. To collect OTel trace details on an OpenShift deployment of {{site.data.reuse.es_name}}, use an Instana host agent as described in the following sections. + + +### Configuring the host agent and OpenTelemetry + +Instana host agent for OpenTelemetry is a powerful tool for collecting and analyzing telemetry data from hosts, infrastructure, and various applications. + + +Use the host agent to seamlessly integrate OpenTelemetry data into the Instana observability platform, providing a unified view of telemetry data, which you can use to analyze and troubleshoot issues quickly and efficiently. + +To configure the host agent and OpenTelemetry: +1. Install and configure an [Instana host agent](../../administering/cluster-health/#instana) for Event Streams. + + +2. [Configure OpenTelemetry](https://www.ibm.com/docs/en/instana-observability/current?topic=apis-opentelemetry#sending-otlp-data-to-instana-agent){:target="_blank"} to send tracing and metric data to the Instana host agent. + +### Enabling OpenTelemetry tracing + +You can enable OpenTelemetry tracing for the following components in {{site.data.reuse.es_name}}: +- Kafka Connect +- Kafka Bridge + +To enable OpenTelemetry tracing: +1. Enable OpenTelemetry by setting the `spec.tracing.type` property to `opentelemetry` in the custom resource for the selected component as follows (`KafkaConnect` and `KafkaBridge`): + + ```yaml + spec: + # ... + tracing: + type: opentelemetry + ``` + +2. Configure the tracing environment variables in `spec.template` for the required custom resource as follows: + + Configuration for Kafka Bridge: + ```yaml + spec: + #... + template: + bridgeContainer: + env: + - name: OTEL_SERVICE_NAME + value: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: + tracing: + type: opentelemetry + ``` + + Configuration for Kafka Connect: + ```yaml + spec: + #... + template: + connectContainer: + env: + - name: OTEL_SERVICE_NAME + value: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: + tracing: + type: opentelemetry + ``` + + +For more information about tracing in Apache Kafka and OpenTelemetry, see the [Strimzi documentation](https://strimzi.io/blog/2023/03/01/opentelemetry/){:target="_blank"}. diff --git a/_es_11.5/07-administering/05a-audit-logging.md b/_es_11.5/07-administering/05a-audit-logging.md new file mode 100644 index 00000000..dfa9cf9e --- /dev/null +++ b/_es_11.5/07-administering/05a-audit-logging.md @@ -0,0 +1,314 @@ +--- +title: "Auditing Kafka" +excerpt: "Use logs for creating an audit trail of activities within your Event Streams Kafka cluster." +categories: administering +slug: auditing-kafka +toc: true +--- + +{{site.data.reuse.es_name}} can be configured to generate a sequential record of activities within the {{site.data.reuse.es_name}} Kafka cluster. By reviewing this audit trail, administrators can track user activities and investigate security breaches. + +## Audit trail for Kafka + +{{site.data.reuse.es_name}} uses log records from every request that is sent to Kafka brokers, and extracts records for events that are useful for tracking activities within the broker. The information that is captured for each event can vary in structure and content, but they all include the following key information: + +- The user (Kafka principal) that initiated the request. +- The action (Kafka operation) that was requested. +- The entity (Kafka resource) on which the operation was requested. +- The date and time when the request was received. +- The result of the request, including relevant information about reasons for any failures. + +As Kafka is a distributed system, the audit events from all brokers must be aggregated in a central location to create a complete audit trail for the {{site.data.reuse.es_name}} Kafka cluster. Aggregating audit records in a separate log aggregator enables the retention and visualization of audit trails without impacting the storage requirements of the {{site.data.reuse.es_name}} instance. + +## Before you begin + +- Enabling audit trail for Kafka introduces additional processing that can impact the performance of the Kafka system. Ensure that the impact is assessed in your environment before enabling the audit feature. +- The storage that is used for the audit trail must be configured with appropriate size and retention policies to handle the volume of audit records. +- Ensure access to the audit trail is secured by restricting access only to authorized users. + +## Configuring audit trail for Kafka + +To configure {{site.data.reuse.es_name}} to provide audit trail for the Kafka cluster, complete the following steps: + +1. {{site.data.reuse.cncf_cli_login}} + +2. Create a file named `es-audit-config.yaml` and copy the following YAML content into the file to create the ConfigMap that has the log4j configuration properties: + + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: + namespace: + data: + log4j.properties: |- + + ``` + + Where: + + - `` is the namespace where your {{site.data.reuse.es_name}} instance is installed. + - `` is the log4j configuration to: + - Configure the Kafka root logger with default settings to have standard logs for your Kafka pods as follows: + + ``` + # Kafka root logger + log4j.rootLogger=INFO, stdout + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + ``` + + - Configure the Kafka request logger at `TRACE` level as the source for audit information, and set `additivity=false` to keep audit information separate from the standard logs for your Kafka pods as follows: + + ``` + # Kafka request logger + log4j.logger.kafka.request.logger=TRACE, audit + log4j.additivity.kafka.request.logger=false + ``` + + - Define an output destination by setting up an [appender](https://logging.apache.org/log4j/1.2/manual.html#Appenders_and_Layouts){:target="_blank"} to direct filtered audit records to a central system for aggregation, retention, and visualization. Ensure access to the audit trail is secured by restricting access only to authorized users. + + - Define a set of filters based on [Kafka API Keys (protocols)](https://kafka.apache.org/37/documentation/#operations_resources_and_protocols){:target="_blank"} to include or exclude specific requests in the audit trail. + + For sample log4j configurations, see the [example snippets](#example-log4j-configurations) later. + +3. Apply the ConfigMap by running the following command: + + ```shell + kubectl apply -f es-audit-config.yaml + ``` + +4. After the ConfigMap is created, Kafka auditing can be enabled by setting `spec.strimziOverrides.kafka.logging` property in the `EventStreams` custom resource to point to the `` ConfigMap. + + ```yaml + apiVersion: eventstreams.ibm.com/v1beta2 + kind: EventStreams + # ... + spec: + strimziOverrides: + kafka: + # ... + logging: + type: external + valueFrom: + configMapKeyRef: + key: log4j.properties + name: + ``` + +The {{site.data.reuse.es_name}} operator applies the previous changes to the Kafka pods one by one. After all Kafka pods have rolled successfully, the Kafka audit trail will be available in the central system configured for aggregation, retention, and visualization. + +## Example log4j configurations + +See the following log4j configuration examples for auditing purposes. These examples use Syslog for aggregating the audit records from all brokers. + +### Auditing all Kafka users and all topics + +Use the following log4j configuration to audit all Kafka users and all topics. + +``` +# Kafka root logger +log4j.rootLogger=INFO, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Kafka request logger +log4j.logger.kafka.request.logger=TRACE, audit +log4j.additivity.kafka.request.logger=false + +# Syslog Appender +log4j.appender.audit=org.apache.log4j.net.SyslogAppender +log4j.appender.audit.syslogHost=rsyslog-service.es-audit.svc.cluster.local:514 +log4j.appender.audit.facility=AUDIT +log4j.appender.audit.layout=org.apache.log4j.PatternLayout +log4j.appender.audit.layout.conversionPattern=%m%n + +# Accept requests for Create/Delete Topics +log4j.appender.audit.filter.1=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.1.StringToMatch="requestApiKeyName":"CREATE_TOPICS" +log4j.appender.audit.filter.1.AcceptOnMatch=true + +log4j.appender.audit.filter.2=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.2.StringToMatch="requestApiKeyName":"DELETE_TOPICS" +log4j.appender.audit.filter.2.AcceptOnMatch=true + +# Accept requests for Create/Delete User +log4j.appender.audit.filter.3=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.3.StringToMatch="requestApiKeyName":"CREATE_ACLS" +log4j.appender.audit.filter.3.AcceptOnMatch=true + +log4j.appender.audit.filter.4=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.4.StringToMatch="requestApiKeyName":"DELETE_ACLS" +log4j.appender.audit.filter.4.AcceptOnMatch=true + +# Accept requests for Alter Topic/User/Cluster +log4j.appender.audit.filter.5=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.5.StringToMatch="requestApiKeyName":"ALTER_CONFIGS" +log4j.appender.audit.filter.5.AcceptOnMatch=true + +log4j.appender.audit.filter.6=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.6.StringToMatch="requestApiKeyName":"INCREMENTAL_ALTER_CONFIGS" +log4j.appender.audit.filter.6.AcceptOnMatch=true + +# Deny All entries that do not match other filters +log4j.appender.audit.filter.7=org.apache.log4j.varia.DenyAllFilter +``` + +### Auditing all Kafka users and user-created topics + +Use the following log4j configuration to audit all Kafka users and only topics created by users (excluding internal {{site.data.reuse.es_name}} topics). + +``` +# Kafka root logger +log4j.rootLogger=INFO, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Kafka request logger +log4j.logger.kafka.request.logger=TRACE, audit +log4j.additivity.kafka.request.logger=false + +# SysLog Appender +log4j.appender.audit=org.apache.log4j.net.SyslogAppender +log4j.appender.audit.syslogHost=rsyslog-service.es-audit.svc.cluster.local:514 +log4j.appender.audit.facility=AUDIT +log4j.appender.audit.layout=org.apache.log4j.PatternLayout +log4j.appender.audit.layout.conversionPattern=%m%n + +# Reject internal topics +log4j.appender.audit.filter.1=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.1.StringToMatch="topics":[{"name":"__ +log4j.appender.audit.filter.1.AcceptOnMatch=false + +log4j.appender.audit.filter.2=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.2.StringToMatch="topics":[{"name":"eventstreams-apicurio-registry-kafkasql-topic" +log4j.appender.audit.filter.2.AcceptOnMatch=false + +# Accept requests for Create/Delete Topics +log4j.appender.audit.filter.3=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.3.StringToMatch="requestApiKeyName":"CREATE_TOPICS" +log4j.appender.audit.filter.3.AcceptOnMatch=true + +log4j.appender.audit.filter.4=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.4.StringToMatch="requestApiKeyName":"DELETE_TOPICS" +log4j.appender.audit.filter.4.AcceptOnMatch=true + +# Accept requests for Create/Delete User +log4j.appender.audit.filter.5=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.5.StringToMatch="requestApiKeyName":"CREATE_ACLS" +log4j.appender.audit.filter.5.AcceptOnMatch=true + +log4j.appender.audit.filter.6=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.6.StringToMatch="requestApiKeyName":"DELETE_ACLS" +log4j.appender.audit.filter.6.AcceptOnMatch=true + +# Accept requests for Alter Topic/User/Cluster +log4j.appender.audit.filter.7=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.7.StringToMatch="requestApiKeyName":"ALTER_CONFIGS" +log4j.appender.audit.filter.7.AcceptOnMatch=true + +log4j.appender.audit.filter.8=org.apache.log4j.varia.StringMatchFilter +log4j.appender.audit.filter.8.StringToMatch="requestApiKeyName":"INCREMENTAL_ALTER_CONFIGS" +log4j.appender.audit.filter.8.AcceptOnMatch=true + +# Deny All entries that do not match other filters +log4j.appender.audit.filter.9=org.apache.log4j.varia.DenyAllFilter +``` + +## Example audit records + +See the following examples for audit records of different actions and outcomes from within an audit trail. + +### Successful user creation + +The following audit record indicates an attempt to create a Kafka user that was successful: + +```json +{ + "requestHeader": { + "clientId": "adminclient-1", + "requestApiKeyName": "CREATE_ACLS" + ... + }, + "request": { + "creations": [ + { + "resourceType": 2, + "resourceName": "your.topic.name", + "resourcePatternType": 3, + "principal": "User:consumer", + "host": "*", + "operation": 3, + "permissionType": 3 + } + ... + ] + }, + "response": { + "results": [ + { + "errorCode": 0, + "errorMessage": "" + } + ... + ] + ... + }, + "securityProtocol": "SSL", + "principal": "User:CN=dev-scram-entity-user-operator,O=io.strimzi", + "listener": "REPLICATION-9091", + ... +} +``` + +### Failed topic creation + +The following audit record indicates an attempt to create a Kafka topic that has failed due to user authorization failure: + +```json +{ + "requestHeader": { + "clientId": "adminclient-2955", + "requestApiKeyName": "CREATE_TOPICS", + ... + }, + "request": { + "topics": [ + { + "name": "aaaa", + "numPartitions": 1, + "replicationFactor": 1, + "assignments": [], + "configs": [ + { + "name": "min.insync.replicas", + "value": "1" + }, + ... + ] + } + ] + ... + }, + "response": { + "throttleTimeMs": 0, + "topics": [ + { + "name": "aaaa", + "topicId": "AAAAAAAAAAAAAAAAAAAAAA", + "errorCode": 29, + "errorMessage": "Authorization failed.", + ... + } + ] + }, + "securityProtocol": "SASL_SSL", + "principal": "User:user2", + "listener": "EXTERNAL-9094", + ... +} +``` + diff --git a/_es_11.5/07-administering/06-external-monitoring.md b/_es_11.5/07-administering/06-external-monitoring.md new file mode 100644 index 00000000..43ce67f2 --- /dev/null +++ b/_es_11.5/07-administering/06-external-monitoring.md @@ -0,0 +1,15 @@ +--- +title: "Monitoring with external tools" +excerpt: "You can use third-party monitoring tools to monitor your Event Streams Kafka cluster." +categories: administering +slug: external-monitoring +toc: true +--- + +You can use third-party monitoring tools to monitor the deployed {{site.data.reuse.es_name}} Kafka cluster by connecting to the JMX port on the Kafka brokers and reading Kafka metrics. + +You must [configure](../../installing/configuring/#configuring-external-monitoring-through-jmx) your installation to set up access for external monitoring tools. + +For examples about setting up monitoring with external tools such as Datadog, Prometheus, and Splunk, see the [tutorials]({{ 'tutorials' | relative_url }}) page. + +If you have a tool or service you want to use to monitor your clusters, you can [contact support]({{ 'support' | relative_url }}). diff --git a/_es_11.5/07-administering/07-modifying-installation.md b/_es_11.5/07-administering/07-modifying-installation.md new file mode 100644 index 00000000..03c32e36 --- /dev/null +++ b/_es_11.5/07-administering/07-modifying-installation.md @@ -0,0 +1,67 @@ +--- +title: "Modifying installation settings" +excerpt: "Modify your existing Event Streams installation." +categories: administering +slug: modifying-installation +toc: true +--- + +You can modify the configuration settings for your existing {{site.data.reuse.es_name}} installation by using the {{site.data.reuse.openshift_short}} web console or the Kubernetes command-line tool (`kubectl`). The configuration changes are applied by updating the `EventStreams` custom resource. +You can modify existing values and introduce new properties as outlined under [configuration settings](../../installing/configuring). + +**Note:** Some settings might cause affected components of your {{site.data.reuse.es_name}} instance to restart. + +For examples of changes you might want to make for performance reasons, see [scaling your {{site.data.reuse.es_name}} instance](../scaling/). + +## Using the {{site.data.reuse.openshift_short}} web console + +To modify configuration settings by using the {{site.data.reuse.openshift_short}} web console: +1. {{site.data.reuse.openshift_ui_login}} +2. {{site.data.reuse.task_openshift_navigate_installed_operators}} +3. {{site.data.reuse.task_openshift_select_operator}} +4. {{site.data.reuse.task_openshift_select_instance}} +5. Click the **YAML** tab to edit the custom resource. +6. Make the required changes on the page, or you can click **Download** and make the required changes in a text editor. + If you clicked **Download** you will need to drag and drop the modified custom resource file onto the page so that it updates in the web console. +7. Click the **Save** button to apply your changes. + + +## Using the Kubernetes command-line tool (`kubectl`) + +To modify configuration settings by using the Kubernetes command-line tool (`kubectl`): +1. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to edit your `EventStreams` custom resource in your default editor: + + ```shell + kubectl edit eventstreams + ``` + +3. Make the required changes in your editor. +4. Save and quit the editor to apply your changes. + + +## Modifying Kafka broker configuration settings + +Kafka supports a number of [key/value pair settings](http://kafka.apache.org/37/documentation/#brokerconfigs){:target="_blank"} for broker configuration, typically provided in a properties file. + +In {{site.data.reuse.es_name}}, these settings are defined in an `EventStreams` custom resource under the `spec.strimziOverrides.kafka.config` property. + +For example, to set the number of I/O threads to `24` you can add the `spec.strimziOverrides.kafka.config["num.io.threads"]` property: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +metadata: + name: example-broker-config + namespace: myproject +spec: + # ... + strimziOverrides: + kafka: + # ... + config: + # ... + num.io.threads: 24 +``` + +You can specify all the broker configuration options supported by Kafka except those managed directly by {{site.data.reuse.es_name}}. For further information, see the list of [supported configuration options](https://strimzi.io/docs/operators/0.42.0/configuring.html#type-KafkaClusterSpec-reference){:target="_blank"}. diff --git a/_es_11.5/07-administering/08-cruise-control.md b/_es_11.5/07-administering/08-cruise-control.md new file mode 100644 index 00000000..db7c97e7 --- /dev/null +++ b/_es_11.5/07-administering/08-cruise-control.md @@ -0,0 +1,376 @@ +--- +title: "Optimizing Kafka cluster with Cruise Control" +excerpt: "Use Cruise Control to manage your brokers and topic partitions." +categories: administering +slug: cruise-control +toc: true +--- + +## Overview + +Cruise Control is an open-source system for optimizing your Kafka cluster by monitoring cluster workload, rebalancing a cluster based on predefined constraints, and detecting and fixing anomalies. + +![Event Streams cruise control architecture]({{ 'images' | relative_url }}/architectures/ibm-event-automation-diagrams-es-cruisecontrol.svg "Diagram showing the architecture of the Event Streams Cruise Control as part of IBM Event Automation.") + +You can set up {{site.data.reuse.es_name}} to use the following Cruise Control features: + +- Generating optimization proposals from multiple optimization goals. +- Rebalancing a Kafka cluster based on an optimization proposal. + +**Note:** {{site.data.reuse.es_name}} does not support other Cruise Control features. + + + +Cruise Control can be used to dynamically optimize the distribution of the partitions on your brokers so that resources are used more efficiently. + +Cruise Control reduces the time and effort involved in running an efficient and balanced Kafka cluster. + +A typical cluster can become unevenly loaded over time. Partitions that handle large amounts of message traffic might be unevenly distributed across the available brokers. To rebalance the cluster, administrators must monitor the load on brokers and manually reassign busy partitions to brokers with spare capacity. + +Cruise Control automates the cluster rebalancing process. It constructs a workload model of resource utilization for the cluster based on CPU, disk, and network load, ​and generates optimization proposals (that you can approve or reject) for more balanced partition assignments. A set of configurable optimization goals is used to calculate these proposals. + +When you approve an optimization proposal, Cruise Control applies it to your Kafka cluster. When the cluster rebalancing operation is complete, the broker pods are used more effectively and the Kafka cluster is more evenly balanced. + + + +### Steps for rebalancing a cluster + +Follow these steps to rebalance a cluster by using Cruise Control: + +1. Ensure you have an {{site.data.reuse.es_name}} installation that has Cruise Control [enabled](../../installing/configuring/#enabling-and-configuring-cruise-control) in the `EventStreams` custom resource. +2. In your `EventStreams` custom resource, configure the optimization goals that are available for rebalancing, and set capacity limits for broker resources. You can use Cruise Control defaults or define your own goals and capacity limits, as described in [enabling and configuring Cruise Control](../../installing/configuring/#enabling-and-configuring-cruise-control). + + **Important:** The configuration settings in the `EventStreams` custom resource are used by all optimization proposals defined in the `KafkaRebalance` custom resource, and can constrain the possible proposals a user can make. + +3. [Set up the optimization proposal](#setting-up-optimization) by using the `KafkaRebalance` custom resource. + + **Important:** Deploying Cruise Control allows for rebalancing a cluster based on the predefined constraints defined in the `KafkaRebalance` custom resource. If Cruise Control is not enabled, the `KafkaRebalance` custom resource has no effect on a cluster, and similarly, without a `KafkaRebalance` custom resource Cruise Control will make no changes to a cluster even if it is enabled. + +4. Wait for an [optimization proposal](#receiving-an-optimization-proposal) to appear in the status of the `KafkaRebalance` custom resource. +5. [Accept the optimization proposal](#approving-an-optimization-proposal) in the `KafkaRebalance` custom resource by annotating it. +6. Check for the `KafkaRebalance` proposal to be finished processed. +7. Depending on the outcome of the `KafkaRebalance` proposal, perform the following steps: + - If **`status.condtions[0].status` = Ready:** Cruise Control has successfully optimized the Kafka cluster and no further action is needed. + - If **`status.condtions[0].status` = NotReady:** Cruise Control was unable to optimize the Kafka cluster and you need to [address the error](#dealing-with-rebalancing-errors) + +## Setting up optimization + +To rebalance a Kafka cluster, Cruise Control uses the [available](../../installing/configuring/#enabling-and-configuring-cruise-control) optimization goals to generate optimization proposals, which you can approve or reject. + +### Creating an optimization proposal + +To define the [goals](#optimization-goals) to use when rebalancing the cluster, use the `KafkaRebalance` custom resource. The goals defined in the custom resource determine how the cluster will be optimized during the calculation of the `KafkaRebalance` proposal. + +**Important:** The [hard goals](../../installing/configuring/#hard-goals) that are configured in `spec.strimziOverrides.config["hard.goals"]` in the `EventStreams` custom resource must be satisfied, whereas the goals defined in the `KafkaRebalance` custom resource will be optimized if possible, but not at the cost of violating any of the hard goals. + +To create a `KafkaRebalance` custom resource that creates a proposal that does not consider the hard goals set in `spec.strimziOverrides.cruiseControl.config["hard.goals"]` in the `EventStreams` custom resource, set `spec.skipHardGoalsCheck` to true in the `KafkaRebalance` custom resource. + +Hard goals include settings that must be met by an optimization proposal and cannot be ignored. Other goals in the `KafkaRebalance` custom resource are optimized only if possible. + +To create a `KafkaRebalance` custom resource for your {{site.data.reuse.es_name}} instance, add the `eventstreams.ibm.com/cluster=` label to `metadata.labels` in your `KafkaRebalance` custom resource as follows, where `` is the name of your {{site.data.reuse.es_name}} cluster. + +```yaml +# ... +metadata: + # ... + labels: + eventstreams.ibm.com/cluster: +spec: + # ... + goals: + - NetworkInboundCapacityGoal + - NetworkOutboundCapacityGoal + skipHardGoalCheck: true +``` + +To add a list of goals to your `KafkaRebalance` custom resource, add the subset of [defined goals](#optimization-goals) to the `spec.goals` property. The previous example has the `NetworkInboundCapacityGoal` and `NetworkOutboundCapacityGoal` goals added. + +To configure a `KafkaRebalance` custom resource, use the {{site.data.reuse.openshift_short}} web console or the Kubernetes command-line tool (`kubectl`) as follows. + + +#### Using the {{site.data.reuse.openshift_short}} web console + +1. {{site.data.reuse.openshift_ui_login}} +2. From the navigation menu, click **Operators > Installed Operators**. +3. In the **Projects** dropdown list, select the project that contains the {{site.data.reuse.es_name}} instance. +4. Select the {{site.data.reuse.es_name}} operator in the list of installed operators. +5. In the **Operator Details > Overview** page, find the **KafkaRebalance** tile in the list of **Provided APIs** and click **Create Instance**. +6. In the **Create KafkaRebalance** page, edit the provided YAML to set values for the following properties. + - In the **metadata.labels** section, set the **eventstreams.ibm.com/cluster** property value to the name of your {{site.data.reuse.es_name}} instance that you want to rebalance. + - Set the `metadata.name` property value to what you want to call the `KafkaRebalance` custom resource. + - Set the `spec.goals` to provide a list of goals you want to optimize. + - Set the `spec.skipHardGoalCheck` to provide true if you want to skip the hard goals. + + See the previous YAML snippet as an example. +7. Click **Create**. +8. The new `KafkaRebalance` custom resource is listed in the **Operator Details > KafkaRebalance** page. + +#### Using the CLI + +1. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to select the namespace that contains the existing {{site.data.reuse.es_name}} cluster: + + ```shell + kubectl config set-context --current --namespace= + ``` + +3. Define a `KafkaRebalance` custom resource in a file. For example, the following YAML defines a `KafkaRebalance` custom resource that will rebalance the Kafka cluster associated with the {{site.data.reuse.es_name}} instance named `my-cluster` and will create a proposal to satisfy the `NetworkInboundCapacityGoal` goal. + + + ```yaml + apiVersion: eventstreams.ibm.com/v1alpha1 + kind: KafkaRebalance + metadata: + labels: + eventstreams.ibm.com/cluster: my-cluster + backup.eventstreams.ibm.com/component: kafkarebalance + name: my-test-kafka-rebalance + spec: + goals: + - NetworkInboundCapacityGoal + ``` + + **Note:** The `KafkaRebalance` must have a label under `metadata.labels` with the key `eventstreams.ibm.com/cluster`, and the value must be set to the name of the {{site.data.reuse.es_name}} instance that you are rebalancing. + + + +4. Run the following command to create the `KafkaRebalance` custom resource: + + ```shell + kubectl create -f + ``` + +5. Verify the `KafkaRebalance` custom resource has been created by running: + + + ```shell + kubectl get kafkarebalances + ``` + + Ensure the `KafkaRebalance` custom resource you are trying to create is listed. +6. To view the `KafkaRebalance` proposal by running: + + ```shell + kubectl get KafkaRebalance -o yaml + ``` + +### Receiving an optimization proposal + +After you have created the `KafkaRebalance` custom resource, Cruise Control creates an optimization proposal if it can, and will add the `ProposalReady` status to the `status.conditions` property and display an overview of the proposal in the `status.optimizationResult` field. + +The following is an example of a successful proposal: + +```yaml +# ... +status: + # ... + conditions: + - lastTransitionTime: '2020-07-07T08:36:09.193Z' + status: ProposalReady + type: State + observedGeneration: 5 + optimizationResult: + intraBrokerDataToMoveMB: 0 + onDemandBalancednessScoreBefore: 0 + recentWindows: 1 + dataToMoveMB: 3 + excludedTopics: [] + excludedBrokersForReplicaMove: [] + numReplicaMovements: 7 + onDemandBalancednessScoreAfter: 100 + numLeaderMovements: 0 + monitoredPartitionsPercentage: 100 + numIntraBrokerReplicaMovements: 0 + excludedBrokersForLeadership: [] + sessionId: 03974f67-b208-4133-9f54-305d268a1a22 +``` + +For more information about optimization proposals, see the Strimzi [documentation](https://strimzi.io/docs/operators/0.42.0/deploying.html#con-optimization-proposals-str){:target="_blank"}. + +### Refreshing an optimization proposal + +A `KafkaRebalance` custom resource does not automatically refresh the optimization proposal, it requires a manual refresh to be triggered. This is to ensure that a proposal does not change without your permission, and also to ensure the proposal does not change before it is approved. + +To refresh the optimization proposal, you will need to add `eventstreams.ibm.com/rebalance: refresh` to the `KafkaRebalance` custom resource as described in [adding an annotation](#adding-annotations-to-a-kafkarebalance-custom-resource). + +If a valid optimization exists for the configuration and the goals specified, then the `KafkaRebalance` custom resource will get updated, and `status.optimizationResult` shows the updated proposal. + +### Approving an optimization proposal + +Cruise Control is used for generating and implementing Kafka rebalance proposals. The `KafkaRebalance` custom resource is the way a user interacts with Cruise Control via the operator. The operator has automated the process of generating a proposal request, but to approve the proposal a user needs to annotate their `KafkaRebalance` custom resource. + +If the `KafkaRebalance` custom resource contains a rebalance proposal that you want to accept, then you will need to approve the proposal by adding the `eventstreams.ibm.com/rebalance: approve` annotation to your `KafkaRebalance` custom resource by [adding an annotation](#adding-annotations-to-a-kafkarebalance-custom-resource) + +When approved, the `KafkaRebalance` custom resource is updated to show the status `Rebalancing` in `status.conditions`. This means that Cruise Control is currently rebalancing the Kafka cluster. The amount of time required for a rebalance depends on various factors such as the amount of data on a partition and the goals being accomplished. If the rebalance is successful, then the `KafkaRebalance` custom resource will have the status updated to `Ready` in the `status.conditions` field. + +#### Dealing with rebalancing errors + +If Cruise Control is unsuccessful in rebalancing the Kafka brokers, then the `KafkaRebalance` custom resource will be updated to show the `NotReady` status in the `status.conditions` field, and `status.conditions` will contain the error and how to remediate. After you have followed the guidance to fix the error, trigger a [KafkaRebalance refresh](#refreshing-an-optimization-proposal) and then [approve the KafkaRebalance proposal](#approving-an-optimization-proposal). + +### Stopping an inflight optimization proposal + +If you want to stop the process of rebalancing a Kafka cluster, you can add the `eventstreams.ibm.com/rebalance: stop` annotation by [adding an annotation](#adding-annotations-to-a-kafkarebalance-custom-resource). + +**Note:** Cruise Control will not restore the state of the topics and partitions that existed before rebalancing. This means the state of the Kafka brokers might not be the same as before rebalancing started. + +### Adding annotations to a `KafkaRebalance` custom resource + +To add annotations to the `KafkaRebalance` custom resource, use the {{site.data.reuse.openshift_short}} web console or the Kubernetes command-line tool (`kubectl`) as follows. + +#### Using the {{site.data.reuse.openshift_short}} web console + +1. {{site.data.reuse.openshift_ui_login}} +2. The new `KafkaRebalance` custom resource is listed in the **Operator Details > KafkaRebalance** page. +3. Click the name of the `KafkaRebalance` custom resource you want to edit. +4. Click the **YAML** tab. +5. Add the `eventstreams.ibm.com/rebalance` annotation to the `metadata.annotations` field. + + ```yaml + # ... + metadata: + # ... + annotations: + eventstreams.ibm.com/rebalance: + ``` + +6. Click **Save**. + +#### Using the CLI + +1. {{site.data.reuse.cncf_cli_login}} +2. Run the following command to select the namespace that contains the existing `KafkaRebalance` custom resource: + + ```shell + kubectl config set-context --current --namespace= + ``` + +3. Run the following command to see a list of all the `KafkaRebalance` custom resources: + + ```shell + kubectl get kafkarebalances + ``` + +4. Find the name of the `KafkaRebalance` custom resource you want to annotate. +5. Run the following command to view the YAML for your `KafkaRebalance` instance: + + ```shell + kubectl annotate kafkarebalances = + ``` + +## Optimization goals + +Optimization goals determine what the `KafkaRebalance` proposal chooses to optimize when rebalancing the Kafka cluster. + +**Note:** The more goals you use, the harder it will be for Cruise Control to create an optimization proposal that will satisfy all of the goals. Consider creating a `KafkaRebalance` custom resource with fewer goals. + +**Important:** Most goals might result in the moving of partitions across brokers. This might impact performance during the operation, and cause issues with connecting clients. + +You can configure the following optimization goals. the goals are listed in **order of decreasing priority**. + +### RackAwareGoal + +Ensures that all replicas of each partition are assigned in a rack-aware manner. This means no more than one replica of each partition resides on the same rack. + +Using it in a multi-zone environment ensures that partitions are spread across different zones. This will improve the availability of topic data. + +### ReplicaCapacityGoal + +Ensures that the maximum number of replicas per broker is under the specified maximum limit. + +This goal does not ensure that the replicas are evenly distributed across brokers. It ensures that the total replicas on a single broker remains under a specified maximum limit. + +For even distribution of replicas amongst brokers, see [ReplicaDistributionGoal](#replicadistributiongoal). + +### DiskCapacityGoal + +Ensures that disk space usage of each broker is below the threshold set in the `spec.strimziOverrides.cruiseControl.brokerCapacity.disk` property in the `EventStreams` custom resource. + +Use this goal to optimize the distribution of partitions amongst brokers based on the disk usage of a partition. This might not lead to an even distribution of partitions amongst the brokers. For even distribution of disk space usage on brokers, see [DiskUsageDistributionGoal](#diskusagedistributiongoal). + +### NetworkInboundCapacityGoal + +Ensures that inbound network utilization of each broker is below the threshold set in the `spec.strimziOverrides.cruiseControl.brokerCapacity.inboundNetwork` property in the `EventStreams` custom resource. + +This is mainly affected by the number of producers producing and the size of produced messages. Use this goal to rebalance the partitions to optimize the distribution of the partitions on the inbound network traffic. This might not result in the inbound network being evenly distributed amongst brokers, it only ensures that the inbound network of each broker is under a given threshold. For even distribution of inbound network traffic amongst brokers, see [NetworkInboundDistributionUsageGoal](#networkinboundusagedistributiongoal). + +### NetworkOutboundCapacityGoal + +Ensures that outbound network utilization of each broker is below a given threshold set in the `spec.strimziOverrides.cruiseControl.brokerCapacity.inboundNetwork` property in the `EventStreams` custom resource. + +This is mainly affected by the number of consumers consuming and communication between the Kafka Brokers. Use this goal to rebalance the partitions to optimize the distribution of the partitions on the outbound network traffic. This might not result in the outbound network being evenly distributed amongst brokers, it only ensures that the outbound network of each broker is under a given threshold. For even distribution of outbound network traffic amongst brokers, see [NetworkOutboundDistributionUsageGoal](#networkoutboundusagedistributiongoal). + +### ReplicaDistributionGoal + +Attempts to make all the brokers in a cluster have a similar number of replicas. + +Use this goal to ensure that the total number of partition replicas are distributed evenly amongst brokers. This does not mean that partitions will be optimally distributed for other metrics such as network traffic and disk usage, so it might still cause brokers to be unevenly loaded. + +### PotentialNwOutGoal + +Ensures that the potential network output (when all the replicas in the broker become leaders) on each of the broker do not exceed the broker’s network outbound bandwidth capacity. + +This is mainly affected by the number of consumers consuming and the size of the consumed messages. Use this goal to rebalance the partitions to optimize the distribution of the partitions on the potential outbound network traffic. This will ensure that the partitions with the most outbound traffic are moved to the most idle brokers. + +The difference between this goal and [NetworkOutboundCapacityGoal](#networkoutboundcapacitygoal) is that this goal will rebalance the cluster based on the worst-case network outbound usage around the outbound network capacity specified, and the `NetworkOutboundCapacityGoal` will rebalance the outbound network usage around the outbound network capacity specified. + +### DiskUsageDistributionGoal + +Attempts to keep the disk space usage variance among brokers within a certain range relative to the average disk utilization. + +Use this goal to optimize the distribution of partitions amongst brokers to ensure that the disk utilization amongst brokers are similar. + +### NetworkInboundUsageDistributionGoal + +Attempts to keep the inbound network utilization variance among brokers within a certain range relative to the average inbound network utilization. + +This is mainly affected by the number of producers producing and the size of produced messages. Use this goal to rebalance the partitions to optimize the distribution of the partitions on the inbound network traffic. This will ensure that the partitions with the most inbound traffic are moved to the most idle brokers and the inbound network traffic amongst brokers are similar. + +### NetworkOutboundUsageDistributionGoal + +Attempts to keep the outbound network utilization variance among brokers within a certain range relative to the average outbound network utilization. + +This is mainly affected by the number of consumers consuming and the size of the consumed messages. Use this goal to rebalance the partitions to optimize the distribution of the partitions on the outbound network traffic. This will ensure that the partitions with the most outbound traffic are moved to the most idle brokers and the outbound network traffic amongst brokers are similar. + +### CpuUsageDistributionGoal + +Attempts to keep the CPU usage variance among brokers within a certain range relative to the average CPU utilization. + +Use this goal to rebalance the partitions to optimize the distribution of the partitions on the CPU usage of brokers. This will ensure that the CPU usage of brokers are similar. + +### LeaderReplicaDistributionGoal + +Attempts to make all the brokers in a cluster have a similar number of leader replicas. + +Use this goal to ensure the total number of partitions leaders are similar amongst brokers. + +### LeaderBytesInDistributionGoal + +Attempts to equalize the leader bytes in rate on each host. + +This goal may not result in an even distribution of replicas across brokers as it is optimizing the leader bytes on each broker. + +### TopicReplicaDistributionGoal + +Attempts to maintain an even distribution of any topic's partitions across the entire cluster. + +Use this goal to ensure that the total number of partition replicas are distributed evenly amongst brokers. This does not mean that partitions will be optimally distributed for other metrics such as network traffic and disk usage, so might cause brokers to be unevenly loaded. This might also cause a partition on a topic to be present on a subset of the total brokers. + +### PreferredLeaderElectionGoal + +Simply moves the leaders to the first replica of each partition. + +Cruise Control does not try to optimize the distribution of replicas. Before using this goal, check the state of the partitions to see where the leaders will end up to ensure the required outcome. + +### IntraBrokerDiskCapacityGoal + +Ensures that disk space usage of each disk is below a given threshold. + +Use this goal to rebalance the distribution of the partitions on disk space usage under a given threshold. This may not result in the disk space usage being evenly distributed amongst brokers, it only ensures that the disk space usage of each broker is under a given threshold. + +For even distribution of disk space usage amongst brokers, see [IntraBrokerDiskUsageDistributionGoal](#intrabrokerdiskusagedistributiongoal). + +### IntraBrokerDiskUsageDistributionGoal + +Attempts to keep the disk space usage variance among disks within a certain range relative to the average broker disk utilization. + +Use this goal to rebalance the distribution of the partitions on disk space usage. This goal will ensure that the disk space usage amongst brokers are similar. diff --git a/_es_11.5/07-administering/09-scaling.md b/_es_11.5/07-administering/09-scaling.md new file mode 100644 index 00000000..48810d66 --- /dev/null +++ b/_es_11.5/07-administering/09-scaling.md @@ -0,0 +1,169 @@ +--- +title: "Scaling" +excerpt: "Modify the capacity of your system by scaling it." +categories: administering +slug: scaling +toc: true +--- + +You can modify the capacity of your {{site.data.reuse.es_name}} system in a number of ways. See the following sections for details about the different methods, and their impact on your installation. + +The [pre-requisite](../../installing/prerequisites/) guidance gives various examples of different production configurations on which you can base your deployment. To verify it meets your requirements, you should test the system with a workload that is representative of the expected throughput. For this purpose, {{site.data.reuse.es_name}} provides a [workload generator application](../../getting-started/testing-loads/) to test different message loads. + +If this testing shows that your system does not have the capacity needed for the workload, whether this results in excessive lag or delays, or more extreme errors such as `OutOfMemory` errors, then you can incrementally make the increases detailed in the following sections, re-testing after each change to identify a configuration that meets your specific requirements. + +A [performance report]({{ 'pdfs' | relative_url }}/11.0.4 Performance Report_v3.pdf){:target="_blank"} based on example case studies is also available to provide guidance for setting these values. + +**Note:** Although the testing for the report was based on Apache Kafka version 2.3.0, the performance numbers are broadly applicable to current versions of Kafka as well. + +## Modifying the settings + +These settings are defined in the `EventStreams` custom resource under the `spec.strimziOverrides` property. For more information on modifying these settings see [modifying installation](../modifying-installation). + +## Increase the number of Kafka brokers in the cluster + +The number of Kafka brokers is defined in the `EventStreams` custom resource in the `spec.strimziOverrides.kafka.replicas` property. For example to configure {{site.data.reuse.es_name}} to use 6 Kafka brokers: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + # ... + replicas: 6 +``` + + +## Increase the CPU request or limit settings for the Kafka brokers + +The CPU settings for the Kafka brokers are defined in the `EventStreams` custom resource in the `requests` and `limits` properties under `spec.strimziOverrides.kafka.resources`. For example to configure {{site.data.reuse.es_name}} Kafka brokers to have a CPU request set to 2 CPUs and limit set to 4 CPUs: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + # ... + resources: + requests: + cpu: 2000m + limits: + cpu: 4000m +``` + +A description of the syntax for these values can be found in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu){:target="_blank"}. + + +## Increase the memory request or limit settings for the Kafka brokers and ZooKeeper nodes + +The memory settings for the Kafka brokers are defined in the `EventStreams` custom resource in the `requests` and `limits` properties under `spec.strimziOverrides.kafka.resources`. + +The memory settings for the ZooKeeper nodes are defined in the `EventStreams` custom resource in the `requests` and `limits` properties under `spec.strimziOverrides.zookeeper.resources`. + +For example to configure {{site.data.reuse.es_name}} Kafka brokers and ZooKeeper nodes to have a memory request set to `4GB` and limit set to `8GB`: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + # ... + resources: + requests: + memory: 4096Mi + limits: + memory: 8096Mi + zookeeper: + # ... + resources: + requests: + memory: 4096Mi + limits: + memory: 8096Mi +``` + +The syntax for these values can be found in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory){:target="_blank"}. + + + +## Modifying the resources available to supporting components + +The resource settings for each supporting component are defined in the `EventStreams` custom resource in their corresponding component key the `requests` and `limits` properties under `spec..resources`. +For example, to configure the Apicurio Registry to have a memory request set to `4GB` and limit set to `8GB`: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + apicurioRegistry: + # ... + resources: + requests: + memory: 4096Mi + limits: + memory: 8096Mi +``` + +The syntax for these values can be found in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory){:target="_blank"}. + + +## Modifying the JVM settings for Kafka brokers + +If you have specific requirements, you can modify the JVM settings for the Kafka brokers. + +**Note:** Take care when modifying these settings as changes can have an impact on the functioning of the product. + +**Note:** Only a [selected subset](https://strimzi.io/docs/operators/0.42.0/configuring.html#con-common-configuration-jvm-reference){:target="_blank"} of the available JVM options can be configured. + +JVM settings for the Kafka brokers are defined in the `EventStreams` custom resource in the `spec.strimziOverrides.kafka.jvmOptions` property. For example: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + # ... + jvmOptions: + -Xms: 4096m + -Xmx: 4096m +``` + + +## Increase the disk space available to each Kafka broker + +The Kafka brokers need sufficient storage to meet the retention requirements for all of the topics in the cluster. Disk space requirements grow with longer retention periods for messages, increased message sizes and additional topic partitions. + +The amount of storage made available to Kafka brokers is defined at the time of installation in the `EventStreams` custom resource in the `spec.strimziOverrides.kafka.storage.size` property. For example: + +```yaml +apiVersion: eventstreams.ibm.com/v1beta2 +kind: EventStreams +# ... +spec: + # ... + strimziOverrides: + # ... + kafka: + # ... + storage: + # ... + size: 100Gi +``` diff --git a/_es_11.5/07-administering/10-quotas.md b/_es_11.5/07-administering/10-quotas.md new file mode 100644 index 00000000..e2086310 --- /dev/null +++ b/_es_11.5/07-administering/10-quotas.md @@ -0,0 +1,238 @@ +--- +title: "Setting client quotas" +excerpt: "Setting quotas" +categories: administering +slug: quotas +toc: true +--- + +Kafka quotas enforce limits on produce and fetch requests to control the broker resources used by clients. + +Using quotas, administrators can throttle client access to the brokers by imposing network bandwidth or data limits, or both. + +## About Kafka quotas + +In a collection of clients, quotas protect from any single client producing or consuming significantly larger amounts of data than the other clients in the collection. This prevents issues with broker resources not being available to other clients, DoS attacks on the cluster, or badly behaved clients impacting other users of the cluster. + +After a client that has a quota defined reaches the maximum amount of data it can send or receive, their throughput is stopped until the end of the current quota window. The client automatically resumes receiving or sending data when the quota window of 1 second ends. + +By default, clients have unlimited quotas. + +For more information about quotas, see the [Kafka documentation](https://kafka.apache.org/37/documentation/#design_quotas){:target="_blank"}. + +## Setting user quotas + +To configure Kafka user quotas, either update an existing `KafkaUser` resource, or manually create a new `KafkaUser` by using the {{site.data.reuse.openshift_short}} web console or the Kubernetes command-line tool (`kubectl`), ensuring that the configuration has a `quotas` section with the relevant quotas defined. + +For example: + +```yaml +spec: + #... + quotas: + producerByteRate: 1048576 + consumerByteRate: 2097152 + requestPercentage: 55 +``` + +Decide what you want to limit by defining one or more of the following quota types: + +| Property | Type | Description | +|:-----------------|:-----------------|:-----------------| +| `producerByteRate` | integer | This quota limits the number of bytes that a producer application is allowed to send per second. | +| `consumerByteRate` | integer | This quota limits the number of bytes that a consumer application is allowed to receive per second. | +| `requestPercentage` | integer | This quota limits all clients based on [thread utilization](https://kafka.apache.org/37/documentation/#design_quotascpu){:target="_blank"}. | + + +**Note:** Quotas can only be applied to individual `KafkaUser` resources. It is advised to apply a quota to an existing `KafkaUser` custom resource, as described in the following sections. You can [create](../../security/managing-access/#creating-a-kafkauser-in-the-event-streams-ui) a `Kafkauser` custom resource beforehand. + +### Using the {{site.data.reuse.openshift_short}} web console + +To update an existing `KafkaUser` custom resource by using the {{site.data.reuse.openshift_short}} web console: + +1. {{site.data.reuse.task_openshift_navigate_installed_operators}} +2. {{site.data.reuse.task_openshift_select_operator}} +3. Select the **Kafka User** tab, then select the `KafkaUser` resource you want to update from the list of existing users. +4. Expand the **Actions** dropdown, and select the `Edit KafkaUser` option. +5. In the YAML editor, add a [quotas section](#setting-user-quotas) with the required quotas. + + +### Using the CLI + +To update an existing `KafkaUser` by using the CLI: + +1. {{site.data.reuse.cncf_cli_login}} +2. Run one of the following commands to add or update one or more quota types: + + Setting `producerByteRate`: + + ```shell + QUOTA=; \ + kubectl patch kafkauser --namespace \ + -p "{\"spec\":{\"quotas\": \ + {\"producerByteRate\":$QUOTA}}}" \ + --type=merge + ``` + + Setting `consumerByteRate`: + + ```shell + QUOTA=; \ + kubectl patch kafkauser --namespace \ + -p "{\"spec\":{\"quotas\": \ + {\"consumerByteRate\":$QUOTA}}}" \ + --type=merge + ``` + + Setting `requestPercentage`: + ```shell + QUOTA=; \ + kubectl patch kafkauser --namespace \ + -p "{\"spec\":{\"quotas\": \ + {\"requestPercentage\":$QUOTA}}}" \ + --type=merge + ``` + + Setting all quotas: + ```shell + PRODUCER_QUOTA=; \ + CONSUMER_QUOTA=; \ + PERCENTAGE_QUOTA=; \ + kubectl patch kafkauser --namespace \ + -p "{\"spec\":{\"quotas\": \ + {\"producerByteRate\":$PRODUCER_QUOTA, \ + \"consumerByteRate\":$CONSUMER_QUOTA, \ + \"requestPercentage\":$PERCENTAGE_QUOTA}}}" \ + --type=merge + ``` + +## Setting broker quotas + +To configure throughput and storage limits on Kafka brokers in your {{site.data.reuse.es_name}} cluster, update an existing `EventStreams` custom resource by using the {{site.data.reuse.openshift_short}} web console or CLI, ensuring that the configuration has a `config` section with the relevant quotas defined. For example: + +```yaml +spec: + strimziOverrides: + kafka: + # ... + config: + client.quota.callback.class: io.strimzi.kafka.quotas.StaticQuotaCallback + client.quota.callback.static.produce: 1048576 + client.quota.callback.static.fetch: 2097152 + client.quota.callback.static.storage.soft: 429496729600 + client.quota.callback.static.storage.hard: 536870912000 + client.quota.callback.static.storage.check-interval: 8 +``` + +**Warning:** When setting the broker quotas, any previously configured user quotas will be ignored and overridden without warning. Additionally, the per-client group metrics that are provided by Kafka by default will not be available. For more information, see [Strimzi GitHub issue 32](https://github.com/strimzi/kafka-quotas-plugin/issues/32){:target="_blank"}. + + +Decide what you want to limit by defining one or more of the following quota types: + +| Property | Type | Description | +|:-----------------|:-----------------|:-----------------| +| `produce` | integer | This quota limits the number of bytes that a producer application is allowed to send per second. | +| `fetch` | integer | This quota limits the number of bytes that a consumer application is allowed to receive per second. | +| `storage.soft` | integer | This quota limits the number of bytes to consider as the lower soft limit for storage. | +| `storage.hard` | integer | This quota limits the number of bytes to consider as the higher hard limit for storage. | +| `storage.check-interval` | integer | This quota limits the number of seconds that a storage application is allowed to wait till the next check. To disable this limit, set the value to 0. | + +**Note:** Quotas can only be applied to individual `EventStreams` custom resources and apply across all Kafka brokers running within the cluster. + +### Using the {{site.data.reuse.openshift_short}} web console + +To update an existing `EventStreams` custom resource by using the {{site.data.reuse.openshift_short}} web console: + +1. {{site.data.reuse.task_openshift_navigate_installed_operators}} +2. {{site.data.reuse.task_openshift_select_operator}} +3. Select the **Event Streams** tab, then select the `EventStreams` custom resource you want to update from the list of existing instances. +4. Expand the **Actions** dropdown, and select the `Edit EventStreams` option. +5. In the YAML editor, add a [quotas section](#setting-broker-quotas) with the required quotas. + +### Using the CLI + +To update an existing `EventStreams` custom resource by using the {{site.data.reuse.openshift_short}} CLI: + +1. {{site.data.reuse.openshift_cli_login}} +2. Run one of the following commands to add or update one or more quota types: + + Setting `produce`: + + ```shell + PRODUCE_QUOTA=; \ + kubectl patch eventstreams --namespace \ + -p "{\"spec\":{\"strimziOverrides\":{\"kafka\":{\"config\": \ + {\"client.quota.callback.class\": + \"io.strimzi.kafka.quotas.StaticQuotaCallback\" , \ + \"client.quota.callback.static.produce\":$PRODUCE_QUOTA}}}}}" \ + --type merge + ``` + + Setting `fetch`: + + ```shell + FETCH_QUOTA=; \ + kubectl patch eventstreams --namespace \ + -p "{\"spec\":{\"strimziOverrides\":{\"kafka\":{\"config\": \ + {\"client.quota.callback.class\": + \"io.strimzi.kafka.quotas.StaticQuotaCallback\" , \ + \"client.quota.callback.static.produce\":$FETCH_QUOTA}}}}}" \ + --type merge + ``` + + + Setting `storage.soft`: + + ```shell + SOFTSTORAGE_QUOTA=; \ + kubectl patch eventstreams --namespace \ + -p "{\"spec\":{\"strimziOverrides\":{\"kafka\":{\"config\": \ + {\"client.quota.callback.class\": + \"io.strimzi.kafka.quotas.StaticQuotaCallback\" , \ + \"client.quota.callback.static.produce\":$SOFTSTORAGE_QUOTA}}}}}" \ + --type merge + ``` + + Setting `storage.hard`: + + ```shell + HARDSTORAGE_QUOTA=; \ + kubectl patch eventstreams --namespace \ + -p "{\"spec\":{\"strimziOverrides\":{\"kafka\":{\"config\": \ + {\"client.quota.callback.class\": + \"io.strimzi.kafka.quotas.StaticQuotaCallback\" , \ + \"client.quota.callback.static.produce\":$HARDSTORAGE_QUOTA}}}}}" \ + --type merge + ``` + + Setting `storage.check-interval`: + + ```shell + CHECKINTERVAL_QUOTA=; \ + kubectl patch eventstreams --namespace \ + -p "{\"spec\":{\"strimziOverrides\":{\"kafka\":{\"config\": \ + {\"client.quota.callback.class\": + \"io.strimzi.kafka.quotas.StaticQuotaCallback\" , \ + \"client.quota.callback.static.produce\":$CHECKINTERVAL_QUOTA}}}}}" \ + --type merge + ``` + + Setting all quotas: + + ```shell + PRODUCE_QUOTA=; \ + FETCH_QUOTA=; \ + SOFTSTORAGE_QUOTA=; \ + HARDSTORAGE_QUOTA=; \ + CHECKINTERVAL_QUOTA=; \ + kubectl patch eventstreams --namespace \ + -p "{\"spec\":{\"strimziOverrides\":{\"kafka\":{\"config\": \ + {\"client.quota.callback.class\": + \"io.strimzi.kafka.quotas.StaticQuotaCallback\" , \ + \"client.quota.callback.static.produce\":$PRODUCE_QUOTA , \ + \"client.quota.callback.static.fetch\": $FETCH_QUOTA , \ + \"client.quota.callback.static.storage.soft\": $SOFTSTORAGE_QUOTA , \ + \"client.quota.callback.static.storage.hard\": $HARDSTORAGE_QUOTA , \ + \"client.quota.callback.static.storage.check-interval\": $CHECKINTERVAL_QUOTA}}}}}" \ + --type merge + ``` \ No newline at end of file diff --git a/_es_11.5/07-administering/11-managing-multizone.md b/_es_11.5/07-administering/11-managing-multizone.md new file mode 100644 index 00000000..e51ed731 --- /dev/null +++ b/_es_11.5/07-administering/11-managing-multizone.md @@ -0,0 +1,22 @@ +--- +title: "Managing a multizone setup" +excerpt: "Find out more about managing multizone clusters, including what to do in the event of a failure." +categories: administering +slug: managing-multizone +toc: true +--- + +If you have set up your {{site.data.reuse.es_name}} installation to use [multiple availability zones](../../installing/preparing-multizone/), follow the guidance here if one of your nodes containing Kafka or ZooKeeper experiences problems. + +## Topic configuration + +Only create Kafka topics where the minimum in-sync replicas configuration can be met in the event of a zone failure. This requires considering the minimum in-sync replicas value in relation to the replication factor set for the topic, and the number of availability zones being used for spreading out your Kafka brokers. + +For example, if you have 3 availability zones and 6 Kafka brokers, losing a zone means the loss of 2 brokers. In the event of such a zone failure, the following topic configurations will guarantee that you can continue to produce to and consume from your topics: + +- If the replication factor is set to 6, then the suggested minimum in-sync replica is 4. +- If the replication factor is set to 5, then the suggested minimum in-sync replica is 3. + +## Updating an existing installation + +If you are updating an existing installation, for example, adding a new Kafka broker node or replacing a failing node that contains a Kafka broker, you can [label another node](../../installing/preparing-multizone/#zone-awareness) with the zone label. When the new node label is applied, Kubernetes will then be able to schedule a Kafka broker to run on that node. diff --git a/_es_11.5/07-administering/12-graceful-shutdown.md b/_es_11.5/07-administering/12-graceful-shutdown.md new file mode 100644 index 00000000..fff948c5 --- /dev/null +++ b/_es_11.5/07-administering/12-graceful-shutdown.md @@ -0,0 +1,110 @@ +--- +title: "Stopping and starting Event Streams" +excerpt: "Find out how to gracefully shut down an Event Streams instance, for example, in preparation for maintenance." +categories: administering +slug: stopping-starting +toc: true +--- + +You can stop or shut down your {{site.data.reuse.es_name}} instance if required. +You might want to do this in cases of hardware maintenance or during scheduled power outages. + +Use the following instructions to gracefully shut down your {{site.data.reuse.es_name}} instance. The instance can be started again following the [starting up {{site.data.reuse.es_name}}](#starting-up-event-streams) instructions. + +## Stopping {{site.data.reuse.es_name}} + +To shut down your cluster gracefully, follow these steps: +1. Enable `StrimziPodSetOnlyReconciliation` mode +2. Scale down components +3. Uninstall the operator + +### Enabling StrimziPodSetOnlyReconciliation mode in {{site.data.reuse.es_name}} operator + +Running the operator in this mode ensures that your {{site.data.reuse.es_name}} instance is no longer managed by the `EventStreams` operator. However, the operator still continues to reconcile the `StrimziPodSets` resources, which can be used to scale down Kafka and Zookeeper pods. + +#### Using the Kubernetes command-line tool (`kubectl`) + +1. To enable the `StrimziPodSetOnlyReconciliation` mode, follow these steps: + + **Note:** This command requires the `yq` [YAML](https://github.com/mikefarah/yq){:target="_blank"} parsing and editing tool. + - If you are running on the {{site.data.reuse.openshift_short}}, run the following command to update the `ClusterServiceVersion` of the `EventStreams` operator: + + ```shell + kubectl get csv -n ibm-eventstreams.v -oyaml | yq e "(.spec.install.spec.deployments[0].spec.template.spec.containers[0].env[] | select(.name==\"STRIMZI_POD_SET_RECONCILIATION_ONLY\")) .value=\"true\"" | kubectl apply -f - + ``` + + - If you are running on other Kubernetes platforms, run the following command to update the deployment resource of the `EventStreams` operator: + + ```shell + kubectl get deploy -n eventstreams-cluster-operator -oyaml | yq e "(.spec.template.spec.containers[0].env[] | select(.name==\"STRIMZI_POD_SET_RECONCILIATION_ONLY\")) .value=\"true\"" | kubectl apply -f - + ``` + + Where `` is the namespace your `EventStreams` operator is installed in and `` is the version of the operator that is installed. +2. Wait for the operator pod to reconcile. This is indicated by the `Running` state. You can watch the progress by running the following command: + + ```shell + kubectl get pods -n --watch + ``` + + Where `` is the namespace your `EventStreams` operator is installed in. + +#### Using the {{site.data.reuse.openshift_short}} web console + +1. {{site.data.reuse.openshift_ui_login}} +2. {{site.data.reuse.task_openshift_navigate_installed_operators}} +3. {{site.data.reuse.task_openshift_select_operator}} +4. Navigate to the `ClusterServiceVersion` yaml of the `EventStreams` operator. +5. Set the following environment variable under `spec.install.spec.deployments[0].spec.template.spec.containers[0].env[]` to `true`: + + ```yaml + - name: STRIMZI_POD_SET_RECONCILIATION_ONLY + value: 'true' + ``` + +6. Wait for all {{site.data.reuse.es_name}} pods to reconcile. This is indicated by the `Running` state. You can watch the progress by running the following command: + + ```shell + kubectl get pods -n --watch + ``` + + Where `` is the namespace your `EventStreams` operator is installed in. + +### Scale down components + +After starting the operator in `StrimziPodSetOnlyReconciliation` mode, you can safely scale down all components to `0` replicas, ensuring no pods are running. + +To do this for all components of type `Deployment`, run: + +```shell +kubectl get deployments -n -l app.kubernetes.io/instance= -o custom-columns=NAME:.metadata.name,REPLICAS:.spec.replicas --no-headers > deployment.txt && while read -ra deploy; do kubectl -n scale --replicas=0 deployment/${deploy}; done < deployment.txt +``` + +Where `` is the namespace your {{site.data.reuse.es_name}} instance is installed in and `` is the name of your {{site.data.reuse.es_name}} instance. + +**Note:** This saves the state of your deployments to a file called `deployment.txt` in the current working directory. + +To scale down all components of type `StrimziPodSet`, run the following command to delete the `spec.pods` configuration in the `StrimziPodSet` resources: + +```shell +kubectl patch -n `kubectl get sps -n -o name` --type json -p '[{"op":"replace","path":"/spec/pods","value":[]}]' +``` + +Where `` is the namespace your {{site.data.reuse.es_name}} instance is installed in. + +### Uninstall the operator + +Select your platform to see the instructions for: + +- Uninstalling {{site.data.reuse.es_name}} operator on [{{site.data.reuse.openshift_short}}](../../installing/uninstalling/#uninstalling-an-event-streams-operator-on-openshift-container-platform). +- Uninstalling {{site.data.reuse.es_name}} operator on other [Kubernetes platforms](../../installing/uninstalling/#uninstalling-an-event-streams-operator-on-other-kubernetes-platforms). + +## Starting up {{site.data.reuse.es_name}} + +To scale the {{site.data.reuse.es_name}} instance back up to the original state, install the {{site.data.reuse.es_name}} operator again following the steps in the [installing]({{ 'installpagedivert' | relative_url }}) section. + +1. Install the operator, ensuring it is configured with the same [installation mode]({{ 'installpagedivert' | relative_url }}) as used for the previous installation to manage the instances of {{site.data.reuse.es_name}} in **any namespace** or a **single namespace**. +2. Wait for the operator to scale all resources back up to their original state. You can watch the progress by running: + + ```shell + kubectl get pods --watch + ``` diff --git a/_es_11.5/07a-reference/01-EventStreams-api-reference.md b/_es_11.5/07a-reference/01-EventStreams-api-reference.md new file mode 100644 index 00000000..cc6f2cc2 --- /dev/null +++ b/_es_11.5/07a-reference/01-EventStreams-api-reference.md @@ -0,0 +1,336 @@ +--- +title: "API reference for the `EventStreams/v1beta2` CRDs" +excerpt: "API reference for the Custom Resource Definitions (CRDs) that are used by Event Streams." +categories: reference +slug: api-reference-es +toc: true +--- + +Find out more abut the Custom Resource Definitions (CRDs) that are used by {{site.data.reuse.es_name}}. + +## spec + +| Property | Type | Description | +|----------|--------|--------------------------------------------------| +| spec | object | The specification of the {{site.data.reuse.es_name}} instance. | + +### adminApi + +| Property | Type | Description | +| --- | --- | --- | +| spec.adminApi | object | Configuration of the {{site.data.reuse.es_name}} administration API server. | +| spec.adminApi.endpoints | array | Defines endpoints that will be created to communicate with the component. If nothing is specified, a default endpoint is created that is externally accessible via a Route with Bearer Authentication. | +| spec.adminApi.env | array | Apply additional custom environment variables to this component. | +| spec.adminApi.image | string | Identify a custom image to use for this component. | +| spec.adminApi.livenessProbe | object | Modify the Kubernetes liveness probe applied to this component. | +| spec.adminApi.livenessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.adminApi.livenessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.adminApi.livenessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.adminApi.livenessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.adminApi.livenessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.adminApi.logging | object | Specify custom logging for this component. | +| spec.adminApi.logging.loggers | object | A Map from logger name to logger level. | +| spec.adminApi.logging.type | string | Logging type, must be either 'inline' or 'external'. | +| spec.adminApi.logging.valueFrom | object | `ConfigMap` entry where the logging configuration is stored. | +| spec.adminApi.logging.valueFrom.configMapKeyRef | object | Reference to the key in the ConfigMap containing the configuration. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data){:target="_blank"}. | +| spec.adminApi.readinessProbe | object | Modify the Kubernetes readiness probe applied to this component. | +| spec.adminApi.readinessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.adminApi.readinessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.adminApi.readinessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.adminApi.readinessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.adminApi.readinessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.adminApi.replicas | integer | The number of instances to deploy. | +| spec.adminApi.resources | object | Modifies the resource limits and requests to apply to this component. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits){:target="_blank"}. | +| spec.adminApi.template | object | Template to specify how resources are generated. | +| spec.adminApi.template.pod | object | Template to override attributes for pods created for this component. | +| spec.adminApi.template.pod.affinity | object | The pod's affinity rules. | +| spec.adminApi.template.pod.enableServiceLinks | boolean | Indicates whether information about services should be injected into Pod's environment variables. | +| spec.adminApi.template.pod.hostAliases | array | The pod's HostAliases. HostAliases is an optional list of hosts and IPs that will be injected into the Pod's hosts file if specified. | +| spec.adminApi.template.pod.imagePullSecrets | array | List of references to secrets in the same namespace to use for pulling any of the images used by this Pod. When the `STRIMZI_IMAGE_PULL_SECRETS` environment variable in Cluster Operator and the `imagePullSecrets` option are specified, only the `imagePullSecrets` variable is used and the `STRIMZI_IMAGE_PULL_SECRETS` variable is ignored. | +| spec.adminApi.template.pod.metadata | object | Metadata applied to the resource. | +| spec.adminApi.template.pod.metadata.annotations | object | Annotations added to the Kubernetes resource. | +| spec.adminApi.template.pod.metadata.labels | object | Labels added to the Kubernetes resource. | +| spec.adminApi.template.pod.priorityClassName | string | The name of the priority class used to assign priority to the pods. For more information about priority classes, see {K8sPriorityClass}. | +| spec.adminApi.template.pod.schedulerName | string | The name of the scheduler used to dispatch this `Pod`. If not specified, the default scheduler will be used. | +| spec.adminApi.template.pod.securityContext | object | Configures pod-level security attributes and common container settings. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/){:target="_blank"}. | +| spec.adminApi.template.pod.terminationGracePeriodSeconds | integer | The grace period is the duration in seconds after the processes running in the pod are sent a termination signal, and the time when the processes are forcibly halted with a kill signal. Set this value to longer than the expected cleanup time for your process. Value must be a non-negative integer. A zero value indicates delete immediately. You might need to increase the grace period for very large Kafka clusters, so that the Kafka brokers have enough time to transfer their work to another broker before they are terminated. Defaults to 30 seconds. | +| spec.adminApi.template.pod.tmpDirSizeLimit | string | Defines the total amount (for example `1Gi`) of local storage required for temporary EmptyDir volume (`/tmp`). Default value is `5Mi`. | +| spec.adminApi.template.pod.tolerations | array | The pod's tolerations. | +| spec.adminApi.template.pod.topologySpreadConstraints | array | The pod's topology spread constraints. | + + +### adminUI + +| Property | Type | Description | +| --- | --- | --- | +| spec.adminUI | object | Configuration of the web server that hosts the administration user interface. | +| spec.adminUI.authentication | array | Defines the authentication mechanism for the UI. | +| spec.adminUI.endpoints | array | Defines endpoints that will be created to communicate with the component. If nothing is specified, a default endpoint is created that is externally accessible via a Route with Bearer Authentication. | +| spec.adminUI.env | array | Apply additional custom environment variables to this component. | +| spec.adminUI.image | string | Identify a custom image to use for this component. | +| spec.adminUI.livenessProbe | object | Modify the Kubernetes liveness probe applied to this component. | +| spec.adminUI.livenessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.adminUI.livenessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.adminUI.livenessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.adminUI.livenessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.adminUI.livenessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.adminUI.logging | object | Specify custom logging for this component. | +| spec.adminUI.logging.loggers | object | A Map from logger name to logger level. | +| spec.adminUI.logging.type | string | Logging type, must be either 'inline' or 'external'. | +| spec.adminUI.logging.valueFrom | object | `ConfigMap` entry where the logging configuration is stored. | +| spec.adminUI.logging.valueFrom.configMapKeyRef | object | Reference to the key in the ConfigMap containing the configuration. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data){:target="_blank"}. | +| spec.adminUI.readinessProbe | object | Modify the Kubernetes readiness probe applied to this component. | +| spec.adminUI.readinessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.adminUI.readinessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.adminUI.readinessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.adminUI.readinessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.adminUI.readinessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.adminUI.redis | object | Configuration options for the redis container used to store UI login sessions. | +| spec.adminUI.redis.env | array | Apply additional custom environment variables to this component. | +| spec.adminUI.redis.image | string | Identify a custom image to use for this component. | +| spec.adminUI.redis.livenessProbe | object | Modify the Kubernetes liveness probe applied to this component. | +| spec.adminUI.redis.livenessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.adminUI.redis.livenessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.adminUI.redis.livenessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.adminUI.redis.livenessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.adminUI.redis.livenessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.adminUI.redis.readinessProbe | object | Modify the Kubernetes readiness probe applied to this component. | +| spec.adminUI.redis.readinessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.adminUI.redis.readinessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.adminUI.redis.readinessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.adminUI.redis.readinessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.adminUI.redis.readinessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.adminUI.redis.resources | object | Modifies the resource limits and requests to apply to this component. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits){:target="_blank"}. | +| spec.adminUI.replicas | integer | The number of instances to deploy. | +| spec.adminUI.resources | object | Modifies the resource limits and requests to apply to this component. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits){:target="_blank"}. | + +### apicurioRegistry + +| Property | Type | Description | +| --- | --- | --- | +| spec.apicurioRegistry | object | Configuration of the Apicurio Registry server. | +| spec.apicurioRegistry.config | object | Apicurio registry config properties with the following prefixes cannot be set: (with the exception of: ). | +| spec.apicurioRegistry.endpoints | array | Defines endpoints that will be created to communicate with the component. If nothing is specified, a default endpoint is created that is externally accessible via a Route with Bearer Authentication. | +| spec.apicurioRegistry.env | array | Apply additional custom environment variables to this component. | +| spec.apicurioRegistry.image | string | Identify a custom image to use for this component. | +| spec.apicurioRegistry.livenessProbe | object | Modify the Kubernetes liveness probe applied to this component. | +| spec.apicurioRegistry.livenessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.apicurioRegistry.livenessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.apicurioRegistry.livenessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.apicurioRegistry.livenessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.apicurioRegistry.livenessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.apicurioRegistry.logging | object | Specify custom logging for this component. | +| spec.apicurioRegistry.logging.loggers | object | A Map from logger name to logger level. | +| spec.apicurioRegistry.logging.type | string | Logging type, must be either 'inline' or 'external'. | +| spec.apicurioRegistry.logging.valueFrom | object | `ConfigMap` entry where the logging configuration is stored. | +| spec.apicurioRegistry.logging.valueFrom.configMapKeyRef | object | Reference to the key in the ConfigMap containing the configuration. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data){:target="_blank"}. | +| spec.apicurioRegistry.proxyContainer | object | Specify overrides for the proxy container. | +| spec.apicurioRegistry.proxyContainer.env | array | Apply additional custom environment variables to this component. | +| spec.apicurioRegistry.proxyContainer.image | string | Identify a custom image to use for this component. | +| spec.apicurioRegistry.proxyContainer.logging | object | Specify custom logging for this component. | +| spec.apicurioRegistry.proxyContainer.logging.loggers | object | A Map from logger name to logger level. | +| spec.apicurioRegistry.proxyContainer.logging.type | string | Logging type, must be either 'inline' or 'external'. | +| spec.apicurioRegistry.proxyContainer.logging.valueFrom | object | `ConfigMap` entry where the logging configuration is stored. | +| spec.apicurioRegistry.proxyContainer.logging.valueFrom.configMapKeyRef | object | Reference to the key in the ConfigMap containing the configuration. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data){:target="_blank"}. | +| spec.apicurioRegistry.proxyContainer.resources | object | Modifies the resource limits and requests to apply to this component. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits){:target="_blank"}. | +| spec.apicurioRegistry.readinessProbe | object | Modify the Kubernetes readiness probe applied to this component. | +| spec.apicurioRegistry.readinessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.apicurioRegistry.readinessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.apicurioRegistry.readinessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.apicurioRegistry.readinessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.apicurioRegistry.readinessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.apicurioRegistry.replicas | integer | The number of instances to deploy. | +| spec.apicurioRegistry.resources | object | Modifies the resource limits and requests to apply to this component. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits){:target="_blank"}. | + +### collector + +| Property | Type | Description | +| --- | --- | --- | +| spec.collector | object | Configuration of the collector server responsible for aggregating metrics from Kafka brokers. | +| spec.collector.env | array | Apply additional custom environment variables to this component. | +| spec.collector.image | string | Identify a custom image to use for this component. | +| spec.collector.livenessProbe | object | Modify the Kubernetes liveness probe applied to this component. | +| spec.collector.livenessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.collector.livenessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.collector.livenessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.collector.livenessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.collector.livenessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.collector.logging | object | Specify custom logging for this component. | +| spec.collector.logging.loggers | object | A Map from logger name to logger level. | +| spec.collector.logging.type | string | Logging type, must be either 'inline' or 'external'. | +| spec.collector.logging.valueFrom | object | `ConfigMap` entry where the logging configuration is stored. | +| spec.collector.logging.valueFrom.configMapKeyRef | object | Reference to the key in the ConfigMap containing the configuration. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data){:target="_blank"}. | +| spec.collector.readinessProbe | object | Modify the Kubernetes readiness probe applied to this component. | +| spec.collector.readinessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.collector.readinessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.collector.readinessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.collector.readinessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.collector.readinessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.collector.replicas | integer | The number of instances to deploy. | +| spec.collector.resources | object | Modifies the resource limits and requests to apply to this component. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits){:target="_blank"}. | +| spec.collector.template | object | Template to specify how resources are generated. | +| spec.collector.template.pod | object | Template to override attributes for pods created for this component. | +| spec.collector.template.pod.affinity | object | The pod's affinity rules. | +| spec.collector.template.pod.enableServiceLinks | boolean | Indicates whether information about services should be injected into Pod's environment variables. | +| spec.collector.template.pod.hostAliases | array | The pod's HostAliases. HostAliases is an optional list of hosts and IPs that will be injected into the Pod's hosts file if specified. | +| spec.collector.template.pod.imagePullSecrets | array | List of references to secrets in the same namespace to use for pulling any of the images used by this Pod. When the `STRIMZI_IMAGE_PULL_SECRETS` environment variable in Cluster Operator and the `imagePullSecrets` option are specified, only the `imagePullSecrets` variable is used and the `STRIMZI_IMAGE_PULL_SECRETS` variable is ignored. | +| spec.collector.template.pod.metadata | object | Metadata applied to the resource. | +| spec.collector.template.pod.metadata.annotations | object | Annotations added to the Kubernetes resource. | +| spec.collector.template.pod.metadata.labels | object | Labels added to the Kubernetes resource. | +| spec.collector.template.pod.priorityClassName | string | The name of the priority class used to assign priority to the pods. For more information about priority classes, see {K8sPriorityClass}. | +| spec.collector.template.pod.schedulerName | string | The name of the scheduler used to dispatch this `Pod`. If not specified, the default scheduler will be used. | +| spec.collector.template.pod.securityContext | object | Configures pod-level security attributes and common container settings. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/){:target="_blank"}. | +| spec.collector.template.pod.terminationGracePeriodSeconds | integer | The grace period is the duration in seconds after the processes running in the pod are sent a termination signal, and the time when the processes are forcibly halted with a kill signal. Set this value to longer than the expected cleanup time for your process. Value must be a non-negative integer. A zero value indicates delete immediately. You might need to increase the grace period for very large Kafka clusters, so that the Kafka brokers have enough time to transfer their work to another broker before they are terminated. Defaults to 30 seconds. | +| spec.collector.template.pod.tmpDirSizeLimit | string | Defines the total amount (for example `1Gi`) of local storage required for temporary EmptyDir volume (`/tmp`). Default value is `5Mi`. | +| spec.collector.template.pod.tolerations | array | The pod's tolerations. | +| spec.collector.template.pod.topologySpreadConstraints | array | The pod's topology spread constraints. | + +### images + +| Property | Type | Description | +| --- | --- | --- | +| spec.images | object | Configuration for accessing {{site.data.reuse.es_name}} Docker images. | +| spec.images.pullPolicy | string | The image pull policy to use for the components. | +| spec.images.pullSecrets | array | The image pull secrets to use for the components. | + +### kafkaProxy + +| Property | Type | Description | +| --- | --- | --- | +| spec.kafkaProxy | object | Configuration of the Kafka Proxy. | +| spec.kafkaProxy.env | array | Apply additional custom environment variables to this component. | +| spec.kafkaProxy.image | string | Identify a custom image to use for this component. | +| spec.kafkaProxy.livenessProbe | object | Modify the Kubernetes liveness probe applied to this component. | +| spec.kafkaProxy.livenessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.kafkaProxy.livenessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.kafkaProxy.livenessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.kafkaProxy.livenessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.kafkaProxy.livenessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.kafkaProxy.readinessProbe | object | Modify the Kubernetes readiness probe applied to this component. | +| spec.kafkaProxy.readinessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.kafkaProxy.readinessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.kafkaProxy.readinessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.kafkaProxy.readinessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.kafkaProxy.readinessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.kafkaProxy.resources | object | Modifies the resource limits and requests to apply to this component. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits){:target="_blank"}. | +| spec.kafkaProxy.template | object | Template to specify how resources are generated. | +| spec.kafkaProxy.template.pod | object | Template to override attributes for pods created for this component. | +| spec.kafkaProxy.template.pod.affinity | object | The pod's affinity rules. | +| spec.kafkaProxy.template.pod.enableServiceLinks | boolean | Indicates whether information about services should be injected into Pod's environment variables. | +| spec.kafkaProxy.template.pod.hostAliases | array | The pod's HostAliases. HostAliases is an optional list of hosts and IPs that will be injected into the Pod's hosts file if specified. | +| spec.kafkaProxy.template.pod.imagePullSecrets | array | List of references to secrets in the same namespace to use for pulling any of the images used by this Pod. When the `STRIMZI_IMAGE_PULL_SECRETS` environment variable in Cluster Operator and the `imagePullSecrets` option are specified, only the `imagePullSecrets` variable is used and the `STRIMZI_IMAGE_PULL_SECRETS` variable is ignored. | +| spec.kafkaProxy.template.pod.metadata | object | Metadata applied to the resource. | +| spec.kafkaProxy.template.pod.metadata.annotations | object | Annotations added to the Kubernetes resource. | +| spec.kafkaProxy.template.pod.metadata.labels | object | Labels added to the Kubernetes resource. | +| spec.kafkaProxy.template.pod.priorityClassName | string | The name of the priority class used to assign priority to the pods. For more information about priority classes, see {K8sPriorityClass}. | +| spec.kafkaProxy.template.pod.schedulerName | string | The name of the scheduler used to dispatch this `Pod`. If not specified, the default scheduler will be used. | +| spec.kafkaProxy.template.pod.securityContext | object | Configures pod-level security attributes and common container settings. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/){:target="_blank"}. | +| spec.kafkaProxy.template.pod.terminationGracePeriodSeconds | integer | The grace period is the duration in seconds after the processes running in the pod are sent a termination signal, and the time when the processes are forcibly halted with a kill signal. Set this value to longer than the expected cleanup time for your process. Value must be a non-negative integer. A zero value indicates delete immediately. You might need to increase the grace period for very large Kafka clusters, so that the Kafka brokers have enough time to transfer their work to another broker before they are terminated. Defaults to 30 seconds. | +| spec.kafkaProxy.template.pod.tmpDirSizeLimit | string | Defines the total amount (for example `1Gi`) of local storage required for temporary EmptyDir volume (`/tmp`). Default value is `5Mi`. | +| spec.kafkaProxy.template.pod.tolerations | array | The pod's tolerations. | +| spec.kafkaProxy.template.pod.topologySpreadConstraints | array | The pod's topology spread constraints. | + +### license + +| Property | Type | Description | +| --- | --- | --- | +| spec.license | object | Specify the license information for the instance of {{site.data.reuse.es_name}}. | +| spec.license.accept | boolean | Accept the selected product license by following the guidance in [licensing]({{ 'support/licensing' | relative_url }}){:target="_blank"}. | +| spec.license.license | string | License ID that the user is selecting and accepting. For more information, see [licensing]({{ 'support/licensing' | relative_url }}){:target="_blank"}. | +| spec.license.use | string | Specify if you intend for this installation to be used in a production environment. For more information, see [licensing]({{ 'support/licensing' | relative_url }}).| + +### requestIbmServices + +**Note:** If you are using Keycloak to configure access to the {{site.data.reuse.es_name}} UI, ensure you removed the `spec.requestIbmServices` section as described in [upgrading](../../installing/upgrading/#configure-your-instance-to-use-keycloak). + + +| Property | Type | Description | +| --- | --- | --- | +| spec.requestIbmServices | object | Specify the IBM Cloud Pak foundational services you want to configure. The Identity and Access Management (IAM) service and the Monitoring service are removed from {{site.data.reuse.icpfs}} 4.0.x and later. This field is ignored when {{site.data.reuse.fs}} version 4.0.x. or later is installed. For more information, see [the {{site.data.reuse.fs}} documentation](https://www.ibm.com/docs/en/cloud-paks/foundational-services/4.0?topic=about-deprecated-changed-services-features){:target="_blank"}. | +| spec.requestIbmServices.iam | boolean | Specifies whether to create a request for deploying the Identity and Access Management (IAM) service as part of the IBM Cloud Pak foundational services. You can use the IAM service for controlling access to the UI. | +| spec.requestIbmServices.monitoring | boolean | Specifies whether to create a request for deploying the Monitoring service as part of the IBM Cloud Pak foundational services. You can use the Monitoring service to monitor the status of your cluster and applications by using Grafana dashboards. | + +### restProducer + +| Property | Type | Description | +| --- | --- | --- | +| spec.restProducer | object | Configuration of the REST Producer server that allows messages to be produced to Kafka topics from REST clients. | +| spec.restProducer.endpoints | array | Defines endpoints that will be created to communicate with the component. If nothing is specified, a default endpoint is created that is externally accessible via a Route with Bearer Authentication. | +| spec.restProducer.env | array | Apply additional custom environment variables to this component. | +| spec.restProducer.image | string | Identify a custom image to use for this component. | +| spec.restProducer.livenessProbe | object | Modify the Kubernetes liveness probe applied to this component. | +| spec.restProducer.livenessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.restProducer.livenessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.restProducer.livenessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.restProducer.livenessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.restProducer.livenessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.restProducer.logging | object | Specify custom logging for this component. | +| spec.restProducer.logging.loggers | object | A Map from logger name to logger level. | +| spec.restProducer.logging.type | string | Logging type, must be either 'inline' or 'external'. | +| spec.restProducer.logging.valueFrom | object | `ConfigMap` entry where the logging configuration is stored. | +| spec.restProducer.logging.valueFrom.configMapKeyRef | object | Reference to the key in the ConfigMap containing the configuration. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data){:target="_blank"}. | +| spec.restProducer.readinessProbe | object | Modify the Kubernetes readiness probe applied to this component. | +| spec.restProducer.readinessProbe.failureThreshold | integer | Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. | +| spec.restProducer.readinessProbe.initialDelaySeconds | integer | The initial delay before first the health is first checked. Default to 15 seconds. Minimum value is 0. | +| spec.restProducer.readinessProbe.periodSeconds | integer | How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. | +| spec.restProducer.readinessProbe.successThreshold | integer | Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1. | +| spec.restProducer.readinessProbe.timeoutSeconds | integer | The timeout for each attempted health check. Default to 5 seconds. Minimum value is 1. | +| spec.restProducer.replicas | integer | The number of instances to deploy. | +| spec.restProducer.resources | object | Modifies the resource limits and requests to apply to this component. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits){:target="_blank"}. | +| spec.restProducer.template | object | Template to specify how resources are generated. | +| spec.restProducer.template.pod | object | Template to override attributes for pods created for this component. | +| spec.restProducer.template.pod.affinity | object | The pod's affinity rules. | +| spec.restProducer.template.pod.enableServiceLinks | boolean | Indicates whether information about services should be injected into Pod's environment variables. | +| spec.restProducer.template.pod.hostAliases | array | The pod's HostAliases. HostAliases is an optional list of hosts and IPs that will be injected into the Pod's hosts file if specified. | +| spec.restProducer.template.pod.imagePullSecrets | array | List of references to secrets in the same namespace to use for pulling any of the images used by this Pod. When the `STRIMZI_IMAGE_PULL_SECRETS` environment variable in Cluster Operator and the `imagePullSecrets` option are specified, only the `imagePullSecrets` variable is used and the `STRIMZI_IMAGE_PULL_SECRETS` variable is ignored. | +| spec.restProducer.template.pod.metadata | object | Metadata applied to the resource. | +| spec.restProducer.template.pod.metadata.annotations | object | Annotations added to the Kubernetes resource. | +| spec.restProducer.template.pod.metadata.labels | object | Labels added to the Kubernetes resource. | +| spec.restProducer.template.pod.priorityClassName | string | The name of the priority class used to assign priority to the pods. For more information about priority classes, see {K8sPriorityClass}. | +| spec.restProducer.template.pod.schedulerName | string | The name of the scheduler used to dispatch this `Pod`. If not specified, the default scheduler will be used. | +| spec.restProducer.template.pod.securityContext | object | Configures pod-level security attributes and common container settings. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/){:target="_blank"}. | +| spec.restProducer.template.pod.terminationGracePeriodSeconds | integer | The grace period is the duration in seconds after the processes running in the pod are sent a termination signal, and the time when the processes are forcibly halted with a kill signal. Set this value to longer than the expected cleanup time for your process. Value must be a non-negative integer. A zero value indicates delete immediately. You might need to increase the grace period for very large Kafka clusters, so that the Kafka brokers have enough time to transfer their work to another broker before they are terminated. Defaults to 30 seconds. | +| spec.restProducer.template.pod.tmpDirSizeLimit | string | Defines the total amount (for example `1Gi`) of local storage required for temporary EmptyDir volume (`/tmp`). Default value is `5Mi`. | +| spec.restProducer.template.pod.tolerations | array | The pod's tolerations. | +| spec.restProducer.template.pod.topologySpreadConstraints | array | The pod's topology spread constraints. | + +### security + +| Property | Type | Description | +| --- | --- | --- | +| spec.security | object | Security configuration for the {{site.data.reuse.es_name}} components. | +| spec.security.internalTls | string | Configure what TLS version {{site.data.reuse.es_name}} components use to communicate with one another. | + +### strimziOverrides + +| Property | Type | Description | +| --- | --- | --- | +| spec.strimziOverrides | object | Configuration of the Kafka and ZooKeeper clusters. Spec can be viewed at [Strimzi documentation](https://strimzi.io/docs/operators/0.42.0/configuring.html){:target="_blank"}. | + +### version + +| Property | Type | Description | +| --- | --- | --- | +| spec.version | string | Version of the {{site.data.reuse.es_name}} instance. | + +## status + +| Property | Type | Description | +| --- | --- | --- | +| status | object | The status of the {{site.data.reuse.es_name}} instance. | +| status.adminUiUrl | string | Web address for the {{site.data.reuse.es_name}} administration UI. | +| status.bootstrapRoutes | array | Routes for the new bootstrap connections. | +| status.conditions | array | Current state of the {{site.data.reuse.es_name}} cluster. | +| status.customImages | boolean | Identifies whether any of the Docker images have been modified from the defaults for this version of {{site.data.reuse.es_name}}. | +| status.endpoints | array | Addresses of the interfaces provided by the {{site.data.reuse.es_name}} cluster. | +| status.kafkaListeners | array | Addresses of the internal and external listeners. | +| status.licenseVersion | string | The License version the user has selected. | +| status.observedGeneration | integer | The generation of the resource at the last successful reconciliation. | +| status.phase | string | Identifies the current state of the {{site.data.reuse.es_name}} instance. | +| status.routes | object | OpenShift Routes created as part of the {{site.data.reuse.es_name}} cluster. | +| status.versions | object | Information about the version of this instance and it's upgradable versions. | +| status.versions.available | object | The versions that this instance of {{site.data.reuse.es_name}} can be upgraded to. | +| status.versions.available.channels | array | A list of versions that the operator is able to automatically upgrade from. | +| status.versions.available.versions | array | A list of versions that the operator is able to upgrade this instance of {{site.data.reuse.es_name}} to. | +| status.versions.reconciled | string | The current running version of this operator. | + diff --git a/_es_11.5/07a-reference/02-EventStreamsGeoReplicator-api-reference.md b/_es_11.5/07a-reference/02-EventStreamsGeoReplicator-api-reference.md new file mode 100644 index 00000000..dc53b72c --- /dev/null +++ b/_es_11.5/07a-reference/02-EventStreamsGeoReplicator-api-reference.md @@ -0,0 +1,34 @@ +--- +title: "API reference for the `EventStreamsGeoReplicator/v1beta1` CRDs" +excerpt: "API reference for the Custom Resource Definitions (CRDs) that are used by `EventStreamsGeoReplicator/v1beta1`." +categories: reference +slug: api-reference-esgr +toc: true +--- + +Find out more abut the Custom Resource Definitions (CRDs) used by {{site.data.reuse.es_name}} geo-replicator. + + +## spec + +| Property | Type | Description | +| --- | --- | --- | +| spec | object | The {{site.data.reuse.es_name}} geo-replicator specification. | +| spec.replicas | integer | The number of Kafka Connect workers to start for {{site.data.reuse.es_name}} geo-replication. | +| spec.version | string | Version of the {{site.data.reuse.es_name}} geo-replicator. | + +## status + +| Property | Type | Description | +| --- | --- | --- | +| status | object | The status of the EventStreamsGeoReplicator instance. | +| status.conditions | array | Current state of the {{site.data.reuse.es_name}} cluster. | +| status.customImages | boolean | Identifies whether any of the Docker images have been modified from the defaults for this version of {{site.data.reuse.es_name}}. | +| status.observedGeneration | integer | The generation of the resource at the last successful reconciliation. | +| status.phase | string | Identifies the current state of the {{site.data.reuse.es_name}} instance. | +| status.versions | object | Information about the version of this instance and it's upgradable versions. | +| status.versions.available | object | The versions that this instance of {{site.data.reuse.es_name}} can be upgraded to. | +| status.versions.available.channels | array | A list of versions that the operator is able to automatically upgrade from. | +| status.versions.available.versions | array | A list of versions that the operator is able to upgrade this instance of {{site.data.reuse.es_name}} to. | +| status.versions.reconciled | string | The current running version of this operator. | + diff --git a/_es_11.5/08-troubleshooting/01-ts-intro.md b/_es_11.5/08-troubleshooting/01-ts-intro.md new file mode 100644 index 00000000..535675f7 --- /dev/null +++ b/_es_11.5/08-troubleshooting/01-ts-intro.md @@ -0,0 +1,13 @@ +--- +title: "Troubleshooting overview" +excerpt: "To help troubleshoot issues with your installation, see the troubleshooting topics in this section." +categories: troubleshooting +slug: intro +toc: false +--- + +To help troubleshoot issues with your installation, see the troubleshooting topics in this section. + +In addition, you can check the health information for your environment as described in [monitoring deployment health](../../administering/deployment-health/) and [monitoring Kafka cluster health](../../administering/cluster-health/). + +If you need help, want to raise questions, or have feature requests, see the {{site.data.reuse.es_name}} [support channels]({{ 'support' | relative_url }}). diff --git a/_es_11.5/08-troubleshooting/02-gathering-logs.md b/_es_11.5/08-troubleshooting/02-gathering-logs.md new file mode 100644 index 00000000..43c825e7 --- /dev/null +++ b/_es_11.5/08-troubleshooting/02-gathering-logs.md @@ -0,0 +1,9 @@ +--- +title: "Gathering logs" +excerpt: "To help IBM support troubleshoot any issues with your Event Streams installation, run the log gathering script." +categories: troubleshooting +slug: gathering-logs +toc: true +--- + +To help IBM Support troubleshoot any issues with your installation, run the [log gathering script]({{ 'support/gathering-logs/' | relative_url }}). diff --git a/_es_11.5/08-troubleshooting/03-resources-not-available.md b/_es_11.5/08-troubleshooting/03-resources-not-available.md new file mode 100644 index 00000000..b887aea3 --- /dev/null +++ b/_es_11.5/08-troubleshooting/03-resources-not-available.md @@ -0,0 +1,31 @@ +--- +title: "Resources not available" +excerpt: "Reasons for Event Streams resources not being available." +categories: troubleshooting +slug: resources-not-available +toc: true +--- + +If {{site.data.reuse.es_name}} resources are not available, the following are possible symptoms and causes. + + +## Insufficient system resources + +You can specify the memory and CPU requirements when the {{site.data.reuse.es_name}} instance is installed using the `EventStreams` custom resource. If the values set are larger than the resources available, then pods will fail to start. + +Common error messages in such cases include the following: +- **`pod has unbound PersistentVolumeClaims`** - occurs when there are no Persistent Volumes available that meet the requirements provided at the time of installation. +- **`Insufficient memory`** - occurs when there are no nodes with enough available memory to support the limits provided at the time of installation. +- **`Insufficient CPU`** - occurs when there are no nodes with enough available CPU to support the limits provided at the time of installation. + +To get detailed information on the cause of the error, check the events for the individual pods. + +Ensure that resource requests and limits do not exceed the total memory available. For example, if a system has 16 GB of memory available per node, then the broker memory requirements must be set to be less than 16 GB. This allows resources to be available for the other {{site.data.reuse.es_name}} components which might reside on the same node. + +To correct these issues, increase the amount of system resources available or re-install the {{site.data.reuse.es_name}} instance with lower resource requirements. + +## Problems with secrets + +Before installing the operator, configure secrets with your entitlement key for the IBM Container software library. This will enable container images to be pulled from the registry. See the [installation](../../installing/installing-on-kubernetes/#creating-an-image-pull-secret) section of the documentation for more information. + +If you do not prepare the required secrets, all pods will fail to start with `ImagePullBackOff` errors. In this case, configure the required secrets and allow the pod to restart. diff --git a/_es_11.5/08-troubleshooting/04-es-install-fails.md b/_es_11.5/08-troubleshooting/04-es-install-fails.md new file mode 100644 index 00000000..a4026a94 --- /dev/null +++ b/_es_11.5/08-troubleshooting/04-es-install-fails.md @@ -0,0 +1,36 @@ +--- +title: "Event Streams installation reports `Blocked` status" +excerpt: "Installation of Event Streams instance reports a failed status when Foundational Services is not installed." +categories: troubleshooting +slug: es-install-fails +toc: true +--- + +## Symptoms + +When installing a new instance of {{site.data.reuse.es_name}} on an {{site.data.reuse.openshift_short}} cluster, installation fails with status of the {{site.data.reuse.es_name}} instance as `Blocked`. + +- The following condition message related to {{site.data.reuse.cp4i}} is displayed in the status of the instance: + + ```terminal + This instance requires IBM Cloud Pak for Integration operator version 7.2.0 or later. + Install the required version of the IBM Cloud Pak for Integration operator as described in https://ibm.biz/int-docs. + This instance will remain in the `Blocked` status until the correct operator version is installed. + ``` + +- The following condition message related to {{site.data.reuse.icpfs}} is displayed in the status of the instance: + + ```terminal + This instance requires foundational services version 3.19.0 or later. + Install the required version of the foundational services operator as described in https://ibm.biz/es-cpfs-installation. + This instance will remain in the `Blocked` status until the correct operator version is installed. + ``` + +## Causes + +There could be several reasons, for example, you are trying to install your instance with Keycloak authentication, but {{site.data.reuse.cp4i}} and a supported version of {{site.data.reuse.fs}} is not installed in your cluster. + + +## Resolving the problem + +Based on your requirements, [install](https://www.ibm.com/docs/en/cloud-paks/cp-integration/16.1.0?topic=installing){:target="_blank"} {{site.data.reuse.cp4i}} or its dependencies such as a supported version of {{site.data.reuse.fs}} before installing an instance of {{site.data.reuse.es_name}}. diff --git a/_es_11.5/08-troubleshooting/06-georep-error.md b/_es_11.5/08-troubleshooting/06-georep-error.md new file mode 100644 index 00000000..78883649 --- /dev/null +++ b/_es_11.5/08-troubleshooting/06-georep-error.md @@ -0,0 +1,32 @@ +--- +title: "Error when creating multiple geo-replicators" +excerpt: "No meaningful error message provided when specifying invalid topic list format." +categories: troubleshooting +slug: georeplication-error +toc: true +--- + +## Symptoms + +After providing a list of topic names when creating a geo-replicator, only the first topic successfully replicates data. + +The additional topics specified are either not displayed in the destination cluster UI **Topics** view, or are displayed as `.` but are not enabled for geo-replication. + +## Causes + +When using the CLI to set up replication, the [list of topics to geo-replicate](../../georeplication/setting-up/#using-the-cli-1) included spaces between the topic names in the comma-separated list. + +## Resolving the problem + +Ensure you do not have spaces between the topic names. For example, if you specified the `topics` parameter with spaces such as: + +``` +--topics MyTopicName1, MyTopicName2, MyTopicName3 +--topics "MyTopicName1, MyTopicName2, MyTopicName3" +``` + +Remove the spaces between the topic names and re-apply the command using a `topics` parameter with no spaces such as: + +``` +--topics MyTopicName1,MyTopicName2,MyTopicName3 +``` diff --git a/_es_11.5/08-troubleshooting/07-kakfa-producer.md b/_es_11.5/08-troubleshooting/07-kakfa-producer.md new file mode 100644 index 00000000..3e65bee0 --- /dev/null +++ b/_es_11.5/08-troubleshooting/07-kakfa-producer.md @@ -0,0 +1,54 @@ +--- +title: "TimeoutException when using standard Kafka producer" +excerpt: "Standard Kafka producer fails when no security settings provided." +categories: troubleshooting +slug: kafka-producer-error +toc: true +--- + +## Symptoms + +The standard Kafka producer (`kafka-console-producer.sh`) is unable to send messages and fails with the following timeout error: + +`org.apache.kafka.common.errors.TimeoutException` + +## Causes + +This situation occurs if the producer is invoked without supplying the required security credentials. In this case, the producer fails with +the following error: + +``` +Error when sending message to topic with key: null, value: bytes +``` + +## Resolving the problem + +Create a properties file with the following content, adding by uncommenting either the SCRAM or Mutual TLS authentication settings depending on how the external listener has been [configured.](../../installing/configuring#configuring-access) + +``` +bootstrap.servers= +# SCRAM Properties +#security.protocol=SASL_SSL +#sasl.mechanism=SCRAM-SHA-512 +#sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password=""; +# Mutual auth properties +#security.protocol=SSL +#ssl.keystore.location= +#ssl.keystore.password= +# TLS Properties +ssl.protocol=TLSv1.3 +ssl.truststore.location= +ssl.truststore.password= +``` + +Replace the `` with the address of the [Kafka bootstrap host.](../../getting-started/connecting/#obtaining-the-bootstrap-address) + +If you used SCRAM authentication for the external listener, replace `` with the SCRAM user name and `` with the SCRAM user's password. + +If you used Mutual TLS authentication for the external listener, replace `` with the location of a key store containing the client certificate and `` with the password for the key store. + +Finally, replace `` with the location of a trust store file containing the server certificate (for example, `certs.jks`), and `` with the password for the trust store. + +When running the `kafka-console-producer.sh` command, include the `--producer.config ` option, replacing `` with the name of the property file and the path to it. For example: + +`kafka-console-producer.sh --broker-list : --topic --producer.config ` diff --git a/_es_11.5/08-troubleshooting/08-kakfa-consumer.md b/_es_11.5/08-troubleshooting/08-kakfa-consumer.md new file mode 100644 index 00000000..744202ac --- /dev/null +++ b/_es_11.5/08-troubleshooting/08-kakfa-consumer.md @@ -0,0 +1,48 @@ +--- +title: "Standard Kafka consumer hangs and does not output messages" +excerpt: "Standard Kafka consumer hangs when no security settings provided." +categories: troubleshooting +slug: kafka-consumer-hangs +toc: true +--- + +## Symptoms + +The standard Kafka consumer (`kafka-console-consumer.sh`) is unable to receive messages and hangs without producing any output. + +## Causes + +This situation occurs if the consumer is invoked without supplying the required security credentials. In this case, the consumer +hangs and does not output any messages sent to the topic. + +## Resolving the problem + +Create a properties file with the following content, adding by uncommenting either the SCRAM or Mutual TLS authentication settings depending on how the external listener has been [configured.](../../installing/configuring#configuring-access) + +``` +bootstrap.servers= +# SCRAM Properties +#security.protocol=SASL_SSL +#sasl.mechanism=SCRAM-SHA-512 +#sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password=""; +# Mutual auth properties +#security.protocol=SSL +#ssl.keystore.location= +#ssl.keystore.password= +# TLS Properties +ssl.protocol=TLSv1.3 +ssl.truststore.location= +ssl.truststore.password= +``` + +Replace the `` with the address of the [Kafka bootstrap host.](../../getting-started/connecting/#obtaining-the-bootstrap-address) + +If you used SCRAM authentication for the external listener, replace `` with the SCRAM user name and `` with the SCRAM user's password. + +If you used Mutual TLS authentication for the external listener, replace `` with the location of a key store containing the client certificate and `` with the password for the key store. + +Finally, replace `` with the location of a trust store file containing the server certificate (for example, `certs.jks`), and `` with the password for the trust store. + +When running the `kafka-console-consumer.sh` command include the `--consumer.config ` option, replacing the `` with the name of the property file and the path to it. For example: + +`kafka-console-consumer.sh --bootstrap-server : --topic -consumer.config ` diff --git a/_es_11.5/08-troubleshooting/09-cloudctl-es-not-registered.md b/_es_11.5/08-troubleshooting/09-cloudctl-es-not-registered.md new file mode 100644 index 00000000..0584eb46 --- /dev/null +++ b/_es_11.5/08-troubleshooting/09-cloudctl-es-not-registered.md @@ -0,0 +1,31 @@ +--- +title: "Event Streams CLI fails with 'not a registered command' error" +excerpt: "Event Streams CLI command extension not registered." +categories: troubleshooting +slug: cloudctl-es-not-registered +toc: true +--- + +## Symptoms + +When running the `kubectl es` command, the following error message is displayed: + +``` +FAILED +'es' is not a registered command. See 'kubectl help'. +``` + +A similar error message is displayed when running the `cloudctl es` command: + +``` +FAILED +'es' is not a registered command. See 'cloudctl help'. +``` + +## Causes + +This error occurs when you attempt to use the {{site.data.reuse.es_name}} CLI before it is installed. + +## Resolving the problem + +Log in to the {{site.data.reuse.es_name}} UI, and [install the CLI](../../installing/post-installation/#installing-the-event-streams-command-line-interface). diff --git a/_es_11.5/08-troubleshooting/10-cloudctl-es-fails.md b/_es_11.5/08-troubleshooting/10-cloudctl-es-fails.md new file mode 100644 index 00000000..392f7cb4 --- /dev/null +++ b/_es_11.5/08-troubleshooting/10-cloudctl-es-fails.md @@ -0,0 +1,26 @@ +--- +title: "Event Streams CLI commands produces 'FAILED' message" +excerpt: "Event Streams CLI commands responds with FAILED error." +categories: troubleshooting +slug: cloudctl-es-fails +toc: true +--- + +## Symptoms + +When running the `cloudctl es` or `kubectl es` command, the following error message is displayed: + +```shell +FAILED +... +``` + +## Causes + +This error occurs when you have not logged in to the cluster and initialized the command-line tool. + +## Resolving the problem + +{{site.data.reuse.es_cli_init_111}} + +Re-run the failed operation again. diff --git a/_es_11.5/08-troubleshooting/11-chrome-ubuntu-issue.md b/_es_11.5/08-troubleshooting/11-chrome-ubuntu-issue.md new file mode 100644 index 00000000..40322605 --- /dev/null +++ b/_es_11.5/08-troubleshooting/11-chrome-ubuntu-issue.md @@ -0,0 +1,29 @@ +--- +title: "UI does not open when using Chrome on Ubuntu" +excerpt: "When using Google Chrome browser on Ubuntu operating systems, the Event Streams UI does not open." +categories: troubleshooting +slug: chrome-ubuntu-issue +toc: true +--- + +## Symptoms + +When using a Google Chrome browser on Ubuntu operating systems, the {{site.data.reuse.es_name}} UI does not open, and the browser displays an error message about invalid certificates, similar to the following example: + +``` +192.0.2.24 normally uses encryption to protect your information. +When Google Chrome tried to connect to 192.0.2.24 this time, the website sent back unusual and incorrect credentials. +This may happen when an attacker is trying to pretend to be 192.0.2.24, or a Wi-Fi sign-in screen has interrupted the connection. +Your information is still secure because Google Chrome stopped the connection before any data was exchanged. + +You cannot visit 192.0.2.24 at the moment because the website sent scrambled credentials that Google Chrome cannot process. +Network errors and attacks are usually temporary, so this page will probably work later. +``` + +## Causes + +The Google Chrome browser on Ubuntu systems requires a certificate that {{site.data.reuse.es_name}} does not currently provide. + +## Resolving the problem + +Use a different browser, such as Firefox, or launch Google Chrome with the following option: `--ignore-certificate-errors` diff --git a/_es_11.5/08-troubleshooting/17-error-removing-destination.md b/_es_11.5/08-troubleshooting/17-error-removing-destination.md new file mode 100644 index 00000000..01e9e5e7 --- /dev/null +++ b/_es_11.5/08-troubleshooting/17-error-removing-destination.md @@ -0,0 +1,37 @@ +--- +title: "Unable to remove destination cluster" +excerpt: "Error is displayed when trying to remove a destination cluster." +categories: troubleshooting +slug: error-removing-destination +toc: true +--- + +## Symptoms + +When trying to remove an offline geo-replication destination cluster, the following error message is displayed in the UI: + +``` +Failed to retrieve data for this destination cluster. +``` + +## Causes + +There could be several reasons, for example, the cluster might be offline, or the service ID of the cluster might have been revoked. + +## Resolving the problem + +1. Go to your origin cluster. {{site.data.reuse.cncf_cli_login}} +2. {{site.data.reuse.es_cli_init_111}} +3. Retrieve destination cluster IDs by using the following command: + + ```shell + kubectl es geo-clusters + ``` + + Look for the destination cluster ID that you want to remove. + +4. Run the following command: + + ```shell + kubectl es geo-cluster-remove --destination --force + ``` diff --git a/_es_11.5/08-troubleshooting/19-ui-403-error.md b/_es_11.5/08-troubleshooting/19-ui-403-error.md new file mode 100644 index 00000000..df16d020 --- /dev/null +++ b/_es_11.5/08-troubleshooting/19-ui-403-error.md @@ -0,0 +1,28 @@ +--- +title: "IAM and Keycloak: 403 error when logging in to Event Streams UI" +excerpt: "When signing into the Event Streams UI, the 403 Not Authorized page is displayed." +categories: troubleshooting +slug: ui-403-error +toc: true +--- + +## Symptoms + +Logging in to the {{site.data.reuse.es_name}} UI as a Keycloak user or an Identity and Access Management (IAM) user fails with the message `403 Not authorized`, indicating that the user does not have permission to access the {{site.data.reuse.es_name}} instance. + +{{site.data.reuse.iam_note}} + +## Causes + +To access the {{site.data.reuse.es_name}} UI: + +- The IAM user must either have the `Cluster Administrator` role or the `Administrator` role and be in a team with a namespace resource added for the namespace containing the {{site.data.reuse.es_name}} instance. If neither of these applies, the error will be displayed. + +- The Keycloak user must either have the `eventstreams-admin` role or the `admin` role and be in a team with a namespace resource added for the namespace containing the {{site.data.reuse.es_name}} instance. If neither of these applies, the error will be displayed. + +## Resolving the problem + +[Assign access to users](../../security/managing-access/#accessing-the-event-streams-ui-and-cli) with an administrator role by ensuring they are in a group with access to the correct namespace. + +- If you configured {{site.data.reuse.es_name}} with Keycloak, assign access to the `eventstreams-admin` or the `admin` role. +- If you configured {{site.data.reuse.es_name}} with IAM, assign the `Cluster Administrator` role or the `Administrator` role. diff --git a/_es_11.5/08-troubleshooting/30-default-scc-issues.md b/_es_11.5/08-troubleshooting/30-default-scc-issues.md new file mode 100644 index 00000000..5ee9e3e4 --- /dev/null +++ b/_es_11.5/08-troubleshooting/30-default-scc-issues.md @@ -0,0 +1,73 @@ +--- +title: "Event Streams not installing due to Security Context Constraint (SCC) issues" +excerpt: "When the default Security Context Constraint (SCC) is updated by user or another operator, Event Streams does not install" +categories: troubleshooting +slug: default-scc-issues +toc: true +--- + +## Symptoms + +{{site.data.reuse.es_name}} components report that an action is forbidden, stating that it is `unable to validate against any security context constraint`. + +This could result in symptoms such as: + +- Installation of the operator is pending and eventually times out. + + - Navigating to the **Conditions** section for the specific operator deployment under **Workloads > Deployment** will display a message similar to the following example: + + ```shell + pods "eventstreams-cluster-operator-55d6f4cdf7-" is forbidden: unable to validate against any security context constraint: [spec.volumes[0]: Invalid value: "secret": secret volumes are not allowed to be used spec.volumes[1]: Invalid value: "secret": secret volumes are not allowed to be used] + ``` + +- The installation of {{site.data.reuse.es_name}} instance is unsuccessful and the instance reports a `Failed` [status](../../installing/post-installation/). + + - The `conditions` field under status contains the following error message: + + ```shell + Exceeded timeout of 300000ms while waiting for Pods resource + light-insecure-zookeeper-0 in namespace es-1 to be ready + ``` + + - The status of the `-zookeeper` StrimziPodSet resource contains the following error message under the `conditions` field: + + ```shell + pods "light-insecure-zookeeper-0" is forbidden: unable to validate against any security context constraint: [provider "anyuid": + Forbidden: not usable by user or serviceaccount, spec.volumes[3]: Invalid value: "secret": secret volumes are not allowed to be used, + ``` + +- On a running instance of {{site.data.reuse.es_name}}, a pod that has bounced never comes back up. + + - Navigating to the **Conditions** section for the specific instance deployment under **Workloads > Deployment** will display a message similar to the following example: + + ```shell + is forbidden: unable to validate against any security context constraint: [spec.initContainers[0].securityContext.readOnlyRootFilesystem: Invalid value: false: ReadOnlyRootFilesystem must be set to true spec.containers[0].securityContext.readOnlyRootFilesystem: Invalid value: false: ReadOnlyRootFilesystem must be set to true] + ``` + +## Causes + +{{site.data.reuse.es_name}} has been tested with the default `restricted-v2` Security Context Constraint (SCC) provided by the {{site.data.reuse.openshift_short}}. + +If a user or any other operator applies a custom SCC that removes permissions required by {{site.data.reuse.es_name}}, then this will cause issues. + +## Resolving the problem + +Apply the custom [Security Context Constraint](https://github.com/IBM/ibm-event-automation/blob/main/support/event-automation-scc.yaml){:target="_blank"} (SCC) provided by {{site.data.reuse.ea_long}} to enable permissions required by the product. + +To do this, edit the `event-automation-scc.yaml` file to add your namespace and apply it using `oc` tool as follows: + +1. Edit the `event-automation-scc.yaml` and add the namespace where your {{site.data.reuse.es_name}} instance is installed. + +2. {{site.data.reuse.openshift_cli_login}} + +3. Run the following command to apply the SCC: + + ```shell + oc apply -f + ``` + + For example: + + ```shell + oc apply -f event-automation-scc.yaml + ``` diff --git a/_es_11.5/08-troubleshooting/32-authorization-failed-exceptions.md b/_es_11.5/08-troubleshooting/32-authorization-failed-exceptions.md new file mode 100644 index 00000000..87faab0e --- /dev/null +++ b/_es_11.5/08-troubleshooting/32-authorization-failed-exceptions.md @@ -0,0 +1,33 @@ +--- +title: "Client receives AuthorizationException when communicating with brokers" +excerpt: "When connecting a client to Event Streams, operations return AuthorizationException errors when executing." +categories: troubleshooting +slug: authorization-failed-exceptions +toc: true +--- + +## Symptoms + +When executing operations with a Java client connected to {{site.data.reuse.es_name}}, the client fails with an error similar to the following message: + +```shell +[err] [kafka-producer-network-thread | my-client-id] ERROR org.apache.kafka.clients.producer.internals.Sender - [Producer clientId=my-client-id] Aborting producer batches due to fatal error +[err] org.apache.kafka.common.errors.ClusterAuthorizationException: Cluster authorization failed. +``` + +Similar messages might also be displayed when using clients written in other languages such as NodeJS. + +## Causes + +The `KafkaUser` does not have the authorization to perform one of the operations: + +- If there is an authorization error with a `topic` resource, then a `TOPIC_AUTHORIZATION_FAILED (error code: 29)` will be returned. +- If there is an authorization error with a `group` resource, then a `GROUP_AUTHORIZATION_FAILED (error code: 30)` will be returned. +- If there is an authorization error with a `cluster` resource, then a `CLUSTER_AUTHORIZATION_FAILED (error code: 31)` will be returned. +- If there is an authorization error with a `transactionalId` resource, then a `TRANSACTIONAL_ID_AUTHORIZATION_FAILED (error code: 32)` will be returned. + +In Java, the errors are thrown in the format `AuthorizationException`. Other clients might return the error directly or translate it into an error with a similar name. + +## Resolving the problem + +Ensure the `KafkaUser` has the required permissions as described in [managing access](../../security/managing-access). diff --git a/_es_11.5/08-troubleshooting/42-pkcs12-keystore-java-client.md b/_es_11.5/08-troubleshooting/42-pkcs12-keystore-java-client.md new file mode 100644 index 00000000..f8b1e718 --- /dev/null +++ b/_es_11.5/08-troubleshooting/42-pkcs12-keystore-java-client.md @@ -0,0 +1,66 @@ +--- +title: "Client receives 'Failed to load SSL keystore' message when communicating with brokers" +excerpt: "When connecting a client to Event Streams, operations return 'Failed to load SSL keystore' errors when executing." +categories: troubleshooting +slug: pkcs12-keystore-java-client +toc: true +--- + +## Symptoms + +When executing operations with a Java client connected to {{site.data.reuse.es_name}}, the client fails with an error similar to the following: + +```java +org.apache.kafka.common.KafkaException: Failed to load SSL keystore /path/to/es-cert.p12 of type PKCS12 + at org.apache.kafka.common.security.ssl.DefaultSslEngineFactory$FileBasedStore.load(DefaultSslEngineFactory.java:377) + at org.apache.kafka.common.security.ssl.DefaultSslEngineFactory$FileBasedStore.(DefaultSslEngineFactory.java:349) + at org.apache.kafka.common.security.ssl.DefaultSslEngineFactory.createkeystore(DefaultSslEngineFactory.java:322) + at org.apache.kafka.common.security.ssl.DefaultSslEngineFactory.configure(DefaultSslEngineFactory.java:168) + ... +Caused by: java.io.IOException: Error extracting keyentry aliases from PFX + at com.ibm.crypto.provider.PKCS12KeyStoreOracle.engineLoad(Unknown Source) + at java.security.KeyStore.load(KeyStore.java:1456) +``` + +or: + +```shell +Caused by: java.io.IOException: parseAlgParameters failed: ObjectIdentifier() -- data isn't an object ID (tag = 48) + at sun.security.pkcs12.PKCS12KeyStore.parseAlgParameters(PKCS12KeyStore.java:829) + at sun.security.pkcs12.PKCS12KeyStore.engineLoad(PKCS12KeyStore.java:2037) + at java.security.KeyStore.load(KeyStore.java:1445) +``` + +This is seen in various Java-based clients including the Kafka connectors included in IBM App Connect Enterprise Version 12.0. + +## Causes + +The PKCS12-format keystore file that is generated in an {{site.data.reuse.es_name}} instance cannot be parsed by previous versions of Java. In {{site.data.reuse.es_name}} version 10.4.0 and later, the PKCS12-format keystore is generated by using a version of Java that includes [an upgrade to the default algorithms used in PKCS12 to use new encryption and stronger algorithms](https://bugs.openjdk.java.net/browse/JDK-8228481){:target="_blank"}. + +Previous releases of Java do not include the upgrade and cannot successfully parse the PKCS12-format keystore from {{site.data.reuse.es_name}}. + +## Resolving the problem + +To resolve the issue, you can update the version of Java that is running the Kafka client. Alternatively, you can work around the issue by converting the PCKS12-format keystore to a JKS-format keystore. + +The upgrade to the default algorithms used in PKCS12 was back-ported to various Java releases. To determine which version of your Java release includes the upgrade, see [the 'Backports' section in the associated OpenJDK issue](https://bugs.openjdk.java.net/browse/JDK-8214513){:target="_blank"}. + +To work around the issue you can convert the PCKS12-format keystore to a JKS-format keystore by using a tool such as `keytool`, and then provide that to the Java client instead. The following command will create a JKS-format keystore containing the certificates in the PKCS12-format keystore: + +```shell +keytool -importkeystore \ + -srckeystore es-cert.p12 \ + -srcstoretype PKCS12 \ + -srcstorepass \ + -destkeystore es-cert.jks \ + -deststoretype jks \ + -deststorepass +``` + +The Java client properties will then need to be updated to point to the new file and set the truststore type, for example: + +```shell +ssl.truststore.location=/path/to/es-cert.jks +ssl.truststore.password= +ssl.truststore.type=JKS +``` diff --git a/_es_11.5/08-troubleshooting/44-ocp-upgrade-fail.md b/_es_11.5/08-troubleshooting/44-ocp-upgrade-fail.md new file mode 100644 index 00000000..47325f70 --- /dev/null +++ b/_es_11.5/08-troubleshooting/44-ocp-upgrade-fail.md @@ -0,0 +1,57 @@ +--- +title: "OpenShift upgrade: fixing scheduling on node and node degraded errors" +excerpt: "OpenShift upgrade for Event Streams installations: how to fix failed to schedule on nodes and node degraded errors." +categories: troubleshooting +slug: ocp-upgrade-fail +toc: true +--- + +## Symptoms + +Upgrading your {{site.data.reuse.openshift_short}} version where {{site.data.reuse.es_name}} is installed fails, with error messages about `failing to schedule nodes` or nodes reporting a `degraded status`, similar to the following: + +```shell +Cluster operator machine-config cannot be upgraded between minor versions: One or more machine config pool is degraded. +``` + +## Causes + +When your {{site.data.reuse.openshift_short}} has less than 3 worker nodes, the {{site.data.reuse.es_name}} pod anti-affinity rules allow multiple Kafka or ZooKeeper pods to be scheduled on the same node. This can cause a block for {{site.data.reuse.openshift_short}} upgrades where terminating multiple Kafka or ZooKeeper pods on the node will violate the pod disruption budget, which prevents the node being drained. + +## Resolving the problem + +To resolve this issue, you can add an extra node, or force a rebalance of the Kafka and ZooKeeper pods across the nodes to allow the upgrade to continue. + +You can initiate a rebalance as follows: + +1. Identify the nodes that have multiple Kafka pods: + + ```shell + oc get pods -o wide + ``` + +2. Cordon off the nodes with any Kafka pods running, leaving only the node available that has no running Kafka pods scheduled or ready: + + ```shell + oc adm cordon + ``` + +3. Delete one of the Kafka pods on the node with two Kafka pods running. As only one node is scheduled or ready, the Kafka pod restarts on the available node. + + ```shell + oc delete pod + ``` + +4. Verify that the Kafka pods are now all distributed across the nodes: + + ```shell + oc get pods -o wide + ``` + +5. Remove the cordon from the nodes to make all nodes available: + + ```shell + oc adm uncordon + ``` + +6. Repeat the previous steps for ZooKeeper pods. diff --git a/_es_11.5/08-troubleshooting/48-watched-namespace.md b/_es_11.5/08-troubleshooting/48-watched-namespace.md new file mode 100644 index 00000000..611568fe --- /dev/null +++ b/_es_11.5/08-troubleshooting/48-watched-namespace.md @@ -0,0 +1,23 @@ +--- +title: "Apicurio authentication errors due to User Operator watchedNamespace" +excerpt: "Apicurio authentication errors due to User Operator watchedNamespace being set." +categories: troubleshooting +slug: watched-namespace +toc: true +--- + +## Symptoms + +Specifying `EventStreams.spec.strimziOverrides.entityOperator.userOperator.watchedNamespace` when installing {{site.data.reuse.es_name}} produces the following error in the Apicurio logs: + +```shell +2022-07-27 19:15:44,304 WARN [org.apa.kaf.cli.adm.int.AdminMetadataManager] (kafka-admin-client-thread | eventstreams-apicurio-registry-1c2d1d16-63b2-4369-bfd0-6f51552a9a01-admin) [AdminClient clientId=eventstreams-apicurio-registry-1c2d1d16-63b2-4369-bfd0-6f51552a9a01-admin] Metadata update failed due to authentication error: org.apache.kafka.common.errors.SaslAuthenticationException: Authentication failed during authentication due to invalid credentials with SASL mechanism SCRAM-SHA-512 +``` + +## Causes + +Apicurio uses `scram-sha-512` credentials generated by a `KafkaUser` resource. If the `watchedNamespace` field is set to a namespace that is not the same as the namespace set in the `EventStreams` custom resource, then the `KafkaUsers` created by {{site.data.reuse.es_name}} are ignored. + +## Resolving the problem + +Remove the field `EventStreams.spec.strimziOverrides.entityOperator.userOperator.watchedNamespace` from the `EventStreams` custom resource. diff --git a/_es_11.5/08-troubleshooting/49-apicurio-annotation.md b/_es_11.5/08-troubleshooting/49-apicurio-annotation.md new file mode 100644 index 00000000..bf823467 --- /dev/null +++ b/_es_11.5/08-troubleshooting/49-apicurio-annotation.md @@ -0,0 +1,50 @@ +--- +title: "Clients using schemas fail with Apicurio 2.5.0 or later" +excerpt: "Clients that use schemas with an earlier Apicurio version than 2.3.1 fail when the registry is migrated to 2.5.0 or later." +categories: troubleshooting +slug: upgrade-apicurio +toc: true +--- + +## Symptoms + +After upgrading to {{site.data.reuse.es_name}} 11.4.x and migrating to use the latest version of Apicurio that is included in {{site.data.reuse.es_name}}, client applications that use the schema registry with an earlier Apicurio version than 2.3.1 fail with errors similar to the following example: + +```shell +java.io.UncheckedIOException: com.fasterxml.jackson.databind.exc.InvalidFormatException: Cannot deserialize value of type `java.util.Date` from String "2022-12-13T20:10:29Z": expected format "yyyy-MM-dd'T'HH:mm:ssZ" + at [Source: (jdk.internal.net.http.ResponseSubscribers$HttpResponseInputStream); line: 1, column: 54] (through reference chain: io.apicurio.registry.rest.v2.beans.ArtifactMetaData["createdOn"]) + at io.apicurio.rest.client.handler.BodyHandler.lambda$toSupplierOfType$1(BodyHandler.java:60) + at io.apicurio.rest.client.JdkHttpClient.sendRequest(JdkHttpClient.java:204) + at io.apicurio.registry.rest.client.impl.RegistryClientImpl.createArtifact(RegistryClientImpl.java:263) + at io.apicurio.registry.rest.client.RegistryClient.createArtifact(RegistryClient.java:134) + at io.apicurio.registry.resolver.DefaultSchemaResolver.lambda$handleAutoCreateArtifact$2(DefaultSchemaResolver.java:236) + at io.apicurio.registry.resolver.ERCache.lambda$getValue$0(ERCache.java:142) + at io.apicurio.registry.resolver.ERCache.retry(ERCache.java:181) + at io.apicurio.registry.resolver.ERCache.getValue(ERCache.java:141) + at io.apicurio.registry.resolver.ERCache.getByContent(ERCache.java:121) + at io.apicurio.registry.resolver.DefaultSchemaResolver.handleAutoCreateArtifact(DefaultSchemaResolver.java:234) + at io.apicurio.registry.resolver.DefaultSchemaResolver.getSchemaFromRegistry(DefaultSchemaResolver.java:115) + at io.apicurio.registry.resolver.DefaultSchemaResolver.resolveSchema(DefaultSchemaResolver.java:88) + at io.apicurio.registry.serde.AbstractKafkaSerializer.serialize(AbstractKafkaSerializer.java:83) + at org.apache.kafka.clients.producer.KafkaProducer.doSend(KafkaProducer.java:929) + at org.apache.kafka.clients.producer.KafkaProducer.send(KafkaProducer.java:889) + at org.apache.kafka.clients.producer.KafkaProducer.send(KafkaProducer.java:775) +``` + +## Causes + +Apicurio client libraries versions 2.3.1 and earlier use a date format that is not compatible with Apicurio Registry server versions 2.5.0 or later. + +## Resolving the problem + +Ensure you are using the latest Apicurio version as follows: + +1. Ensure all applications connecting to your instance of {{site.data.reuse.es_name}} that use the schema registry are using Apicurio client libraries version 2.5.0 or later. + +1. Ensure that the migration to using the latest Apicurio image has completed by verifying that the image for the `apicurio-registry` container in the schema registry pod is `cp.icr.io/cp/ibm-eventstreams-apicurio-registry-v24@`. + + To get the images used in the schema registry pod, use the following command: + + ```shell + oc get pods -n es -l eventstreams.ibm.com/component-type=apicurio-registry-v2 -o jsonpath="{.items[*].spec.containers[*].image}" + ``` diff --git a/_es_11.5/08-troubleshooting/52-cruise-control-topics.md b/_es_11.5/08-troubleshooting/52-cruise-control-topics.md new file mode 100644 index 00000000..a1d88cec --- /dev/null +++ b/_es_11.5/08-troubleshooting/52-cruise-control-topics.md @@ -0,0 +1,43 @@ +--- +title: "KafkaRebalance custom resource remains in PendingProposal state" +excerpt: "KafkaRebalance custom resource remains in PendingProposal state due to incorrect Cruise Control topic configuration." +categories: troubleshooting +slug: kafkarebalance-pendingproposal +toc: true +--- + +## Symptoms + +A `KafkaRebalance` custom resource remains in the `PendingProposal` state, does not move to the `ProposalReady` state, and does not populate the `status.optimizationResult` property. The Cruise Control pod logs contain the following error: + +```shell +2023-04-13 10:55:09 ERROR KafkaCruiseControlRequestHandler:88 - Error processing POST request '/rebalance' due to: +'com.linkedin.kafka.cruisecontrol.exception.KafkaCruiseControlException: com.linkedin.cruisecontrol.exception.NotEnoughValidWindowsException: +There is no window available in range [-1, 1681383309158] (index [1, -1]). Window index (current: 0, oldest: 0).'. +java.util.concurrent.ExecutionException: com.linkedin.kafka.cruisecontrol.exception.KafkaCruiseControlException: +com.linkedin.cruisecontrol.exception.NotEnoughValidWindowsException: +There is no window available in range [-1, 1681383309158] (index [1, -1]). Window index (current: 0, oldest: 0). +at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] +Caused by: com.linkedin.kafka.cruisecontrol.exception.KafkaCruiseControlException: com.linkedin.cruisecontrol.exception.NotEnoughValidWindowsException: +There is no window available in range [-1, 1681383309158] (index [1, -1]). Window index (current: 0, oldest: 0). + +``` + +## Causes + +Cruise Control uses Kafka topics to store and process the metrics used to generate the optimization proposal. This error occurs because the default Cruise Control topic names are incorrectly set by the {{site.data.reuse.es_name}} operator. + +## Resolving the problem + +To set the correct topic names to be used by Cruise Control, add the following configuration to the `spec.strimziOverrides.cruiseControl.config` field in the `EventStreams` custom resource: + +```yaml +spec: + strimziOverrides: + cruiseControl: + config: + metric.reporter.topic: "eventstreams.cruisecontrol.metrics" + broker.metric.sample.store.topic: "eventstreams.cruisecontrol.modeltrainingsamples" + partition.metric.sample.store.topic: "eventstreams.cruisecontrol.partitionmetricsamples" +``` + diff --git a/_es_11.5/08-troubleshooting/53-default-psp-issues.md b/_es_11.5/08-troubleshooting/53-default-psp-issues.md new file mode 100644 index 00000000..aa78e17f --- /dev/null +++ b/_es_11.5/08-troubleshooting/53-default-psp-issues.md @@ -0,0 +1,59 @@ +--- +title: "Event Streams not installing due to Pod Security Policies (PSP) issues" +excerpt: "When the default Pod Security Policies (PSP) is updated by user or another operator, Event Streams does not install" +categories: troubleshooting +slug: default-psp-issues +toc: true +--- + +## Symptoms + +For Kubernetes versions earlier than 1.24, {{site.data.reuse.es_name}} components report that an action is forbidden, stating that it is `unable to validate against any pod security policy`. + +This might result in symptoms such as: + +- Installation of the operator is pending and eventually times out. + + - Navigating to the **Conditions** section for the specific operator deployment under **Workloads > Deployment** displays a message similar to the following example: + + ```shell + pods "eventstreams-cluster-operator-55d6f4cdf7-" is forbidden: unable to validate against any pod security policy: [spec.volumes[0]: Invalid value: "secret": secret volumes are not allowed to be used spec.volumes[1]: Invalid value: "secret": secret volumes are not allowed to be used] + ``` + +- The installation of {{site.data.reuse.es_name}} instance is unsuccessful and the instance reports a `Failed` [status](../../installing/post-installation/). + + - The `conditions` field under status contains the following error message: + + ```shell + Exceeded timeout of 300000ms while waiting for Pods resource + light-insecure-zookeeper-0 in namespace es-1 to be ready + ``` + + - The status of the `-zookeeper` StrimziPodSet resource contains the following error message under the `conditions` field: + + ```shell + pods "light-insecure-zookeeper-0" is forbidden: unable to validate against any pod security policy: [provider "anyuid": + Forbidden: not usable by user or serviceaccount, spec.volumes[3]: Invalid value: "secret": secret volumes are not allowed to be used, + ``` + +- On a running instance of {{site.data.reuse.es_name}}, a pod that has bounced never comes back up. + + - Navigating to the **Conditions** section for the specific instance deployment under **Workloads > Deployment** displays a message similar to the following example: + + ```shell + is forbidden: unable to validate against any pod security policy: [spec.initContainers[0].securityContext.readOnlyRootFilesystem: Invalid value: false: ReadOnlyRootFilesystem must be set to true spec.containers[0].securityContext.readOnlyRootFilesystem: Invalid value: false: ReadOnlyRootFilesystem must be set to true] + ``` + +## Causes + +{{site.data.reuse.es_name}} has been tested with the default `ibm-restricted-psp` Pod Security Policy (PSP) provided by IBM Cloud Pak. + +If a user or any other operator applies a custom PSP that removes permissions that are required by {{site.data.reuse.es_name}}, then it will cause issues. + +## Resolving the problem + +Apply the custom Pod Security Policy (PSP) provided by [IBM Cloud Pak](https://github.com/IBM/cloud-pak/blob/master/spec/security/psp/ibm-restricted-psp.yaml){:target="_blank"} to enable permissions required by the product. + +For information about applying the PSP, see the [Kubernetes documentation](https://kubernetes.io/docs/concepts/security/pod-security-policy){:target="_blank"}. + +**Note:** Pod Security Policies (PSP) is removed from Kubernetes versions 1.25 and later. diff --git a/_es_11.5/08-troubleshooting/54-georep-replicate-topic-fails.md b/_es_11.5/08-troubleshooting/54-georep-replicate-topic-fails.md new file mode 100644 index 00000000..3d1e43fb --- /dev/null +++ b/_es_11.5/08-troubleshooting/54-georep-replicate-topic-fails.md @@ -0,0 +1,41 @@ +--- +title: "Geo-replicator fails when replicating a topic" +excerpt: "EventStreamsGeoReplicator fails when replicating a topic." +categories: troubleshooting +slug: georep-fails +toc: true +--- + +## Symptoms + +{{site.data.reuse.es_name}} [geo-replicator](../../georeplication/about/) fails when replicating a topic with the following error: + +```shell +FAILED +Event Streams API request failed: +Error response from server. Status code: 500. Connection was closed +``` + +## Causes + +The {{site.data.reuse.es_name}} operator uses the `EventStreamsGeoReplicator` custom resource to create a configured `KafkaMirrorMaker2` custom resource. + +The behavior of the `EventStreamsGeoReplicator` has changed after moving to `StrimziPodSets` in {{site.data.reuse.es_name}} versions 11.2.0 and later. When you configure the `KafkaMirrorMaker2` custom resource, `StrimziPodSets` roll the pod, which means there is a period of time where no pods are available leading to the error mentioned earlier. + +## Resolving the problem + +You can resolve the problem in one of the following ways: + +- Specify multiple replicas in the `EventStreamsGeoReplicator` custom resource. + + This means that there is always a pod available to process the `geo-replicator-create` command, even when the connector is not available (it will not be available until both pods have rolled). + + For example, to configure geo-replication with `2` replicas, set the value of the `spec.replicas` field to `2`: + + ```yaml + spec: + # ... + replicas: 2 + ``` + +- Before entering the `geo-replicator-create` command, ensure the `KafkaMirrorMaker2` pods are completely rolled by checking their status is `Ready`. diff --git a/_es_11.5/08-troubleshooting/55-mq-connector-fails.md b/_es_11.5/08-troubleshooting/55-mq-connector-fails.md new file mode 100644 index 00000000..415bd251 --- /dev/null +++ b/_es_11.5/08-troubleshooting/55-mq-connector-fails.md @@ -0,0 +1,41 @@ +--- +title: "Errors in IBM MQ connectors" +excerpt: "Find out how to troubleshoot the common errors in IBM MQ connectors" +categories: troubleshooting +slug: mq-connector-fails +toc: true +--- + +Find out how to troubleshoot the common errors in IBM MQ connectors such as the IBM MQ source connector and the IBM MQ sink connector. + +## Exactly-once failure scenarios + +The MQ connector is designed to fail on start-up in certain cases to ensure that exactly-once delivery is not compromised. +In some of these failure scenarios, an IBM MQ administrator should remove messages from the exactly-once state queue before the MQ connector can start up and deliver messages from the sink or the source queue again. In these cases, the MQ connector has the `FAILED` status and the Kafka Connect logs describe any required administrative action. + +## `MQRC_NOT_AUTHORIZED` exception in MQ sink connector when enabling MQMD + +### Sypmtoms +When attempting to send a message to an IBM MQ queue, an MQException with code `MQRC_NOT_AUTHORIZED` (reason code `2035`) and completion code 2 is thrown. + +### Causes + +The `MQRC_NOT_AUTHORIZED` exception indicates insufficient permissions on the queue and the queue manager. + +### Resolving the problem + +1. **Review permissions**: Ensure that the user has necessary permissions for accessing the queue and the queue manager. +2. **Grant authority**: If the user does not have the necessary permissions, assign required authorities to the user. +3. **Set Context**: Set `WMQ_MQMD_MESSAGE_CONTEXT` property for required properties. + +### Additional guidance + +See the following guidance to help you avoid this error: + +- Verify that the length of all properties are correctly set within the allowed limit. +- Do not set the `JMS_IBM_MQMD_BackoutCount` property. +- Refer to the IBM MQ documentation for detailed configuration guidance: + + - [IBM MQ JMS Message Object Properties](https://www.ibm.com/docs/en/ibm-mq/9.3?topic=application-jms-message-object-properties): This documentation provides details about various properties that can be set on IBM MQ JMS message objects, including their names, types, and descriptions. + - [IBM MQ Developer Community](https://community.ibm.com/community/user/integration/home): The developer community for IBM MQ, where you can find forums, articles, and resources related to development and troubleshooting for IBM MQ. + - [IBM MQ troubleshooting guide](https://www.ibm.com/docs/en/ibm-mq/9.3?topic=mq-troubleshooting-support): IBM guide for troubleshooting common issues and errors in IBM MQ. diff --git a/_es_11.5/08-troubleshooting/56-kafkaprotocol.md b/_es_11.5/08-troubleshooting/56-kafkaprotocol.md new file mode 100644 index 00000000..bca21637 --- /dev/null +++ b/_es_11.5/08-troubleshooting/56-kafkaprotocol.md @@ -0,0 +1,48 @@ +--- +title: "Upgrading Event Streams fails due to `inter.broker.protocol.version` mismatch" +excerpt: "Find out how to troubleshoot the error when upgrading Event Streams on IBM Cloud Pak for Integration deployments." +categories: troubleshooting +slug: kafka-protocol-error +toc: true +--- + +## Symptoms + +When upgrading from an earlier version of {{site.data.reuse.es_name}} that is deployed on {{site.data.reuse.cp4i}} and you did not explicitly set the `inter.broker.protocol.version` in the `spec.strimziOverrides.kafka.config` section of your `EventStreams` custom resource, the upgrade fails with the following errors and warnings: + + +- `EventStreams` custom resource shows status as `Failed` with the following error: + + ```shell + FatalProblem-Error while waiting for restarted pod to become ready. + ``` + +- Kafka pods go into a `CrashLoopBackOff` state, and the following error is displayed in the logs: + + ```shell + CrashLoopBackOff indicates that the application within the container is failing to start properly. + ``` + +- A warning similar to the following is displayed in the Kafka pod logs: + + ```shell + Node disconnected. + Cancelled in-flight FETCH request with correlation id due to node being disconnected Client requested connection close from node + Error sending fetch request (sessionId=INVALID, epoch=INITIAL) to node : + java.io.IOException: Connection to 2 was disconnected before the response was read + ... + Error in response for fetch request... + java.io.IOException: Connection to 2 was disconnected before the response was read + ... + ``` + +## Causes + +Some Kafka brokers are using the newer protocol version while other Kafka brokers are still on the earlier versions. + +## Resolving the problem + +1. In the `spec.strimziOverrides.kafka.config` section of your `EventStreams` custom resource, add the `inter.broker.protocol.version` value to match the Kafka version that you are using. For example, if you are currently using the Kafka version `3.6.1`, set the value to `3.6`. +1. Wait for the Kafka pods to roll and the `EventStreams` instance to become ready. +1. Continue to [upgrade](../../installing/upgrading/) your {{site.data.reuse.es_name}} version. +1. Change the `inter.broker.protocol.version` value in the `EventStreams` custom resource to the Kafka version that is supported in your {{site.data.reuse.es_name}} version as described in the [post-upgrade tasks](../../installing/upgrading/#upgrade-the-kafka-broker-protocol-version). diff --git a/_includes/masthead.html b/_includes/masthead.html index 7701bc00..dfd761d6 100644 --- a/_includes/masthead.html +++ b/_includes/masthead.html @@ -6,7 +6,8 @@ ***DRAFT preview – intended for internal IBM use only***

-{% endif %} + {% endif %} + {% if page.collection contains "es_2019" or page.collection contains "es_2018" or page.collection contains "es_10" and page.collection != "es_10.5" %}
@@ -15,6 +16,7 @@
{% endif %} +
- +

Event Endpoint Management

11.3.0 - what's new: