diff --git a/docs/products/kafka/kafka-mirrormaker/howto/exactly-once-delivery.md b/docs/products/kafka/kafka-mirrormaker/howto/exactly-once-delivery.md new file mode 100644 index 000000000..9858d73bc --- /dev/null +++ b/docs/products/kafka/kafka-mirrormaker/howto/exactly-once-delivery.md @@ -0,0 +1,192 @@ +--- +title: Exactly-once delivery in Aiven for Apache Kafka MirrorMaker 2 +sidebar_label: Exactly-once delivery +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import ConsoleLabel from "@site/src/components/ConsoleIcons" +import ConsoleIcon from "@site/src/components/ConsoleIcons" + +Exactly-once delivery in Aiven for Apache Kafka MirrorMaker 2 replicates each message exactly once between clusters, preventing duplicates or data loss. + +## Exactly-once delivery semantics + +Exactly-once delivery semantics provide a transactional guarantee for message +replication, ensuring that all messages in a batch are either fully committed to the +target cluster or not replicated at all. This maintains data consistency across clusters. + + +:::note +Exactly-once delivery does not require ACL modifications by default. However, in +external Apache Kafka setups where ACLs are applied to the `TransactionalId` resource, +review and adjust these ACLs as needed to enable exactly-once delivery. +::: + +## Prerequisites + +- Aiven for Apache Kafka service with [Aiven for Apache Kafka MirrorMaker 2 integration + enabled](/docs/products/kafka/kafka-mirrormaker/get-started) +- Access to [create or modify replication flows](/docs/products/kafka/kafka-mirrormaker/howto/setup-replication-flow) +- Access to one of the following, depending on your preferred method: + - [Aiven Console](https://console.aiven.io/) + - [Aiven CLI](/docs/tools/cli) + - [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest/docs) + - [Aiven API](https://api.aiven.io/) + +## Enable or disable exactly-once delivery + + + + +1. In the [Aiven Console](https://console.aiven.io/), open your Aiven for Apache Kafka + service with the Aiven for Apache Kafka MirrorMaker 2 integration. +1. Click . +1. Click the Aiven for Apache Kafka MirrorMaker 2 integration. +1. Click in the sidebar. + - To create a new flow, click **Create replication flow** . + - To modify an existing flow, select . +1. Set **Exactly-once message delivery enabled** to **Enabled** to turn it on or + **Disabled** to turn it off. +1. Click **Create** or **Save**. + + + + +- Create a new replication flow with exactly-once delivery enabled: + + ```bash + avn mirrormaker replication-flow create \ + --source-cluster \ + --target-cluster \ + --replication_flow_config '{ "exactly_once_delivery_enabled": true }' + ``` + +- Enable exactly-once delivery on an existing replication flow: + + ```bash + avn mirrormaker replication-flow update \ + --source-cluster \ + --target-cluster \ + --replication_flow_config '{ "exactly_once_delivery_enabled": true }' + ``` + +- Disable exactly-once delivery on an existing replication flow: + + ```bash + avn mirrormaker replication-flow update \ + --source-cluster \ + --target-cluster \ + --replication_flow_config '{ "exactly_once_delivery_enabled": false }' + ``` + +Parameters: + +- `SERVICE_NAME`: Name of your Aiven for Apache Kafka MirrorMaker 2 service. +- `SOURCE_CLUSTER`: Alias of the source Aiven for Apache Kafka cluster. +- `TARGET_CLUSTER`: Alias of the target Aiven for Apache Kafka cluster. +- `EXACTLY_ONCE_DELIVERY_ENABLED`: Set to `true` to enable or `false` to disable + exactly-once delivery. + + + + + +Use the [ServiceKafkaMirrorMakerCreateReplicationFlow](https://api.aiven.io/doc/#tag/Service:_Kafka_MirrorMaker/operation/ServiceKafkaMirrorMakerCreateReplicationFlow) API to create or update a replication flow +with `exactly_once_delivery_enabled` in the configuration. + +- Create a new replication flow with exactly-once delivery enabled: + + ```bash + curl -X POST "https://console.aiven.io/v1/project//service//mirrormaker/replication-flows" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "source_cluster": "", + "target_cluster": "", + "enabled": true, + "exactly_once_delivery_enabled": true + }' + ``` + +- Update an existing replication flow to enable exactly-once delivery: + + ```bash + curl -X PUT "https://console.aiven.io/v1/project//service//mirrormaker/replication-flows//" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "exactly_once_delivery_enabled": true + }' + ``` + +- Update an existing replication flow to disable exactly-once delivery: + + ```bash + curl -X PUT "https://console.aiven.io/v1/project//service//mirrormaker/replication-flows//" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "exactly_once_delivery_enabled": false + }' + ``` + +Parameters: + +- `PROJECT_NAME`: Name of your project. +- `SERVICE_NAME`: Name of your Aiven for Apache Kafka MirrorMaker 2 service. +- `API_TOKEN`: Your personal Aiven API + [token](/docs/platform/howto/create_authentication_token). +- `SOURCE_CLUSTER`: Alias of the source Aiven for Apache Kafka cluster. +- `TARGET_CLUSTER`: Alias of the target Aiven for Apache Kafka cluster. +- `EXACTLY_ONCE_DELIVERY_ENABLED`: Set to `true` to enable or `false` to disable + exactly-once delivery. + + + + +Find information about the `aiven_mirrormaker_replication_flow` resource in the +[Aiven Provider for Terraform documentation](https://registry.terraform.io/providers/aiven/aiven/latest/docs/resources/mirrormaker_replication_flow). + +To enable exactly-once delivery: + +1. Update or create the `aiven_mirrormaker_replication_flow` resource in your + Terraform configuration file: + + - Add `exactly_once_delivery_enabled = true` to the resource block. + - Specify the `topics` to replicate using patterns, such as `.*` for all topics. + + ```hcl + resource "aiven_mirrormaker_replication_flow" "example_flow" { + project = "PROJECT_NAME" + service_name = "SERVICE_NAME" + source_cluster = "SOURCE_CLUSTER" + target_cluster = "TARGET_CLUSTER" + enable = true + exactly_once_delivery_enabled = true + topics = [".*"] + } + + ``` + +1. Run the Terraform commands to apply the configuration: + + ```bash + terraform init + terraform plan + terraform apply + ``` + +Parameters + +- `PROJECT_NAME`: Name of your Aiven project. +- `SERVICE_NAME`: Name of your Aiven for Apache Kafka MirrorMaker 2 service. +- `SOURCE_CLUSTER`: Alias of the source Aiven for Apache Kafka service. +- `TARGET_CLUSTER`: Alias of the target Aiven for Apache Kafka service. +- `EXACTLY_ONCE_DELIVERY_ENABLED`: Set to `true` to enable or `false` to disable + exactly-once delivery. +- `TOPICS`: Pattern to define which topics to replicate. The default value, `.*`, + replicates all topics. + + + diff --git a/sidebars.ts b/sidebars.ts index fd17ab4d8..6198cb5c5 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -1060,6 +1060,7 @@ const sidebars: SidebarsConfig = { 'products/kafka/kafka-mirrormaker/howto/remove-mirrormaker-prefix', 'products/kafka/kafka-mirrormaker/howto/datadog-customised-metrics', 'products/kafka/kafka-mirrormaker/howto/log-analysis-offset-sync-tool', + 'products/kafka/kafka-mirrormaker/howto/exactly-once-delivery', ], }, { diff --git a/src/components/ConsoleIcons/index.tsx b/src/components/ConsoleIcons/index.tsx index 8f35521a3..bc893d786 100644 --- a/src/components/ConsoleIcons/index.tsx +++ b/src/components/ConsoleIcons/index.tsx @@ -500,6 +500,14 @@ export default function ConsoleLabel({name}): ReactElement { Query editor ); + case 'replicationflow': + return ( + <> + + Replication flow + + ); + default: return (