diff --git a/docs/products/kafka/kafka-connect/concepts/list-of-connector-plugins.md b/docs/products/kafka/kafka-connect/concepts/list-of-connector-plugins.md index f424ad0c..c7887019 100644 --- a/docs/products/kafka/kafka-connect/concepts/list-of-connector-plugins.md +++ b/docs/products/kafka/kafka-connect/concepts/list-of-connector-plugins.md @@ -7,12 +7,10 @@ Discover a variety of connectors available for use with any Aiven for Apache Kaf ## Source connectors -Source connectors enable the integration of data from an existing -technology into an Apache Kafka topic. The following is the list of -available source connectors: +Source connectors enable the integration of data from an existing technology into an +Apache Kafka topic. The available source connectors include: - [Couchbase](https://github.com/couchbase/kafka-connect-couchbase) -- [Official MongoDB®](https://www.mongodb.com/docs/kafka-connector/current/) - [Debezium for MongoDB®](https://debezium.io/docs/connectors/mongodb/) - [Debezium for MySQL](https://debezium.io/docs/connectors/mysql/) - [Debezium for PostgreSQL®](/docs/products/kafka/kafka-connect/howto/debezium-source-connector-pg) @@ -20,27 +18,29 @@ available source connectors: - [Google Cloud Pub/Sub](https://github.com/googleapis/java-pubsub-group-kafka-connector/) - [Google Cloud Pub/Sub Lite](https://github.com/googleapis/java-pubsub-group-kafka-connector/) - [JDBC](https://github.com/aiven/jdbc-connector-for-apache-kafka/blob/master/docs/source-connector.md) +- [Official MongoDB®](https://www.mongodb.com/docs/kafka-connector/current/) - [Stream Reactor Cassandra®](https://docs.lenses.io/5.1/connectors/sources/cassandrasourceconnector/) - [Stream Reactor MQTT](https://docs.lenses.io/5.1/connectors/sources/mqttsourceconnector/) ## Sink connectors -Sink connectors enable the integration of data from an existing Apache -Kafka topic to a target technology. The following is the list of -available sink connectors: +Sink connectors enable the integration of data from an existing Apache Kafka topic to a +target technology. The available sink connectors include: -- [Aiven for Apache Kafka® S3 sink connector](/docs/products/kafka/kafka-connect/howto/s3-sink-connector-aiven) +- [Amazon S3 sink connector](/docs/products/kafka/kafka-connect/howto/s3-sink-connector-aiven) +- [Azure Blob Storage sink connector](/docs/products/kafka/kafka-connect/howto/azure-blob-sink) - [Confluent Amazon S3 sink](/docs/products/kafka/kafka-connect/howto/s3-sink-connector-confluent) - [Couchbase®](https://github.com/couchbase/kafka-connect-couchbase) -- [OpenSearch®](/docs/products/kafka/kafka-connect/howto/opensearch-sink) - [Elasticsearch](/docs/products/kafka/kafka-connect/howto/elasticsearch-sink) - [Google BigQuery](https://github.com/confluentinc/kafka-connect-bigquery) - [Google Cloud Pub/Sub](https://github.com/googleapis/java-pubsub-group-kafka-connector/) - [Google Cloud Pub/Sub Lite](https://github.com/googleapis/java-pubsub-group-kafka-connector/) - [Google Cloud Storage](/docs/products/kafka/kafka-connect/howto/gcs-sink) - [HTTP](https://github.com/aiven/http-connector-for-apache-kafka) +- [IBM MQ sink connector](/docs/products/kafka/kafka-connect/howto/ibm-mq-sink-connector) - [JDBC](https://github.com/aiven/jdbc-connector-for-apache-kafka/blob/master/docs/sink-connector.md) - [Official MongoDB®](https://docs.mongodb.com/kafka-connector/current/) +- [OpenSearch®](/docs/products/kafka/kafka-connect/howto/opensearch-sink) - [Snowflake](https://docs.snowflake.com/en/user-guide/kafka-connector) - [Splunk](https://github.com/splunk/kafka-connect-splunk) - [Stream Reactor Cassandra®](https://docs.lenses.io/5.1/connectors/sinks/cassandrasinkconnector/) @@ -48,7 +48,6 @@ available sink connectors: - [Stream Reactor MongoDB®](https://docs.lenses.io/5.1/connectors/sinks/mongosinkconnector/) - [Stream Reactor MQTT](https://docs.lenses.io/5.1/connectors/sinks/mqttsinkconnector/) - [Stream Reactor Redis®*](https://docs.lenses.io/5.1/connectors/sinks/redissinkconnector/) -- [IBM MQ sink connector](/docs/products/kafka/kafka-connect/howto/ibm-mq-sink-connector) ## Preview connectors diff --git a/docs/products/kafka/kafka-connect/howto/azure-blob-sink.md b/docs/products/kafka/kafka-connect/howto/azure-blob-sink.md new file mode 100644 index 00000000..2bf2892b --- /dev/null +++ b/docs/products/kafka/kafka-connect/howto/azure-blob-sink.md @@ -0,0 +1,137 @@ +--- +title: Create an Azure Blob Storage sink connector for Aiven for Apache Kafka® +sidebar_label: Azure Blob sink connector +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import ConsoleLabel from "@site/src/components/ConsoleIcons"; + +The Azure Blob Storage sink connector moves data from Apache Kafka® topics to Azure Blob Storage containers for long-term storage, such as archiving or creating backups. + +## Prerequisites + +Before you begin, make sure you have: + +- An + [Aiven for Apache Kafka® service](https://docs.aiven.io/docs/products/kafka/kafka-connect/howto/enable-connect) + with Kafka Connect enabled, or a + [dedicated Aiven for Apache Kafka Connect® service](https://docs.aiven.io/docs/products/kafka/kafka-connect/get-started#apache_kafka_connect_dedicated_cluster). +- Access to an Azure Storage account and a container with the following: + - Azure Storage connection string: Required to authenticate and + connect to your Azure Storage account. + - Azure Storage container name: Name of the Azure Blob Storage container where data is + saved. + +## Create an Azure Blob Storage sink connector configuration file + +Create a file named `azure_blob_sink_connector.json` with the following configuration: + +```json +{ + "name": "azure_blob_sink", + "connector.class": "io.aiven.kafka.connect.azure.sink.AzureBlobSinkConnector", + "tasks.max": "1", + "topics": "test-topic", + "azure.storage.connection.string": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net", + "azure.storage.container.name": "my-container", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter", + "header.converter": "org.apache.kafka.connect.storage.StringConverter", + "file.name.prefix": "connect-azure-blob-sink/test-run/", + "file.compression.type": "gzip", + "format.output.fields": "key,value,offset,timestamp", + "reload.action": "restart" +} +``` + +Parameters: + +- `name`: Name of the connector. +- `topics`: Apache Kafka topics to sink data from. +- `azure.storage.connection.string`: Azure Storage connection string. +- `azure.storage.container.name`: Azure Blob Storage container name. +- `key.converter`: Class used to convert the Apache Kafka record key. +- `value.converter`: Class used to convert the Apache Kafka record value. +- `header.converter`: Class used to convert message headers. +- `file.name.prefix`: Prefix for the files created in Azure Blob Storage. +- `file.compression.type`: Compression type for the files, such as `gzip`. +- `reload.action`: Action to take when reloading the connector, set to `restart`. + +You can view the full set of available parameters and advanced configuration options +in the [Aiven Azure Blob Storage sink connector GitHub repository](https://github.com/Aiven-Open/cloud-storage-connectors-for-apache-kafka/blob/main/azure-sink-connector/README.md). + +## Create the connector + + + + +1. Access the [Aiven Console](https://console.aiven.io/). +1. Select your Aiven for Apache Kafka® or Aiven for Apache Kafka Connect® service. +1. Click . +1. Click **Create connector** if Kafka Connect is already enabled on the service. + If not, click **Enable connector on this service**. + + Alternatively, to enable connectors: + + 1. Click in the sidebar. + 1. In the **Service management** section, click + > **Enable Kafka connect**. + +1. In the sink connectors, find **Azure Blob Storage sink**, and click **Get started**. +1. On the **Azure Blob Storage sink** connector page, go to the **Common** tab. +1. Locate the **Connector configuration** text box and click . +1. Paste the configuration from your `azure_blob_sink_connector.json` file into the + text box. +1. Click **Create connector**. +1. Verify the connector status on the page. + + + + + + +To create the Azure Blob Storage sink connector using the Aiven CLI, run: + +```bash +avn service connector create SERVICE_NAME @azure_blob_sink_connector.json +``` + +Parameters: + +- `SERVICE_NAME`: Name of your Aiven for Apache Kafka® service. +- `@azure_blob_sink_connector.json`: Path to your JSON configuration file. + + + + +## Example: Define and create an Azure Blob Storage sink connector + +This example shows how to create an Azure Blob Storage sink connector with the following properties: + +- Connector name: `azure_blob_sink` +- Apache Kafka topic: `test-topic` +- Azure Storage connection string: `DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net` +- Azure container: `my-container` +- Output fields: `key, value, offset, timestamp` +- File name prefix: `connect-azure-blob-sink/test-run/` +- Compression type: `gzip` + +```json +{ + "name": "azure_blob_sink", + "connector.class": "io.aiven.kafka.connect.azure.sink.AzureBlobSinkConnector", + "tasks.max": "1", + "topics": "test-topic", + "azure.storage.connection.string": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net", + "azure.storage.container.name": "my-container", + "format.output.fields": "key,value,offset,timestamp", + "file.name.prefix": "connect-azure-blob-sink/test-run/", + "file.compression.type": "gzip" +} +``` + +Once this configuration is saved in the `azure_blob_sink_connector.json` file, you can +create the connector using the Aiven Console or CLI, and verify that data from the +Apache Kafka topic `test-topic` is successfully delivered to your Azure Blob +Storage container. diff --git a/sidebars.ts b/sidebars.ts index 2f8e230d..40cf1107 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -963,33 +963,33 @@ const sidebars: SidebarsConfig = { { type: 'category', label: 'Sink connectors', - items: [ - 'products/kafka/kafka-connect/howto/jdbc-sink', - 'products/kafka/kafka-connect/howto/s3-sink-prereq', - 'products/kafka/kafka-connect/howto/s3-sink-connector-aiven', - 'products/kafka/kafka-connect/howto/s3-iam-assume-role', - 'products/kafka/kafka-connect/howto/s3-sink-connector-confluent', - 'products/kafka/kafka-connect/howto/gcs-sink-prereq', - 'products/kafka/kafka-connect/howto/gcs-sink', + 'products/kafka/kafka-connect/howto/azure-blob-sink', + 'products/kafka/kafka-connect/howto/cassandra-streamreactor-sink', + 'products/kafka/kafka-connect/howto/couchbase-sink', + 'products/kafka/kafka-connect/howto/elasticsearch-sink', 'products/kafka/kafka-connect/howto/gcp-bigquery-sink-prereq', 'products/kafka/kafka-connect/howto/gcp-bigquery-sink', - 'products/kafka/kafka-connect/howto/opensearch-sink', - 'products/kafka/kafka-connect/howto/elasticsearch-sink', - 'products/kafka/kafka-connect/howto/snowflake-sink-prereq', - 'products/kafka/kafka-connect/howto/snowflake-sink', + 'products/kafka/kafka-connect/howto/gcp-pubsub-lite-sink', + 'products/kafka/kafka-connect/howto/gcp-pubsub-sink', + 'products/kafka/kafka-connect/howto/gcs-sink-prereq', + 'products/kafka/kafka-connect/howto/gcs-sink', 'products/kafka/kafka-connect/howto/http-sink', - 'products/kafka/kafka-connect/howto/mongodb-sink-mongo', - 'products/kafka/kafka-connect/howto/mongodb-sink-lenses', + 'products/kafka/kafka-connect/howto/ibm-mq-sink-connector', 'products/kafka/kafka-connect/howto/influx-sink', + 'products/kafka/kafka-connect/howto/jdbc-sink', + 'products/kafka/kafka-connect/howto/mongodb-sink-lenses', + 'products/kafka/kafka-connect/howto/mongodb-sink-mongo', + 'products/kafka/kafka-connect/howto/mqtt-sink-connector', + 'products/kafka/kafka-connect/howto/opensearch-sink', 'products/kafka/kafka-connect/howto/redis-streamreactor-sink', - 'products/kafka/kafka-connect/howto/cassandra-streamreactor-sink', - 'products/kafka/kafka-connect/howto/couchbase-sink', - 'products/kafka/kafka-connect/howto/gcp-pubsub-sink', - 'products/kafka/kafka-connect/howto/gcp-pubsub-lite-sink', + 'products/kafka/kafka-connect/howto/s3-iam-assume-role', + 'products/kafka/kafka-connect/howto/s3-sink-connector-aiven', + 'products/kafka/kafka-connect/howto/s3-sink-connector-confluent', + 'products/kafka/kafka-connect/howto/s3-sink-prereq', + 'products/kafka/kafka-connect/howto/snowflake-sink-prereq', + 'products/kafka/kafka-connect/howto/snowflake-sink', 'products/kafka/kafka-connect/howto/splunk-sink', - 'products/kafka/kafka-connect/howto/mqtt-sink-connector', - 'products/kafka/kafka-connect/howto/ibm-mq-sink-connector', ], }, ],