diff --git a/docs/lakehouse-sink.md b/docs/lakehouse-sink.md index fbcc04f..f730a35 100644 --- a/docs/lakehouse-sink.md +++ b/docs/lakehouse-sink.md @@ -42,7 +42,7 @@ To build the Lakehouse sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-lakehouse-{{connector:version}}.nar + pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar ``` # How to configure @@ -127,7 +127,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-hudi-3.2.0-SNAPSHOT.nar", "processingGuarantees": "EFFECTIVELY_ONCE", "parallelism": 1, "configs": { @@ -151,7 +151,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-{{connector:version}}-cloud.nar", + "archive": "connectors/pulsar-io-hudi-3.2.0-SNAPSHOT-cloud.nar", "parallelism": 1, "processingGuarantees": "EFFECTIVELY_ONCE", "configs": { @@ -179,7 +179,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -207,7 +207,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.0-SNAPSHOT-cloud.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -238,7 +238,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -260,7 +260,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.0-SNAPSHOT-cloud.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -330,7 +330,7 @@ This example describes how to use the Lakehouse sink connector to fetch data fro 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar ``` 2. Start Pulsar in standalone mode. @@ -363,7 +363,7 @@ This example explains how to create a Lakehouse sink connector in an on-premises 1. Copy the NAR package of the Lakehouse sink connector to the Pulsar connectors directory. ```bash - cp pulsar-io-lakehouse-{{connector:version}}.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/docs/lakehouse-source.md b/docs/lakehouse-source.md index ccf4194..38601c4 100644 --- a/docs/lakehouse-source.md +++ b/docs/lakehouse-source.md @@ -42,7 +42,7 @@ To build the Lakehouse source connector from the source code, follow these steps ```bash ls target - pulsar-io-lakehouse-{{connector:version}}.nar + pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar ``` # How to configure @@ -90,7 +90,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -115,7 +115,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.0-SNAPSHOT-cloud.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -175,7 +175,7 @@ This example describes how to use the Lakehouse source connector to fetch data f 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar ``` 2. Start Pulsar in standalone mode. @@ -206,7 +206,7 @@ This example explains how to create a Lakehouse source connector in an on-premis 1. Copy the NAR package of the Lakehouse source connector to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.0-SNAPSHOT.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/pom.xml b/pom.xml index 1ed035f..8bb481c 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,4 @@ - + - + org.apache apache 18 - 4.0.0 - org.apache.pulsar.ecosystem pulsar-io-lakehouse - 2.11.0-SNAPSHOT + 3.2.0-SNAPSHOT Pulsar Ecosystem :: IO Connector :: Lake House It is a lakehouse connector streaming convert data between lakehouse and Apache Pulsar. - 8 8 @@ -45,11 +40,10 @@ true 4.2.2 0 - 2.13.2.1 1.18.22 - 2.10.0.4 + 3.2.0-SNAPSHOT 2.17.2 1.7.25 3.2.4 @@ -62,12 +56,10 @@ 1.12.220 hadoop3-2.2.1 2.12.0 - 7.3.0 4.0.3 3.12.4 - 3.0 3.1.2 @@ -76,10 +68,9 @@ 1.2.0 8.37 4.2.2 - - + + - Apache License, Version 2.0 @@ -87,7 +78,6 @@ repo - @@ -181,8 +171,6 @@ log4j-slf4j-impl ${log4j2.version} - - org.apache.iceberg @@ -214,20 +202,17 @@ iceberg-hive-metastore ${iceberg.version} - org.apache.parquet parquet-avro ${parquet.avro.version} - io.delta delta-standalone_2.12 ${delta.version} - org.apache.parquet parquet-column @@ -238,7 +223,6 @@ parquet-hadoop ${parquet.version} - org.slf4j slf4j-api @@ -249,7 +233,6 @@ slf4j-simple ${slf4j.version} - org.apache.hudi @@ -263,7 +246,6 @@ - org.apache.hadoop @@ -281,14 +263,12 @@ aws-java-sdk-bundle ${aws.sdk.version} - com.google.cloud.bigdataoss gcs-connector ${gcs.version} shaded - org.apache.hadoop hadoop-azure @@ -304,10 +284,7 @@ - - - org.testng @@ -333,13 +310,10 @@ ${mockito.version} test - - - org.projectlombok @@ -351,13 +325,11 @@ spotbugs-annotations provided - io.streamnative pulsar-io-core - com.fasterxml.jackson.core jackson-databind @@ -371,37 +343,30 @@ org.apache.hadoop hadoop-client - org.apache.iceberg iceberg-api - org.apache.iceberg iceberg-core - org.apache.iceberg iceberg-parquet - org.apache.iceberg iceberg-common - org.apache.iceberg iceberg-data - org.apache.iceberg iceberg-hive-metastore - io.delta delta-standalone_2.12 @@ -410,7 +375,6 @@ org.apache.parquet parquet-avro - org.apache.parquet parquet-column @@ -419,7 +383,6 @@ org.apache.parquet parquet-hadoop - org.slf4j slf4j-api @@ -428,7 +391,6 @@ org.slf4j slf4j-simple - org.apache.logging.log4j log4j-slf4j-impl @@ -449,12 +411,10 @@ org.apache.logging.log4j log4j-slf4j-impl - io.netty netty-common - org.apache.hudi hudi-java-client @@ -473,7 +433,6 @@ - org.apache.curator curator-framework @@ -485,7 +444,6 @@ - org.apache.hudi hudi-client-common @@ -499,7 +457,6 @@ - org.testng @@ -517,7 +474,6 @@ test - cloud @@ -541,13 +497,11 @@ com.amazonaws aws-java-sdk-bundle - com.google.cloud.bigdataoss gcs-connector shaded - org.apache.hadoop hadoop-azure @@ -585,7 +539,6 @@ - ${project.artifactId}-${project.version}${cloudPkgSuffix} @@ -601,7 +554,7 @@ -Xlint:unchecked -Xpkginfo:always - + @@ -713,38 +666,32 @@ - - + org.apache.maven.plugins maven-compiler-plugin - org.apache.maven.plugins maven-surefire-plugin - org.apache.nifi nifi-nar-maven-plugin - com.mycila license-maven-plugin - org.apache.maven.plugins maven-checkstyle-plugin - com.github.spotbugs @@ -752,5 +699,24 @@ - + + + ossrh + https://s01.oss.sonatype.org/service/local/repositories/0/content + + + nexus-snapshot + https://s01.oss.sonatype.org/content/repositories/snapshots + + + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + +