diff --git a/spark-connector-obkv-hbase/pom.xml b/spark-connector-obkv-hbase/pom.xml index c3ed87a..b496267 100644 --- a/spark-connector-obkv-hbase/pom.xml +++ b/spark-connector-obkv-hbase/pom.xml @@ -42,7 +42,7 @@ under the License. com.oceanbase obkv-hbase-client - 0.1.5 + 2.0.0 junit diff --git a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-2.4/pom.xml b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-2.4/pom.xml index 36d23a4..95c1146 100644 --- a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-2.4/pom.xml +++ b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-2.4/pom.xml @@ -88,24 +88,16 @@ under the License. package + - com.alibaba:* com.oceanbase:* - com.alipay.sofa:bolt - com.alipay.sofa.common:sofa-common-tools + com.alibaba:* + com.alipay.*:* + com.google.*:* + org.apache.hbase*:* mysql:mysql-connector-java - org.apache.hbase:* - false - - - com.oceanbase:obkv-table-client - - **/log4j/log-conf.xml - - - diff --git a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-2.4/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-2.4/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml deleted file mode 100644 index 989e8c7..0000000 --- a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-2.4/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.1/pom.xml b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.1/pom.xml index c9b0441..11f0071 100644 --- a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.1/pom.xml +++ b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.1/pom.xml @@ -69,20 +69,13 @@ under the License. com.oceanbase:* com.alibaba:* - com.alipay.sofa:bolt - com.alipay.sofa.common:sofa-common-tools - org.apache.hbase:* + com.alipay.*:* + com.google.*:* + org.apache.hadoop*:* + org.apache.hbase*:* mysql:mysql-connector-java - - - com.oceanbase:obkv-table-client - - **/log4j/log-conf.xml - - - diff --git a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.1/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.1/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml deleted file mode 100644 index 989e8c7..0000000 --- a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.1/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.2/pom.xml b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.2/pom.xml index 4c319bd..3ecccac 100644 --- a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.2/pom.xml +++ b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.2/pom.xml @@ -69,20 +69,13 @@ under the License. com.oceanbase:* com.alibaba:* - com.alipay.sofa:bolt - com.alipay.sofa.common:sofa-common-tools - org.apache.hbase:* + com.alipay.*:* + com.google.*:* + org.apache.hadoop*:* + org.apache.hbase*:* mysql:mysql-connector-java - - - com.oceanbase:obkv-table-client - - **/log4j/log-conf.xml - - - diff --git a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.2/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.2/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml deleted file mode 100644 index 989e8c7..0000000 --- a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.2/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.3/pom.xml b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.3/pom.xml index 9b52d2f..d46d46c 100644 --- a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.3/pom.xml +++ b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.3/pom.xml @@ -69,19 +69,15 @@ under the License. com.oceanbase:* com.alibaba:* - com.alipay.sofa:bolt - com.alipay.sofa.common:sofa-common-tools - org.apache.hbase:* + com.alipay.*:* + com.google.*:* + org.apache.hadoop*:* + org.apache.hbase*:* mysql:mysql-connector-java + + com.lmax:disruptor - - - - org.apache.logging - shade.org.apache.logging - - diff --git a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.4/pom.xml b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.4/pom.xml index bdb30b6..c5fdb06 100644 --- a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.4/pom.xml +++ b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-3.4/pom.xml @@ -69,15 +69,19 @@ under the License. com.oceanbase:* com.alibaba:* - com.alipay.sofa:bolt - com.alipay.sofa.common:sofa-common-tools - org.apache.hbase:* + com.alipay.*:* + com.google.*:* + org.apache.hadoop*:* + org.apache.hbase*:* mysql:mysql-connector-java + + com.lmax:disruptor - + org.apache.logging shade.org.apache.logging diff --git a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-base/src/main/java/com/oceanbase/spark/obkv/HTableClientUtils.java b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-base/src/main/java/com/oceanbase/spark/obkv/HTableClientUtils.java index 0326a75..93da542 100644 --- a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-base/src/main/java/com/oceanbase/spark/obkv/HTableClientUtils.java +++ b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-base/src/main/java/com/oceanbase/spark/obkv/HTableClientUtils.java @@ -24,14 +24,14 @@ import com.alipay.oceanbase.hbase.OHTableClient; import com.alipay.oceanbase.hbase.constants.OHConstants; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Table; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class HTableClientUtils { private static final Logger LOG = LoggerFactory.getLogger(HTableClientUtils.class); - public static HTableInterface getHTableClient(OBKVHbaseConfig config) { + public static Table getHTableClient(OBKVHbaseConfig config) { try { OHTableClient tableClient = new OHTableClient(config.getTableName(), getConfig(config)); tableClient.init(); diff --git a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-base/src/main/scala/com/oceanbase/spark/HBaseRelation.scala b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-base/src/main/scala/com/oceanbase/spark/HBaseRelation.scala index 75cd4af..d48bab7 100644 --- a/spark-connector-obkv-hbase/spark-connector-obkv-hbase-base/src/main/scala/com/oceanbase/spark/HBaseRelation.scala +++ b/spark-connector-obkv-hbase/spark-connector-obkv-hbase-base/src/main/scala/com/oceanbase/spark/HBaseRelation.scala @@ -15,12 +15,12 @@ */ package com.oceanbase.spark -import com.oceanbase.spark.HBaseRelation.{columnFamilyMap, convertToBytes, parseCatalog} +import com.oceanbase.spark.HBaseRelation.{convertToBytes, parseCatalog} import com.oceanbase.spark.config.OBKVHbaseConfig import com.oceanbase.spark.obkv.HTableClientUtils import com.fasterxml.jackson.core.JsonParser.Feature -import org.apache.hadoop.hbase.client.{HTableInterface, Put} +import org.apache.hadoop.hbase.client.{Put, Table} import org.apache.hadoop.hbase.util.Bytes import org.apache.spark.internal.Logging import org.apache.spark.rdd.RDD @@ -61,7 +61,7 @@ case class HBaseRelation( override def insert(dataFrame: DataFrame, overwrite: Boolean): Unit = { dataFrame.foreachPartition( (rows: Iterator[Row]) => { - val hTableClient: HTableInterface = HTableClientUtils.getHTableClient(config) + val hTableClient: Table = HTableClientUtils.getHTableClient(config) val buffer = ArrayBuffer[Row]() rows.foreach( row => { @@ -75,7 +75,7 @@ case class HBaseRelation( }) } - private def flush(buffer: ArrayBuffer[Row], hTableClient: HTableInterface): Unit = { + private def flush(buffer: ArrayBuffer[Row], hTableClient: Table): Unit = { val putList = new util.ArrayList[Put]() buffer.foreach( row => { @@ -95,7 +95,7 @@ case class HBaseRelation( val cfName = HBaseRelation.columnFamilyMap(rowFieldName)._2 val familyName: Array[Byte] = Bytes.toBytes(cfName) val columnValue = convertToBytes(row.get(i)) - put.add(familyName, Bytes.toBytes(userFieldName), columnValue) + put.addColumn(familyName, Bytes.toBytes(userFieldName), columnValue) } } } diff --git a/spark-connector-oceanbase-e2e-tests/src/test/scala/com/oceanbase/spark/OBKVHBaseE2eITCase.scala b/spark-connector-oceanbase-e2e-tests/src/test/scala/com/oceanbase/spark/OBKVHBaseE2eITCase.scala index 052dc14..7f0df69 100644 --- a/spark-connector-oceanbase-e2e-tests/src/test/scala/com/oceanbase/spark/OBKVHBaseE2eITCase.scala +++ b/spark-connector-oceanbase-e2e-tests/src/test/scala/com/oceanbase/spark/OBKVHBaseE2eITCase.scala @@ -47,10 +47,8 @@ class OBKVHBaseE2eITCase extends SparkContainerTestEnvironment { @Test @DisabledIfSystemProperty( named = "spark_version", - matches = "^2\\.4\\.[0-9]$|^3\\.[2-9]$", - disabledReason = "The spark 2.x docker image fails to execute the spark-sql command. \n" + - "The obkv-hbase library does not support java11 and later versions: Unexpected version format: 11.0.22. \n" + - "The Spark 3.2 and later versions of docker images do not support jdk8." + matches = "^2\\.4\\.[0-9]$", + disabledReason = "The spark 2.x docker image fails to execute the spark-sql command." ) def testInsertValues(): Unit = { val sqlLines: util.List[String] = new util.ArrayList[String] diff --git a/spark-connector-oceanbase/pom.xml b/spark-connector-oceanbase/pom.xml index 6c315a3..25c8c07 100644 --- a/spark-connector-oceanbase/pom.xml +++ b/spark-connector-oceanbase/pom.xml @@ -40,7 +40,7 @@ under the License. com.oceanbase obkv-table-client - 1.2.14-SNAPSHOT + 1.3.0 diff --git a/spark-connector-oceanbase/spark-connector-oceanbase-2.4/pom.xml b/spark-connector-oceanbase/spark-connector-oceanbase-2.4/pom.xml index ca49386..8c4766e 100644 --- a/spark-connector-oceanbase/spark-connector-oceanbase-2.4/pom.xml +++ b/spark-connector-oceanbase/spark-connector-oceanbase-2.4/pom.xml @@ -94,23 +94,8 @@ under the License. com.oceanbase:obkv-table-client com.alipay.sofa:bolt com.alipay.sofa.common:sofa-common-tools - com.lmax:disruptor - - - com.oceanbase:obkv-table-client - - **/log4j/log-conf.xml - - - - *:* - - **/Log4j2Plugins.dat - - - diff --git a/spark-connector-oceanbase/spark-connector-oceanbase-2.4/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml b/spark-connector-oceanbase/spark-connector-oceanbase-2.4/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml deleted file mode 100644 index 989e8c7..0000000 --- a/spark-connector-oceanbase/spark-connector-oceanbase-2.4/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spark-connector-oceanbase/spark-connector-oceanbase-3.1/pom.xml b/spark-connector-oceanbase/spark-connector-oceanbase-3.1/pom.xml index 4ce1bfc..f915fa8 100644 --- a/spark-connector-oceanbase/spark-connector-oceanbase-3.1/pom.xml +++ b/spark-connector-oceanbase/spark-connector-oceanbase-3.1/pom.xml @@ -73,23 +73,8 @@ under the License. com.oceanbase:obkv-table-client com.alipay.sofa:bolt com.alipay.sofa.common:sofa-common-tools - com.lmax:disruptor - - - com.oceanbase:obkv-table-client - - **/log4j/log-conf.xml - - - - *:* - - **/Log4j2Plugins.dat - - - diff --git a/spark-connector-oceanbase/spark-connector-oceanbase-3.1/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml b/spark-connector-oceanbase/spark-connector-oceanbase-3.1/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml deleted file mode 100644 index 989e8c7..0000000 --- a/spark-connector-oceanbase/spark-connector-oceanbase-3.1/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spark-connector-oceanbase/spark-connector-oceanbase-3.2/pom.xml b/spark-connector-oceanbase/spark-connector-oceanbase-3.2/pom.xml index 78c8d67..8854c25 100644 --- a/spark-connector-oceanbase/spark-connector-oceanbase-3.2/pom.xml +++ b/spark-connector-oceanbase/spark-connector-oceanbase-3.2/pom.xml @@ -73,23 +73,8 @@ under the License. com.oceanbase:obkv-table-client com.alipay.sofa:bolt com.alipay.sofa.common:sofa-common-tools - com.lmax:disruptor - - - com.oceanbase:obkv-table-client - - **/log4j/log-conf.xml - - - - *:* - - **/Log4j2Plugins.dat - - - diff --git a/spark-connector-oceanbase/spark-connector-oceanbase-3.2/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml b/spark-connector-oceanbase/spark-connector-oceanbase-3.2/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml deleted file mode 100644 index 989e8c7..0000000 --- a/spark-connector-oceanbase/spark-connector-oceanbase-3.2/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spark-connector-oceanbase/spark-connector-oceanbase-3.3/pom.xml b/spark-connector-oceanbase/spark-connector-oceanbase-3.3/pom.xml index f27858c..7a84417 100644 --- a/spark-connector-oceanbase/spark-connector-oceanbase-3.3/pom.xml +++ b/spark-connector-oceanbase/spark-connector-oceanbase-3.3/pom.xml @@ -73,16 +73,10 @@ under the License. com.oceanbase:obkv-table-client com.alipay.sofa:bolt com.alipay.sofa.common:sofa-common-tools + com.lmax:disruptor - - - - org.apache.logging - shade.org.apache.logging - - diff --git a/spark-connector-oceanbase/spark-connector-oceanbase-3.4/pom.xml b/spark-connector-oceanbase/spark-connector-oceanbase-3.4/pom.xml index d96b781..b7e694a 100644 --- a/spark-connector-oceanbase/spark-connector-oceanbase-3.4/pom.xml +++ b/spark-connector-oceanbase/spark-connector-oceanbase-3.4/pom.xml @@ -73,12 +73,14 @@ under the License. com.oceanbase:obkv-table-client com.alipay.sofa:bolt com.alipay.sofa.common:sofa-common-tools + com.lmax:disruptor - + org.apache.logging shade.org.apache.logging