diff --git a/.github/workflows/utitcase.yml b/.github/workflows/utitcase.yml index bde67cb4c203..8aa33f5b8218 100644 --- a/.github/workflows/utitcase.yml +++ b/.github/workflows/utitcase.yml @@ -54,7 +54,7 @@ jobs: jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" test_modules="!paimon-e2e-tests," - for suffix in 3.5 3.4 3.3 3.2 common_2.12; do + for suffix in 3.5 3.4 3.3 3.2 ut; do test_modules+="!org.apache.paimon:paimon-spark-${suffix}," done test_modules="${test_modules%,}" diff --git a/paimon-spark/paimon-spark-3.2/pom.xml b/paimon-spark/paimon-spark-3.2/pom.xml index 0d259cd45928..957319b47dab 100644 --- a/paimon-spark/paimon-spark-3.2/pom.xml +++ b/paimon-spark/paimon-spark-3.2/pom.xml @@ -67,13 +67,6 @@ under the License. - - org.apache.paimon - paimon-spark3-common - ${project.version} - test - - org.apache.paimon paimon-spark-ut @@ -139,7 +132,7 @@ under the License. - org.apache.paimon:paimon-spark-common_${scala.binary.version} + org.apache.paimon:paimon-spark3-common diff --git a/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/spark/paimon/Utils.scala b/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/spark/paimon/Utils.scala deleted file mode 100644 index 1a899f500153..000000000000 --- a/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/spark/paimon/Utils.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.paimon - -import org.apache.spark.util.{Utils => SparkUtils} - -import java.io.File - -/** - * A wrapper that some Objects or Classes is limited to access beyond [[org.apache.spark]] package. - */ -object Utils { - - def createTempDir: File = SparkUtils.createTempDir() - -} diff --git a/paimon-spark/paimon-spark-3.3/pom.xml b/paimon-spark/paimon-spark-3.3/pom.xml index 51a95b89e626..0a390d926789 100644 --- a/paimon-spark/paimon-spark-3.3/pom.xml +++ b/paimon-spark/paimon-spark-3.3/pom.xml @@ -67,13 +67,6 @@ under the License. - - org.apache.paimon - paimon-spark3-common - ${project.version} - test - - org.apache.paimon paimon-spark-ut @@ -139,7 +132,7 @@ under the License. - org.apache.paimon:paimon-spark-common_${scala.binary.version} + org.apache.paimon:paimon-spark3-common diff --git a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/spark/paimon/Utils.scala b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/spark/paimon/Utils.scala deleted file mode 100644 index 1a899f500153..000000000000 --- a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/spark/paimon/Utils.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.paimon - -import org.apache.spark.util.{Utils => SparkUtils} - -import java.io.File - -/** - * A wrapper that some Objects or Classes is limited to access beyond [[org.apache.spark]] package. - */ -object Utils { - - def createTempDir: File = SparkUtils.createTempDir() - -} diff --git a/paimon-spark/paimon-spark-3.4/pom.xml b/paimon-spark/paimon-spark-3.4/pom.xml index 24a2c3b65473..0f4cb30e4f7f 100644 --- a/paimon-spark/paimon-spark-3.4/pom.xml +++ b/paimon-spark/paimon-spark-3.4/pom.xml @@ -67,13 +67,6 @@ under the License. - - org.apache.paimon - paimon-spark3-common - ${project.version} - test - - org.apache.paimon paimon-spark-ut diff --git a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/spark/paimon/Utils.scala b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/spark/paimon/Utils.scala deleted file mode 100644 index 1a899f500153..000000000000 --- a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/spark/paimon/Utils.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.paimon - -import org.apache.spark.util.{Utils => SparkUtils} - -import java.io.File - -/** - * A wrapper that some Objects or Classes is limited to access beyond [[org.apache.spark]] package. - */ -object Utils { - - def createTempDir: File = SparkUtils.createTempDir() - -} diff --git a/paimon-spark/paimon-spark-3.5/pom.xml b/paimon-spark/paimon-spark-3.5/pom.xml index 5e005741d407..1b9c96888908 100644 --- a/paimon-spark/paimon-spark-3.5/pom.xml +++ b/paimon-spark/paimon-spark-3.5/pom.xml @@ -67,13 +67,6 @@ under the License. - - org.apache.paimon - paimon-spark3-common - ${project.version} - test - - org.apache.paimon paimon-spark-ut @@ -139,7 +132,7 @@ under the License. - org.apache.paimon:paimon-spark-common_${scala.binary.version} + org.apache.paimon:paimon-spark3-common diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala index f6f1072a3e2b..b1db59c60913 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala @@ -63,9 +63,9 @@ abstract class PaimonOptimizationTestBase extends PaimonSparkTestBase { val relation = Utils.createDataset(spark, createRelationV2("T")) val mergedSubquery = relation .select( - Utils.toColumn(count(Literal(1)).as("cnt")), - Utils.toColumn(sum(Utils.toExpression(spark, col("a")).as("sum_a"))), - Utils.toColumn(avg(Utils.toExpression(spark, col("b")).as("avg_b"))) + Utils.toColumn(count(Literal(1))).as("cnt"), + Utils.toColumn(sum(Utils.toExpression(spark, col("a")))).as("sum_a"), + Utils.toColumn(avg(Utils.toExpression(spark, col("b"))).as("avg_b")) ) .select( Utils.toColumn( diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/spark/sql/paimon/Utils.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/spark/sql/paimon/Utils.scala index e72a79aef913..9badca9f052f 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/spark/sql/paimon/Utils.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/spark/sql/paimon/Utils.scala @@ -31,7 +31,7 @@ import java.io.File */ object Utils { - def createTempDir: File = SparkUtils.createTempDir() + def createTempDir: File = SparkUtils.createTempDir(System.getProperty("java.io.tmpdir"), "spark") def waitUntilEventEmpty(spark: SparkSession): Unit = { spark.sparkContext.listenerBus.waitUntilEmpty() diff --git a/paimon-spark/paimon-spark3-common/src/main/resources/META-INF/services/org.apache.paimon.spark.shims.SparkShim b/paimon-spark/paimon-spark3-common/src/main/resources/META-INF/services/org.apache.spark.sql.paimon.shims.SparkShim similarity index 94% rename from paimon-spark/paimon-spark3-common/src/main/resources/META-INF/services/org.apache.paimon.spark.shims.SparkShim rename to paimon-spark/paimon-spark3-common/src/main/resources/META-INF/services/org.apache.spark.sql.paimon.shims.SparkShim index e30a67cdc165..b79ef54f6e30 100644 --- a/paimon-spark/paimon-spark3-common/src/main/resources/META-INF/services/org.apache.paimon.spark.shims.SparkShim +++ b/paimon-spark/paimon-spark3-common/src/main/resources/META-INF/services/org.apache.spark.sql.paimon.shims.SparkShim @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.paimon.spark.shims.Spark3Shim \ No newline at end of file +org.apache.spark.sql.paimon.shims.Spark3Shim \ No newline at end of file diff --git a/paimon-spark/paimon-spark4-common/src/main/resources/META-INF/services/org.apache.paimon.spark.shims.SparkShim b/paimon-spark/paimon-spark4-common/src/main/resources/META-INF/services/org.apache.park.sql.paimon.shims.SparkShim similarity index 100% rename from paimon-spark/paimon-spark4-common/src/main/resources/META-INF/services/org.apache.paimon.spark.shims.SparkShim rename to paimon-spark/paimon-spark4-common/src/main/resources/META-INF/services/org.apache.park.sql.paimon.shims.SparkShim diff --git a/pom.xml b/pom.xml index 0eaff80188a4..24f292331950 100644 --- a/pom.xml +++ b/pom.xml @@ -368,6 +368,7 @@ under the License. 3.3.0 + true spark3 @@ -391,7 +392,6 @@ under the License. 4.0.0-preview2 - true spark4