[update]

apache · Nov 24, 2024 · a32b7e0 · a32b7e0
1 parent ab78b31
commit a32b7e0
Show file tree

Hide file tree

Showing 13 changed files with 10 additions and 134 deletions.
diff --git a/.github/workflows/utitcase.yml b/.github/workflows/utitcase.yml
@@ -54,7 +54,7 @@ jobs:
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
           test_modules="!paimon-e2e-tests,"
-          for suffix in 3.5 3.4 3.3 3.2 common_2.12; do
+          for suffix in 3.5 3.4 3.3 3.2 ut; do
           test_modules+="!org.apache.paimon:paimon-spark-${suffix},"
           done
           test_modules="${test_modules%,}"

diff --git a/paimon-spark/paimon-spark-3.2/pom.xml b/paimon-spark/paimon-spark-3.2/pom.xml
@@ -67,13 +67,6 @@ under the License.
 
         <!-- test -->
 
-        <dependency>
-            <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-spark3-common</artifactId>
-            <version>${project.version}</version>
-            <scope>test</scope>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.paimon</groupId>
             <artifactId>paimon-spark-ut</artifactId>
@@ -139,7 +132,7 @@ under the License.
                             </filters>
                             <artifactSet>
                                 <includes combine.children="append">
-                                     <include>org.apache.paimon:paimon-spark-common_${scala.binary.version}</include>
+                                     <include>org.apache.paimon:paimon-spark3-common</include>
                                 </includes>
                             </artifactSet>
                         </configuration>

diff --git a/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/spark/paimon/Utils.scala b/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/spark/paimon/Utils.scala
diff --git a/paimon-spark/paimon-spark-3.3/pom.xml b/paimon-spark/paimon-spark-3.3/pom.xml
@@ -67,13 +67,6 @@ under the License.
 
         <!-- test -->
 
-        <dependency>
-            <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-spark3-common</artifactId>
-            <version>${project.version}</version>
-            <scope>test</scope>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.paimon</groupId>
             <artifactId>paimon-spark-ut</artifactId>
@@ -139,7 +132,7 @@ under the License.
                             </filters>
                             <artifactSet>
                                 <includes combine.children="append">
-                                     <include>org.apache.paimon:paimon-spark-common_${scala.binary.version}</include>
+                                     <include>org.apache.paimon:paimon-spark3-common</include>
                                 </includes>
                             </artifactSet>
                         </configuration>

diff --git a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/spark/paimon/Utils.scala b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/spark/paimon/Utils.scala
diff --git a/paimon-spark/paimon-spark-3.4/pom.xml b/paimon-spark/paimon-spark-3.4/pom.xml
@@ -67,13 +67,6 @@ under the License.
 
         <!-- test -->
 
-        <dependency>
-            <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-spark3-common</artifactId>
-            <version>${project.version}</version>
-            <scope>test</scope>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.paimon</groupId>
             <artifactId>paimon-spark-ut</artifactId>

diff --git a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/spark/paimon/Utils.scala b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/spark/paimon/Utils.scala
diff --git a/paimon-spark/paimon-spark-3.5/pom.xml b/paimon-spark/paimon-spark-3.5/pom.xml
@@ -67,13 +67,6 @@ under the License.
 
         <!-- test -->
 
-        <dependency>
-            <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-spark3-common</artifactId>
-            <version>${project.version}</version>
-            <scope>test</scope>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.paimon</groupId>
             <artifactId>paimon-spark-ut</artifactId>
@@ -139,7 +132,7 @@ under the License.
                             </filters>
                             <artifactSet>
                                 <includes combine.children="append">
-                                     <include>org.apache.paimon:paimon-spark-common_${scala.binary.version}</include>
+                                     <include>org.apache.paimon:paimon-spark3-common</include>
                                 </includes>
                             </artifactSet>
                         </configuration>

diff --git a/...imon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala b/...imon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala
@@ -63,9 +63,9 @@ abstract class PaimonOptimizationTestBase extends PaimonSparkTestBase {
       val relation = Utils.createDataset(spark, createRelationV2("T"))
       val mergedSubquery = relation
         .select(
-          Utils.toColumn(count(Literal(1)).as("cnt")),
-          Utils.toColumn(sum(Utils.toExpression(spark, col("a")).as("sum_a"))),
-          Utils.toColumn(avg(Utils.toExpression(spark, col("b")).as("avg_b")))
+          Utils.toColumn(count(Literal(1))).as("cnt"),
+          Utils.toColumn(sum(Utils.toExpression(spark, col("a")))).as("sum_a"),
+          Utils.toColumn(avg(Utils.toExpression(spark, col("b"))).as("avg_b"))
         )
         .select(
           Utils.toColumn(

diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/spark/sql/paimon/Utils.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/spark/sql/paimon/Utils.scala
@@ -31,7 +31,7 @@ import java.io.File
  */
 object Utils {
 
-  def createTempDir: File = SparkUtils.createTempDir()
+  def createTempDir: File = SparkUtils.createTempDir(System.getProperty("java.io.tmpdir"), "spark")
 
   def waitUntilEventEmpty(spark: SparkSession): Unit = {
     spark.sparkContext.listenerBus.waitUntilEmpty()

diff --git a/...s/org.apache.paimon.spark.shims.SparkShim → ...g.apache.spark.sql.paimon.shims.SparkShim b/...s/org.apache.paimon.spark.shims.SparkShim → ...g.apache.spark.sql.paimon.shims.SparkShim
@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-org.apache.paimon.spark.shims.Spark3Shim
+org.apache.spark.sql.paimon.shims.Spark3Shim
diff --git a/...s/org.apache.paimon.spark.shims.SparkShim → ...rg.apache.park.sql.paimon.shims.SparkShim b/...s/org.apache.paimon.spark.shims.SparkShim → ...rg.apache.park.sql.paimon.shims.SparkShim
diff --git a/pom.xml b/pom.xml
@@ -368,6 +368,7 @@ under the License.
                 <test.spark.version>3.3.0</test.spark.version>
             </properties>
             <activation>
+                <activeByDefault>true</activeByDefault>
                 <property>
                     <name>spark3</name>
                 </property>
@@ -391,7 +392,6 @@ under the License.
                 <test.spark.version>4.0.0-preview2</test.spark.version>
             </properties>
             <activation>
-                <activeByDefault>true</activeByDefault>
                 <property>
                     <name>spark4</name>
                 </property>