[spark] adjust paimon spark structure (#4573)

apache · Nov 26, 2024 · 408b78d · 408b78d
1 parent f822bec
commit 408b78d
Show file tree

Hide file tree

Showing 125 changed files with 1,138 additions and 627 deletions.
diff --git a/.github/workflows/utitcase-spark-3.x.yml b/.github/workflows/utitcase-spark-3.x.yml
@@ -54,7 +54,7 @@ jobs:
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
           test_modules=""
-          for suffix in common_2.12 3.5 3.4 3.3 3.2; do
+          for suffix in ut 3.5 3.4 3.3 3.2; do
           test_modules+="org.apache.paimon:paimon-spark-${suffix},"
           done
           test_modules="${test_modules%,}"

diff --git a/.github/workflows/utitcase-spark-4.x.yml b/.github/workflows/utitcase-spark-4.x.yml
@@ -54,7 +54,7 @@ jobs:
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
           test_modules=""
-          for suffix in common_2.13 4.0; do
+          for suffix in ut 4.0; do
           test_modules+="org.apache.paimon:paimon-spark-${suffix},"
           done
           test_modules="${test_modules%,}"

diff --git a/.github/workflows/utitcase.yml b/.github/workflows/utitcase.yml
@@ -54,7 +54,7 @@ jobs:
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
           test_modules="!paimon-e2e-tests,"
-          for suffix in 3.5 3.4 3.3 3.2 common_2.12; do
+          for suffix in 3.5 3.4 3.3 3.2 ut; do
           test_modules+="!org.apache.paimon:paimon-spark-${suffix},"
           done
           test_modules="${test_modules%,}"

diff --git a/paimon-spark/paimon-spark-3.2/pom.xml b/paimon-spark/paimon-spark-3.2/pom.xml
@@ -36,6 +36,12 @@ under the License.
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-spark3-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.paimon</groupId>
             <artifactId>paimon-spark-common_${scala.binary.version}</artifactId>
@@ -63,7 +69,7 @@ under the License.
 
         <dependency>
             <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-spark-common_${scala.binary.version}</artifactId>
+            <artifactId>paimon-spark-ut</artifactId>
             <version>${project.version}</version>
             <classifier>tests</classifier>
             <scope>test</scope>
@@ -126,7 +132,7 @@ under the License.
                             </filters>
                             <artifactSet>
                                 <includes combine.children="append">
-                                     <include>org.apache.paimon:paimon-spark-common_${scala.binary.version}</include>
+                                     <include>org.apache.paimon:paimon-spark3-common</include>
                                 </includes>
                             </artifactSet>
                         </configuration>

diff --git a/paimon-spark/paimon-spark-3.3/pom.xml b/paimon-spark/paimon-spark-3.3/pom.xml
@@ -36,6 +36,12 @@ under the License.
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-spark3-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.paimon</groupId>
             <artifactId>paimon-spark-common_${scala.binary.version}</artifactId>
@@ -63,7 +69,7 @@ under the License.
 
         <dependency>
             <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-spark-common_${scala.binary.version}</artifactId>
+            <artifactId>paimon-spark-ut</artifactId>
             <version>${project.version}</version>
             <classifier>tests</classifier>
             <scope>test</scope>
@@ -126,7 +132,7 @@ under the License.
                             </filters>
                             <artifactSet>
                                 <includes combine.children="append">
-                                     <include>org.apache.paimon:paimon-spark-common_${scala.binary.version}</include>
+                                     <include>org.apache.paimon:paimon-spark3-common</include>
                                 </includes>
                             </artifactSet>
                         </configuration>

diff --git a/...ark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTest.scala b/...ark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTest.scala
@@ -248,7 +248,7 @@ class InsertOverwriteTest extends PaimonSparkTestBase {
               spark.sql("SELECT * FROM T ORDER BY a, b"),
               Row(1, 3, "3") :: Row(2, 4, "4") :: Nil)
 
-            withSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") {
+            withSparkSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") {
               // dynamic overwrite the a=1 partition
               spark.sql("INSERT OVERWRITE T VALUES (1, 5, '5'), (1, 7, '7')")
               checkAnswer(
@@ -289,7 +289,7 @@ class InsertOverwriteTest extends PaimonSparkTestBase {
                 "ptv2",
                 22) :: Nil)
 
-            withSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") {
+            withSparkSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") {
               // dynamic overwrite the pt2=22 partition
               spark.sql(
                 "INSERT OVERWRITE T PARTITION (pt2 = 22) VALUES (3, 'c2', 'ptv1'), (4, 'd2', 'ptv3')")

diff --git a/paimon-spark/paimon-spark-3.4/pom.xml b/paimon-spark/paimon-spark-3.4/pom.xml
@@ -36,6 +36,12 @@ under the License.
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-spark3-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.paimon</groupId>
             <artifactId>paimon-spark-common_${scala.binary.version}</artifactId>
@@ -63,7 +69,7 @@ under the License.
 
         <dependency>
             <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-spark-common_${scala.binary.version}</artifactId>
+            <artifactId>paimon-spark-ut</artifactId>
             <version>${project.version}</version>
             <classifier>tests</classifier>
             <scope>test</scope>
@@ -126,7 +132,7 @@ under the License.
                             </filters>
                             <artifactSet>
                                 <includes combine.children="append">
-                                     <include>org.apache.paimon:paimon-spark-common_${scala.binary.version}</include>
+                                     <include>org.apache.paimon:paimon-spark3-common</include>
                                 </includes>
                             </artifactSet>
                         </configuration>

diff --git a/paimon-spark/paimon-spark-3.5/pom.xml b/paimon-spark/paimon-spark-3.5/pom.xml
@@ -36,6 +36,12 @@ under the License.
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-spark3-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.paimon</groupId>
             <artifactId>paimon-spark-common_${scala.binary.version}</artifactId>
@@ -63,7 +69,7 @@ under the License.
 
         <dependency>
             <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-spark-common_${scala.binary.version}</artifactId>
+            <artifactId>paimon-spark-ut</artifactId>
             <version>${project.version}</version>
             <classifier>tests</classifier>
             <scope>test</scope>
@@ -126,7 +132,7 @@ under the License.
                             </filters>
                             <artifactSet>
                                 <includes combine.children="append">
-                                     <include>org.apache.paimon:paimon-spark-common_${scala.binary.version}</include>
+                                     <include>org.apache.paimon:paimon-spark3-common</include>
                                 </includes>
                             </artifactSet>
                         </configuration>

diff --git a/paimon-spark/paimon-spark-4.0/pom.xml b/paimon-spark/paimon-spark-4.0/pom.xml
@@ -36,6 +36,12 @@ under the License.
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-spark4-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.paimon</groupId>
             <artifactId>paimon-spark-common_${scala.binary.version}</artifactId>
@@ -63,7 +69,7 @@ under the License.
 
         <dependency>
             <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-spark-common_${scala.binary.version}</artifactId>
+            <artifactId>paimon-spark-ut</artifactId>
             <version>${project.version}</version>
             <classifier>tests</classifier>
             <scope>test</scope>
@@ -126,7 +132,7 @@ under the License.
                             </filters>
                             <artifactSet>
                                 <includes combine.children="append">
-                                     <include>org.apache.paimon:paimon-spark-common_${scala.binary.version}</include>
+                                     <include>org.apache.paimon:paimon-spark4-common</include>
                                 </includes>
                             </artifactSet>
                         </configuration>

diff --git a/paimon-spark/paimon-spark-common/pom.xml b/paimon-spark/paimon-spark-common/pom.xml
@@ -38,18 +38,6 @@ under the License.
     </properties>
 
     <dependencies>
-        <dependency>
-            <groupId>org.apache.paimon</groupId>
-            <artifactId>${paimon-sparkx-common}</artifactId>
-            <version>${project.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>*</groupId>
-                    <artifactId>*</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_${scala.binary.version}</artifactId>
@@ -72,46 +60,6 @@ under the License.
             <groupId>org.apache.paimon</groupId>
             <artifactId>paimon-bundle</artifactId>
         </dependency>
-
-       <!-- test -->
-
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-sql_${scala.binary.version}</artifactId>
-            <version>${spark.version}</version>
-            <classifier>tests</classifier>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
-            <version>${spark.version}</version>
-            <classifier>tests</classifier>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-core_${scala.binary.version}</artifactId>
-            <version>${spark.version}</version>
-            <classifier>tests</classifier>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-hive_${scala.binary.version}</artifactId>
-            <version>${spark.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-avro_${scala.binary.version}</artifactId>
-            <version>${spark.version}</version>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 
     <build>
@@ -130,7 +78,6 @@ under the License.
                             <artifactSet>
                                 <includes combine.children="append">
                                     <include>org.apache.paimon:paimon-bundle</include>
-                                    <include>org.apache.paimon:${paimon-sparkx-common}</include>
                                 </includes>
                             </artifactSet>
                         </configuration>
@@ -155,21 +102,6 @@ under the License.
                     <sourceDirectory>src/main/antlr4</sourceDirectory>
                 </configuration>
             </plugin>
-
-            <!-- Used for submodules to utilize the test code -->
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-jar-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>prepare-test-jar</id>
-                        <phase>test-compile</phase>
-                        <goals>
-                            <goal>test-jar</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
         </plugins>
     </build>
 </project>