[update]

apache · Nov 26, 2024 · 14b1371 · 14b1371
1 parent 817728a
commit 14b1371
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 13 deletions.
diff --git a/...aimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShimLoader.scala b/...aimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShimLoader.scala
@@ -25,19 +25,18 @@ import scala.collection.JavaConverters._
 /** Load a [[SparkShim]]'s implementation. */
 object SparkShimLoader {
 
-  private var sparkShim: SparkShim = _
+  private lazy val sparkShim: SparkShim = loadSparkShim()
 
   def getSparkShim: SparkShim = {
-    if (sparkShim == null) {
-      sparkShim = loadSparkShim()
-    }
     sparkShim
   }
 
   private def loadSparkShim(): SparkShim = {
     val shims = ServiceLoader.load(classOf[SparkShim]).asScala
-    if (shims.size != 1) {
+    if (shims.isEmpty) {
       throw new IllegalStateException("No available spark shim here.")
+    } else if (shims.size > 1) {
+      throw new IllegalStateException("Found more than one spark shim here.")
     }
     shims.head
   }

diff --git a/...ions/PaimonSparkSqlExtensionsParser.scala → ...ons/PaimonSpark3SqlExtensionsParser.scala b/...ions/PaimonSparkSqlExtensionsParser.scala → ...ons/PaimonSpark3SqlExtensionsParser.scala
@@ -16,9 +16,10 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.catalyst.parser.extensions
+package org.apache.paimon.spark.catalyst.parser.extensions
 
 import org.apache.spark.sql.catalyst.parser.ParserInterface
+import org.apache.spark.sql.catalyst.parser.extensions.AbstractPaimonSparkSqlExtensionsParser
 
-class PaimonSparkSqlExtensionsParser(override val delegate: ParserInterface)
+class PaimonSpark3SqlExtensionsParser(override val delegate: ParserInterface)
   extends AbstractPaimonSparkSqlExtensionsParser(delegate) {}
diff --git a/...rk/paimon-spark3-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark3Shim.scala b/...rk/paimon-spark3-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark3Shim.scala
@@ -18,13 +18,13 @@
 
 package org.apache.spark.sql.paimon.shims
 
+import org.apache.paimon.spark.catalyst.parser.extensions.PaimonSpark3SqlExtensionsParser
 import org.apache.paimon.spark.data.{Spark3ArrayData, Spark3InternalRow, SparkArrayData, SparkInternalRow}
 import org.apache.paimon.types.{DataType, RowType}
 
 import org.apache.spark.sql.{Column, SparkSession}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
 import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.parser.extensions.PaimonSparkSqlExtensionsParser
 import org.apache.spark.sql.catalyst.plans.logical.Aggregate
 import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog}
 import org.apache.spark.sql.connector.expressions.Transform
@@ -35,7 +35,7 @@ import java.util.{Map => JMap}
 class Spark3Shim extends SparkShim {
 
   override def createSparkParser(delegate: ParserInterface): ParserInterface = {
-    new PaimonSparkSqlExtensionsParser(delegate)
+    new PaimonSpark3SqlExtensionsParser(delegate)
   }
 
   override def createSparkInternalRow(rowType: RowType): SparkInternalRow = {

diff --git a/...ions/PaimonSparkSqlExtensionsParser.scala → ...ons/PaimonSpark4SqlExtensionsParser.scala b/...ions/PaimonSparkSqlExtensionsParser.scala → ...ons/PaimonSpark4SqlExtensionsParser.scala
@@ -16,11 +16,12 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.catalyst.parser.extensions
+package org.apache.paimon.spark.catalyst.parser.extensions
 
 import org.apache.spark.sql.catalyst.parser.{CompoundBody, ParserInterface}
+import org.apache.spark.sql.catalyst.parser.extensions.AbstractPaimonSparkSqlExtensionsParser
 
-class PaimonSparkSqlExtensionsParser(override val delegate: ParserInterface)
+class PaimonSpark4SqlExtensionsParser(override val delegate: ParserInterface)
   extends AbstractPaimonSparkSqlExtensionsParser(delegate) {
 
   def parseScript(sqlScriptText: String): CompoundBody = delegate.parseScript(sqlScriptText)

diff --git a/...rk/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala b/...rk/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala
@@ -18,13 +18,13 @@
 
 package org.apache.spark.sql.paimon.shims
 
+import org.apache.paimon.spark.catalyst.parser.extensions.PaimonSpark4SqlExtensionsParser
 import org.apache.paimon.spark.data.{Spark4ArrayData, Spark4InternalRow, SparkArrayData, SparkInternalRow}
 import org.apache.paimon.types.{DataType, RowType}
 
 import org.apache.spark.sql.{Column, SparkSession}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
 import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.parser.extensions.PaimonSparkSqlExtensionsParser
 import org.apache.spark.sql.catalyst.plans.logical.Aggregate
 import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, Table, TableCatalog}
 import org.apache.spark.sql.connector.expressions.Transform
@@ -36,7 +36,7 @@ import java.util.{Map => JMap}
 class Spark4Shim extends SparkShim {
 
   override def createSparkParser(delegate: ParserInterface): ParserInterface = {
-    new PaimonSparkSqlExtensionsParser(delegate)
+    new PaimonSpark4SqlExtensionsParser(delegate)
   }
   override def createSparkInternalRow(rowType: RowType): SparkInternalRow = {
     new Spark4InternalRow(rowType)