Merge pull request #16 from tongcheng-elong/tc-paimon-0.9_compatible_…

…timestamp Tc paimon 0.9 compatible spark3.4 timestamp
tongcheng-elong · Oct 21, 2024 · c9a572f · c9a572f
2 parents 9886884 + b6ff32f
commit c9a572f
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 1 deletion.
diff --git a/...park/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkConnectorOptions.java b/...park/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkConnectorOptions.java
@@ -18,6 +18,7 @@
 
 package org.apache.paimon.spark;
 
+import org.apache.paimon.annotation.Documentation;
 import org.apache.paimon.options.ConfigOption;
 
 import static org.apache.paimon.options.ConfigOptions.key;
@@ -77,4 +78,16 @@ public class SparkConnectorOptions {
                     .defaultValue(false)
                     .withDescription(
                             "Whether to read row in the form of changelog (add rowkind column in row to represent its change type).");
+
+    @Documentation.ExcludeFromDocumentation(
+            "In Spark >= 3.4, does Paimon's TimestampType type conversion use the old version method by default")
+    public static final ConfigOption<Boolean> PAIMON_INFER_TIMESTAMP_NTZ_ENABLED =
+            key("spark.sql.paimon.inferTimestampNTZ.enabled")
+                    .booleanType()
+                    .noDefaultValue()
+                    .withDescription(
+                            "When false, the default is to follow the historical version method. "
+                                    + "LocalZonedTimestamp and TimestampType are converted to "
+                                    + "Spark's TimestampType. Otherwise, the new version method is used."
+                                    + "The default value is true");
 }
diff --git a/...park/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/shim/TypeUtils.scala b/...park/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/shim/TypeUtils.scala
@@ -18,7 +18,16 @@
 
 package org.apache.paimon.spark.util.shim
 
+import org.apache.paimon.spark.SparkConnectorOptions
+
+import org.apache.spark.SparkEnv
+
 object TypeUtils {
 
-  def treatPaimonTimestampTypeAsSparkTimestampType(): Boolean = false
+  def treatPaimonTimestampTypeAsSparkTimestampType(): Boolean = {
+    !SparkEnv.get.conf
+      .getOption(SparkConnectorOptions.PAIMON_INFER_TIMESTAMP_NTZ_ENABLED.key())
+      .getOrElse("true")
+      .toBoolean
+  }
 }