[format] Fix orc and parquet writer about timestamp not contains [loc…

…al timezone] and [is_adjust_to_utc] (apache#2739)
tongcheng-elong · Jan 20, 2024 · c6e182a · c6e182a
1 parent e26e6b5
commit c6e182a
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 6 deletions.
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcSplitReaderUtil.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcSplitReaderUtil.java
@@ -74,8 +74,9 @@ public static TypeDescription toOrcType(DataType type) {
             case DATE:
                 return TypeDescription.createDate();
             case TIMESTAMP_WITHOUT_TIME_ZONE:
-            case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
                 return TypeDescription.createTimestamp();
+            case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+                return TypeDescription.createTimestampInstant();
             case ARRAY:
                 ArrayType arrayType = (ArrayType) type;
                 return TypeDescription.createList(toOrcType(arrayType.getElementType()));

diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java
@@ -115,11 +115,11 @@ private static Type convertToParquetType(
             case TIMESTAMP_WITHOUT_TIME_ZONE:
                 TimestampType timestampType = (TimestampType) type;
                 return createTimestampWithLogicalType(
-                        name, timestampType.getPrecision(), repetition);
+                        name, timestampType.getPrecision(), repetition, false);
             case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
                 LocalZonedTimestampType localZonedTimestampType = (LocalZonedTimestampType) type;
                 return createTimestampWithLogicalType(
-                        name, localZonedTimestampType.getPrecision(), repetition);
+                        name, localZonedTimestampType.getPrecision(), repetition, true);
             case ARRAY:
                 ArrayType arrayType = (ArrayType) type;
                 return ConversionPatterns.listOfElements(
@@ -151,13 +151,21 @@ private static Type convertToParquetType(
     }
 
     private static Type createTimestampWithLogicalType(
-            String name, int precision, Type.Repetition repetition) {
+            String name, int precision, Type.Repetition repetition, boolean isAdjustToUTC) {
         if (precision <= 3) {
-            return Types.primitive(INT64, repetition).as(OriginalType.TIMESTAMP_MILLIS).named(name);
+            return Types.primitive(INT64, repetition)
+                    .as(
+                            LogicalTypeAnnotation.timestampType(
+                                    isAdjustToUTC, LogicalTypeAnnotation.TimeUnit.MILLIS))
+                    .named(name);
         } else if (precision > 6) {
             return Types.primitive(PrimitiveType.PrimitiveTypeName.INT96, repetition).named(name);
         } else {
-            return Types.primitive(INT64, repetition).as(OriginalType.TIMESTAMP_MICROS).named(name);
+            return Types.primitive(INT64, repetition)
+                    .as(
+                            LogicalTypeAnnotation.timestampType(
+                                    isAdjustToUTC, LogicalTypeAnnotation.TimeUnit.MICROS))
+                    .named(name);
         }
     }