From df867a1531e313e4046ea572e18ba26f60c9587f Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyk10@gmail.com>
Date: Wed, 15 Nov 2023 10:23:21 +0800
Subject: [PATCH] [fix](catalog) Fix ClickHouse DataTime64 precision parsing
 (#26977)

---
 .../clickhouse/init/03-create-table.sql       |   9 ++++++++
 .../clickhouse/init/04-insert.sql             |   2 +-
 .../org/apache/doris/jdbc/JdbcExecutor.java   |   3 ++-
 .../jdbc/client/JdbcClickHouseClient.java     |  20 ++++++------------
 .../jdbc/test_clickhouse_jdbc_catalog.out     | Bin 4900 -> 4955 bytes
 .../jdbc/test_clickhouse_jdbc_catalog.groovy  |   2 ++
 6 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql
index b09c5f1543a350..34f03796e76381 100644
--- a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql
+++ b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql
@@ -279,4 +279,13 @@ CREATE TABLE doris_test.ts
     ts UInt64
 )
 ENGINE = MergeTree
+ORDER BY id;
+
+CREATE TABLE doris_test.dt_with_tz
+(
+    id Int64,
+    dt1 DateTime('Asia/Shanghai'),
+    dt2 DateTime64(6, 'Asia/Shanghai')
+)
+ENGINE = MergeTree
 ORDER BY id;
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql b/docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql
index f8af30e80153f6..08b5d034aa160f 100644
--- a/docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql
+++ b/docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql
@@ -54,4 +54,4 @@ INSERT INTO doris_test.final_test Values (1, 'second');
 
 INSERT INTO doris_test.ts values (1,1694438743);
 
-
+INSERT INTO doris_test.dt_with_tz values(1, '2022-01-02 00:00:00','2022-01-02 00:00:00.000000');
diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
index 7aba1fb62b3bc4..05b327840fc269 100644
--- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
+++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
@@ -560,7 +560,8 @@ private ColumnValueConverter getOutputConverter(
                 }
                 if (OffsetDateTime.class.equals(clz)) {
                     return createConverter(
-                            input -> ((Timestamp) input).toLocalDateTime(), LocalDateTime.class);
+                            input -> ((OffsetDateTime) input).toLocalDateTime(),
+                            LocalDateTime.class);
                 }
                 if (oracle.sql.TIMESTAMP.class.equals(clz)) {
                     return createConverter(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
index f25fabcc3c8e98..a6088ad11caa60 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
@@ -73,20 +73,14 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
             if (ckType.startsWith("DateTime(") || ckType.equals("DateTime")) {
                 return ScalarType.createDatetimeV2Type(0);
             } else {
-                int indexStart = ckType.indexOf('(');
-                int indexEnd = ckType.indexOf(')');
-                if (indexStart != -1 && indexEnd != -1) {
-                    String scaleStr = ckType.substring(indexStart + 1, indexEnd);
-                    int scale = Integer.parseInt(scaleStr);
-                    if (scale > JDBC_DATETIME_SCALE) {
-                        scale = JDBC_DATETIME_SCALE;
-                    }
-                    // return with the actual scale
-                    return ScalarType.createDatetimeV2Type(scale);
-                } else {
-                    // default precision if not specified
-                    return ScalarType.createDatetimeV2Type(JDBC_DATETIME_SCALE);
+                // DateTime64 with millisecond precision
+                // Datetime64(6) / DateTime64(6, 'Asia/Shanghai')
+                String[] accuracy = ckType.substring(11, ckType.length() - 1).split(", ");
+                int precision = Integer.parseInt(accuracy[0]);
+                if (precision > 6) {
+                    precision = JDBC_DATETIME_SCALE;
                 }
+                return ScalarType.createDatetimeV2Type(precision);
             }
         }
 
diff --git a/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.out
index a92a0f5d71af346e655f7604173cbda3e0ea9823..30401d56d4c24750162dae177dbcb67be389f5f6 100644
GIT binary patch
delta 34
pcmZ3Yc3W+Oim<S*u7YApNql)`Nk)80m4dD=m*HeaW+f&rE&#k}2_*mk

delta 7
OcmcbuwnS}%iZB2T9|E%g

diff --git a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
index 987cbe940273bb..294ee63e58364e 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
@@ -91,6 +91,8 @@ suite("test_clickhouse_jdbc_catalog", "p0,external,clickhouse,external_docker,ex
             contains """QUERY: SELECT "id", "ts" FROM "doris_test"."ts" WHERE ("ts" <= toUnixTimestamp(FROM_UNIXTIME(ts, '%Y-%m-%d')))"""
         }
 
+        order_qt_dt_with_tz """ select * from dt_with_tz order by id; """
+
         sql """ drop catalog if exists ${catalog_name} """
     }
 }