branch-3.0: [fix](fold) fixed an issue with be computing constants #4…

…3410 (#45691) Cherry-picked from #43410 Co-authored-by: lw112 <[email protected]>
apache · Dec 20, 2024 · c4bee9a · c4bee9a
1 parent d38f410
commit c4bee9a
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 8 deletions.
diff --git a/...e/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/...e/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
@@ -487,8 +487,16 @@ public static List<Literal> getResultExpression(DataType type, PValues resultCon
         } else if (type.isStringLikeType()) {
             int num = resultContent.getStringValueCount();
             for (int i = 0; i < num; ++i) {
-                Literal literal = new StringLiteral(resultContent.getStringValue(i));
-                res.add(literal);
+                // get the raw byte data to avoid character encoding conversion problems
+                ByteString bytesValues = resultContent.getBytesValue(i);
+                // use UTF-8 encoding to ensure proper handling of binary data
+                String stringValue = bytesValues.toStringUtf8();
+                // handle special NULL value cases
+                if ("\\N".equalsIgnoreCase(stringValue) && resultContent.hasHasNull()) {
+                    res.add(new NullLiteral(type));
+                } else {
+                    res.add(new StringLiteral(stringValue));
+                }
             }
         } else if (type.isArrayType()) {
             ArrayType arrayType = (ArrayType) type;

diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
@@ -233,7 +233,9 @@ public void testGetResultExpressionStruct() {
         PValues.Builder resultContentBuilder = PValues.newBuilder();
         for (int i = 0; i < elementsArray.length; i = i + 2) {
             childBuilder1.addInt32Value(elementsArray[i]);
-            childBuilder2.addStringValue("str" + (i + 1));
+            String strValue = "str" + (i + 1);
+            childBuilder2.addStringValue(strValue);
+            childBuilder2.addBytesValue(com.google.protobuf.ByteString.copyFromUtf8(strValue));
         }
         childBuilder1.setType(childTypeBuilder1.build());
         childBuilder2.setType(childTypeBuilder2.build());
@@ -280,7 +282,9 @@ public void testGetResultExpressionStructArray() {
         PValues.Builder resultContentBuilder = PValues.newBuilder();
         for (int i = 0; i < elementsArray.length; i = i + 2) {
             childBuilder1.addInt32Value(elementsArray[i]);
-            childBuilder2.addStringValue("str" + (i + 1));
+            String strValue = "str" + (i + 1);
+            childBuilder2.addStringValue(strValue);
+            childBuilder2.addBytesValue(com.google.protobuf.ByteString.copyFromUtf8(strValue));
         }
         childBuilder1.setType(childTypeBuilder1.build());
         childBuilder2.setType(childTypeBuilder2.build());

diff --git a/regression-test/data/nereids_p0/expression/fold_constant/fold_constant_by_be.out b/regression-test/data/nereids_p0/expression/fold_constant/fold_constant_by_be.out
@@ -1,4 +1,10 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+C2BD89103557CCBF7ED97B51860225A0
+
+-- !sql --
+C2BD89103557CCBF7ED97B51860225A0
+
 -- !sql_1 --
 80000
 

diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy
@@ -22,6 +22,9 @@ suite("fold_constant_by_be") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set enable_fold_constant_by_be=true'
 
+    qt_sql """ select hex(from_base64('wr2JEDVXzL9+2XtRhgIloA==')); """
+    qt_sql """ select hex(s) from (select from_base64('wr2JEDVXzL9+2XtRhgIloA==') as s) t; """
+
     test {
         sql '''
             select if(
@@ -32,8 +35,8 @@ suite("fold_constant_by_be") {
         result([['9999-07-31']])
     }
 
-    sql """ 
-        CREATE TABLE IF NOT EXISTS str_tb (k1 VARCHAR(10) NULL, v1 STRING NULL) 
+    sql """
+        CREATE TABLE IF NOT EXISTS str_tb (k1 VARCHAR(10) NULL, v1 STRING NULL)
         UNIQUE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 5 properties("replication_num" = "1");
     """
 
@@ -53,7 +56,7 @@ suite("fold_constant_by_be") {
 
     sql 'set query_timeout=12;'
     qt_sql "select sleep(sign(1)*5);"
-    
+
     explain {
         sql("verbose select substring('123456', 1, 3)")
         contains "varchar(3)"
@@ -71,7 +74,7 @@ suite("fold_constant_by_be") {
                     col_varchar_1000__undef_signed varchar(1000)  null  ,
                     col_varchar_1000__undef_signed_not_null varchar(1000)  not null  ,
                     col_varchar_1001__undef_signed varchar(1001)  null  ,
-                    col_varchar_1001__undef_signed_not_null varchar(1001)  not null  
+                    col_varchar_1001__undef_signed_not_null varchar(1001)  not null
                     ) engine=olap
                     DUPLICATE KEY(pk, col_char_255__undef_signed, col_char_100__undef_signed)
                     distributed by hash(pk) buckets 10