From 2ed638108c8aa57d9fe5f19cfaa1e34d48946227 Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Wed, 3 Apr 2024 22:42:32 +0800 Subject: [PATCH] add is_nereids field --- be/src/runtime/fold_constant_executor.cpp | 5 ++-- .../rules/FoldConstantRuleOnBE.java | 28 ++++++++++++------- .../trees/expressions/literal/MapLiteral.java | 5 +++- .../expressions/literal/StructLiteral.java | 5 +++- .../doris/rewrite/FoldConstantsRule.java | 1 + gensrc/thrift/PaloInternalService.thrift | 1 + 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/be/src/runtime/fold_constant_executor.cpp b/be/src/runtime/fold_constant_executor.cpp index 1e86a9aee87f129..5640684a4156583 100644 --- a/be/src/runtime/fold_constant_executor.cpp +++ b/be/src/runtime/fold_constant_executor.cpp @@ -112,7 +112,7 @@ Status FoldConstantExecutor::fold_constant_vexpr(const TFoldConstantParams& para const auto& column_ptr = tmp_block.get_by_position(result_column).column; const auto& column_type = tmp_block.get_by_position(result_column).type; // 4 from fe: Config.be_exec_version maybe need remove after next version, now in 2.1 - if (_runtime_state->be_exec_version() >= 4) { + if (_runtime_state->be_exec_version() >= 4 && params.is_nereids) { auto* p_type_desc = expr_result.mutable_type_desc(); auto* p_values = expr_result.mutable_result_content(); res_type.to_protobuf(p_type_desc); @@ -120,7 +120,8 @@ Status FoldConstantExecutor::fold_constant_vexpr(const TFoldConstantParams& para RETURN_IF_ERROR(datatype_serde->write_column_to_pb( *column_ptr->convert_to_full_column_if_const(), *p_values, 0, 1)); expr_result.set_success(true); - expr_result.set_content(""); + // after refactor, this field is useless, but it's required + expr_result.set_content("ERROR"); expr_result.mutable_type()->set_type(t_type); pexpr_result_map.mutable_map()->insert({n.first, expr_result}); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 5144d2fda25ecda..45f455aa61d0384 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sleep; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; @@ -113,8 +114,8 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory { public List> buildRules() { return ImmutableList.of( root(Expression.class) - .whenCtx(FoldConstantRuleOnBE::isEnableFoldByBe) - .thenApply(FoldConstantRuleOnBE::foldByBE) + .whenCtx(FoldConstantRuleOnBE::isEnableFoldByBe) + .thenApply(FoldConstantRuleOnBE::foldByBE) ); } @@ -187,10 +188,10 @@ private static void collectConst(Expression expr, Map constM // eg: avg_state(1) return is agg function serialize data // and some type can't find a literal to represent if (expr.getDataType().isAggStateType() || expr.getDataType().isObjectType() - || expr.getDataType().isVariantType() || expr.getDataType().isTimeV2Type()) { + || expr.getDataType().isVariantType() || expr.getDataType().isTimeLikeType()) { return; } - if (skipSleepFunction(expr)) { + if (skipSleepFunction(expr) || (expr instanceof TableGeneratingFunction)) { return; } String id = idGenerator.getNextId().toString(); @@ -256,6 +257,7 @@ private static Map evalOnBE(Map> tParams.setVecExec(true); tParams.setQueryOptions(tQueryOptions); tParams.setQueryId(context.queryId()); + tParams.setIsNereids(true); // TODO: will be delete the debug log after find problem of timeout. LOG.info("fold query {} ", DebugUtil.printId(context.queryId())); @@ -436,19 +438,25 @@ public static List getResultExpression(DataType type, PValues resultCon resultContent.getChildElement(i)); List valueLiteral = getResultExpression(mapType.getValueType(), resultContent.getChildElement(i + 1)); - MapLiteral mapLiteral = new MapLiteral(keyLiteral, valueLiteral); + MapLiteral mapLiteral = new MapLiteral(keyLiteral, valueLiteral, mapType); res.add(mapLiteral); } } else if (type.isStructType()) { StructType structType = (StructType) type; int childCount = resultContent.getChildElementCount(); - List fields = new ArrayList<>(childCount); + List> allFields = new ArrayList<>(); for (int i = 0; i < childCount; ++i) { - fields.add(getResultExpression(structType.getFields().get(i).getDataType(), - resultContent.getChildElement(i)).get(0)); + allFields.add(getResultExpression(structType.getFields().get(i).getDataType(), + resultContent.getChildElement(i))); + } + for (int i = 0; i < allFields.get(0).size(); ++i) { + List fields = new ArrayList<>(); + for (int child = 0; child < childCount; ++child) { + fields.add(allFields.get(child).get(i)); + } + StructLiteral structLiteral = new StructLiteral(fields, structType); + res.add(structLiteral); } - StructLiteral structLiteral = new StructLiteral(fields); - res.add(structLiteral); } else { LOG.warn("the type: {} is not support, should implement it", type.toString()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/MapLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/MapLiteral.java index 7dab827509bed41..c57bd3a04875e15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/MapLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/MapLiteral.java @@ -48,7 +48,10 @@ public MapLiteral(List keys, List values) { this(keys, values, computeDataType(keys, values)); } - private MapLiteral(List keys, List values, DataType dataType) { + /** + * create MAP Literal with keys, values and datatype + */ + public MapLiteral(List keys, List values, DataType dataType) { super(dataType); this.keys = ImmutableList.copyOf(Objects.requireNonNull(keys, "keys should not be null")); this.values = ImmutableList.copyOf(Objects.requireNonNull(values, "values should not be null")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java index 2fd2186f4965eb0..3a46f1f5b83e7ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java @@ -49,7 +49,10 @@ public StructLiteral(List fields) { this(fields, computeDataType(fields)); } - private StructLiteral(List fields, DataType dataType) { + /** + * create Struct Literal with fields and datatype + */ + public StructLiteral(List fields, DataType dataType) { super(dataType); this.fields = ImmutableList.copyOf(Objects.requireNonNull(fields, "fields should not be null")); Preconditions.checkArgument(dataType instanceof StructType, diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java index 3ef20a5651d2585..dd37c2fc7b178a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java @@ -384,6 +384,7 @@ private Map> calcConstExpr(Map future = BackendServiceProxy.getInstance().foldConstantExpr(brpcAddress, tParams); diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 97bd7f5552e6b6b..b71ddfa21a36ebe 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -548,6 +548,7 @@ struct TFoldConstantParams { 3: optional bool vec_exec 4: optional TQueryOptions query_options 5: optional Types.TUniqueId query_id + 6: optional bool is_nereids } // TransmitData