diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java index d1e3129244f0f88..4cb6a63165ba8b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java @@ -342,4 +342,9 @@ public TSortInfo toThrift() { } return sortInfo; } + + @Override + public String toString() { + return orderingExprs.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 3264d6627ead5d4..585d9d248302177 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -62,7 +62,9 @@ public static ConnectContext createMTMVContext(MTMV mtmv) { ctx.getSessionVariable().allowModifyMaterializedViewData = true; // Disable add default limit rule to avoid refresh data wrong ctx.getSessionVariable().setDisableNereidsRules( - String.join(",", ImmutableSet.of(RuleType.ADD_DEFAULT_LIMIT.name()))); + String.join(",", ImmutableSet.of( + "COMPRESSED_MATERIALIZE_AGG", "COMPRESSED_MATERIALIZE_SORT", + RuleType.ADD_DEFAULT_LIMIT.name()))); Optional workloadGroup = mtmv.getWorkloadGroup(); if (workloadGroup.isPresent()) { ctx.getSessionVariable().setWorkloadGroup(workloadGroup.get()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 5b276258263f37e..fe1dfaf092e40f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -55,6 +55,7 @@ import org.apache.doris.nereids.rules.rewrite.CountDistinctRewrite; import org.apache.doris.nereids.rules.rewrite.CountLiteralRewrite; import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow; +import org.apache.doris.nereids.rules.rewrite.DecoupleEncodeDecode; import org.apache.doris.nereids.rules.rewrite.DeferMaterializeTopNResult; import org.apache.doris.nereids.rules.rewrite.EliminateAggCaseWhen; import org.apache.doris.nereids.rules.rewrite.EliminateAggregate; @@ -115,6 +116,7 @@ import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoinOneSide; import org.apache.doris.nereids.rules.rewrite.PushDownAggWithDistinctThroughJoinOneSide; import org.apache.doris.nereids.rules.rewrite.PushDownDistinctThroughJoin; +import org.apache.doris.nereids.rules.rewrite.PushDownEncodeSlot; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughProject; import org.apache.doris.nereids.rules.rewrite.PushDownLimit; import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin; @@ -253,6 +255,13 @@ public class Rewriter extends AbstractBatchJobExecutor { new CountLiteralRewrite(), new NormalizeSort() ), + + topDown(// must behind NormalizeAggregate/NormalizeSort + new MergeProjects(), + new PushDownEncodeSlot(), + new DecoupleEncodeDecode() + ), + topic("Window analysis", topDown( new ExtractAndNormalizeWindowExpression(), @@ -372,9 +381,11 @@ public class Rewriter extends AbstractBatchJobExecutor { // generate one PhysicalLimit if current distribution is gather or two // PhysicalLimits with gather exchange topDown(new LimitSortToTopN()), - topDown(new SimplifyEncodeDecode()), - topDown(new LimitAggToTopNAgg()), topDown(new MergeTopNs()), + topDown(new SimplifyEncodeDecode(), + new MergeProjects() + ), + topDown(new LimitAggToTopNAgg()), topDown(new SplitLimit()), topDown( new PushDownLimit(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java index 2cb472ce975f314..3ac5a38c9143672 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java @@ -44,11 +44,6 @@ public class LogicalProperties { protected final Supplier dataTraitSupplier; private Integer hashCode = null; - public LogicalProperties(Supplier> outputSupplier, - Supplier dataTraitSupplier) { - this(outputSupplier, dataTraitSupplier, ImmutableList::of); - } - /** * constructor of LogicalProperties. * @@ -56,8 +51,7 @@ public LogicalProperties(Supplier> outputSupplier, * throw exception for which children have UnboundRelation */ public LogicalProperties(Supplier> outputSupplier, - Supplier dataTraitSupplier, - Supplier> nonUserVisibleOutputSupplier) { + Supplier dataTraitSupplier) { this.outputSupplier = Suppliers.memoize( Objects.requireNonNull(outputSupplier, "outputSupplier can not be null") ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 86d0495b851bd22..4beb31b44315c8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -114,6 +114,9 @@ public enum RuleType { // rewrite rules COMPRESSED_MATERIALIZE_AGG(RuleTypeClass.REWRITE), COMPRESSED_MATERIALIZE_SORT(RuleTypeClass.REWRITE), + COMPRESSED_MATERIALIZE_REPEAT(RuleTypeClass.REWRITE), + PUSH_DOWN_ENCODE_SLOT(RuleTypeClass.REWRITE), + DECOUPLE_DECODE_ENCODE_SLOT(RuleTypeClass.REWRITE), SIMPLIFY_ENCODE_DECODE(RuleTypeClass.REWRITE), NORMALIZE_AGGREGATE(RuleTypeClass.REWRITE), NORMALIZE_SORT(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CompressedMaterialize.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CompressedMaterialize.java index 814269a63984a02..aa7ea284a3b5e07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CompressedMaterialize.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CompressedMaterialize.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeAsSmallInt; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.coercion.CharacterType; @@ -72,6 +73,11 @@ public List buildRules() { logicalSort().when(a -> ConnectContext.get() != null && ConnectContext.get().getSessionVariable().enableCompressMaterialize) .then(this::compressMaterializeSort) + // ), + // RuleType.COMPRESSED_MATERIALIZE_REPEAT.build( + // logicalRepeat().when(r -> ConnectContext.get() != null + // && ConnectContext.get().getSessionVariable().enableCompressMaterialize) + // .then(this::compressMaterializeRepeat) ) ); } @@ -101,6 +107,9 @@ private LogicalSort compressMaterializeSort(LogicalSort sort) { } private Optional getEncodeExpression(Expression expression) { + if (expression.isConstant()) { + return Optional.empty(); + } DataType type = expression.getDataType(); Expression encodeExpr = null; if (type instanceof CharacterType) { @@ -169,4 +178,52 @@ private LogicalAggregate compressedMaterializeAggregate(LogicalAggregate

getEncodeGroupingSets(LogicalRepeat repeat) { + Map encode = Maps.newHashMap(); + // the first grouping set contains all group by keys + for (Expression gb : repeat.getGroupingSets().get(0)) { + Optional encodeExpr = getEncodeExpression(gb); + encodeExpr.ifPresent(expression -> encode.put(gb, expression)); + } + return encode; + } + + private LogicalRepeat compressMaterializeRepeat(LogicalRepeat repeat) { + Map encode = getEncodeGroupingSets(repeat); + if (encode.isEmpty()) { + return repeat; + } + List> newGroupingSets = Lists.newArrayList(); + for (int i = 0; i < repeat.getGroupingSets().size(); i++) { + List grouping = Lists.newArrayList(); + for (int j = 0; j < repeat.getGroupingSets().get(i).size(); j++) { + Expression groupingExpr = repeat.getGroupingSets().get(i).get(j); + grouping.add(encode.getOrDefault(groupingExpr, groupingExpr)); + } + newGroupingSets.add(grouping); + } + List newOutputs = Lists.newArrayList(); + Map decodeMap = new HashMap<>(); + for (Expression gp : encode.keySet()) { + decodeMap.put(gp, new DecodeAsVarchar(encode.get(gp))); + } + for (NamedExpression out : repeat.getOutputExpressions()) { + Expression replaced = ExpressionUtils.replace(out, decodeMap); + if (out != replaced) { + if (out instanceof SlotReference) { + newOutputs.add(new Alias(out.getExprId(), replaced, out.getName())); + } else if (out instanceof Alias) { + newOutputs.add(((Alias) out).withChildren(replaced.children())); + } else { + // should not reach here + Preconditions.checkArgument(false, "output abnormal: " + repeat); + } + } else { + newOutputs.add(out); + } + } + repeat = repeat.withGroupSetsAndOutput(newGroupingSets, newOutputs); + return repeat; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecoupleEncodeDecode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecoupleEncodeDecode.java new file mode 100644 index 000000000000000..10ba8805b434bf0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecoupleEncodeDecode.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DecodeAsVarchar; +import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeString; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; + +import com.google.common.collect.Lists; + +import java.util.List; + +/** + * in project: + * decode_as_varchar(encode_as_xxx(v)) => v + */ +public class DecoupleEncodeDecode extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalProject().then(this::rewrite) + .toRule(RuleType.DECOUPLE_DECODE_ENCODE_SLOT); + } + + private LogicalProject rewrite(LogicalProject project) { + List newProjections = Lists.newArrayList(); + boolean hasNewProjections = false; + for (NamedExpression e : project.getProjects()) { + boolean changed = false; + if (e instanceof Alias) { + Alias alias = (Alias) e; + Expression body = alias.child(); + if (body instanceof DecodeAsVarchar && body.child(0) instanceof EncodeString) { + Expression encodeBody = body.child(0).child(0); + newProjections.add((NamedExpression) alias.withChildren(encodeBody)); + changed = true; + } else if (body instanceof EncodeString && body.child(0) instanceof DecodeAsVarchar) { + Expression decodeBody = body.child(0).child(0); + newProjections.add((NamedExpression) alias.withChildren(decodeBody)); + changed = true; + } + } + if (!changed) { + newProjections.add(e); + hasNewProjections = true; + } + } + if (hasNewProjections) { + project = project.withProjects(newProjections); + } + return project; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java index f0f9c4347252d5c..eea0a22ef31f89e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; @@ -35,6 +34,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.HashMap; import java.util.List; @@ -50,6 +51,8 @@ * 2. push limit to local agg */ public class LimitAggToTopNAgg implements RewriteRuleFactory { + public static final Logger LOG = LogManager.getLogger(LimitAggToTopNAgg.class); + @Override public List buildRules() { return ImmutableList.of( @@ -122,6 +125,8 @@ public List buildRules() { LogicalTopN originTopn = topn; LogicalProject project = topn.child(); LogicalAggregate agg = (LogicalAggregate) project.child(); + StringBuilder builder = new StringBuilder(); + builder.append("@@@@@###"); if (!project.isAllSlots()) { /* topn(orderKey=[a]) @@ -139,6 +144,9 @@ public List buildRules() { keyAsKey.put((SlotReference) e.toSlot(), (SlotReference) e.child(0)); } } + builder.append(topn); + builder.append(project); + List projectOrderKeys = Lists.newArrayList(); boolean hasNew = false; for (OrderKey orderKey : topn.getOrderKeys()) { @@ -157,36 +165,24 @@ public List buildRules() { supplementOrderKeyByGroupKeyIfCompatible(topn, agg); Plan result; if (pair == null) { + builder.append("|not compatible"); result = originTopn; } else { + builder.append("|compatible"); agg = agg.withGroupBy(pair.second); topn = (LogicalTopN) topn.withOrderKeys(pair.first); - if (isOrderKeysInProject(topn, project)) { - project = (LogicalProject) project.withChildren(agg); - topn = (LogicalTopN>>) - topn.withChildren(project); - result = topn; - } else { - topn = (LogicalTopN) topn.withChildren(agg); - project = (LogicalProject) project.withChildren(topn); - result = project; - } + topn = (LogicalTopN) topn.withChildren(agg); + project = (LogicalProject) project.withChildren(topn); + result = project; } + LOG.warn(builder.toString()); + LOG.warn("@@@@@###originTopn " + originTopn.treeString()); + LOG.warn("@@@@@###result " + result.treeString()); return result; }).toRule(RuleType.LIMIT_AGG_TO_TOPN_AGG) ); } - private boolean isOrderKeysInProject(LogicalTopN topn, LogicalProject project) { - Set projectSlots = project.getOutputSet(); - for (OrderKey orderKey : topn.getOrderKeys()) { - if (!projectSlots.contains(orderKey.getExpr())) { - return false; - } - } - return true; - } - private List generateOrderKeyByGroupKey(LogicalAggregate agg) { return agg.getGroupByExpressions().stream() .map(key -> new OrderKey(key, true, false)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpJoinFromUnionAll.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpJoinFromUnionAll.java index b3df9b92c56a84c..8c21b12a6d8fa18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpJoinFromUnionAll.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpJoinFromUnionAll.java @@ -586,10 +586,17 @@ boolean comparePlan(Plan plan1, Plan plan2) { isEqual = false; } for (int i = 0; isEqual && i < plan2.getOutput().size(); i++) { - NamedExpression expr = ((LogicalProject) plan1).getProjects().get(i); - NamedExpression replacedExpr = (NamedExpression) - expr.rewriteUp(e -> plan1ToPlan2.getOrDefault(e, e)); - if (!replacedExpr.equals(((LogicalProject) plan2).getProjects().get(i))) { + Expression expr1 = ((LogicalProject) plan1).getProjects().get(i); + Expression expr2 = ((LogicalProject) plan2).getProjects().get(i); + if (expr1 instanceof Alias) { + if (!(expr2 instanceof Alias)) { + return false; + } + expr1 = expr1.child(0); + expr2 = expr2.child(0); + } + Expression replacedExpr = expr1.rewriteUp(e -> plan1ToPlan2.getOrDefault(e, e)); + if (!replacedExpr.equals(expr2)) { isEqual = false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownEncodeSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownEncodeSlot.java new file mode 100644 index 000000000000000..bff40ae6266ea51 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownEncodeSlot.java @@ -0,0 +1,657 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DecodeAsVarchar; +import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeString; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.types.coercion.CharacterType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * push down encode_as_int(slot) down + * example: + * group by x + * -->project(encode_as_int(A) as x) + * -->Any(A) + * -->project(A) + * --> scan + * => + * group by x + * -->project(x) + * -->Any(x) + * --> project(encode_as_int(A) as x) + * -->scan + * Note: + * do not push down encode if encode.child() is not slot, + * example + * group by encode_as_int(A + B) + * --> any(A, B) + */ +public class PushDownEncodeSlot extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalProject() + .whenNot(project -> project.child() instanceof LogicalRepeat) + .whenNot(project -> (project.child() instanceof LogicalLeaf)) + .then(this::pushDownEncodeSlot) + .toRule(RuleType.PUSH_DOWN_ENCODE_SLOT); + } + + private List collectEncodeAliases(LogicalProject project) { + List encodeAliases = new ArrayList<>(); + Set computingSlots = new HashSet<>(); + for (NamedExpression e : project.getProjects()) { + if (e instanceof Alias) { + Expression aliasBody = e.child(0); + if (!(aliasBody instanceof SlotReference) && !(aliasBody instanceof EncodeString)) { + computingSlots.addAll(e.getInputSlots()); + } + } + } + for (NamedExpression e : project.getProjects()) { + if (e instanceof Alias && e.child(0) instanceof EncodeString + && e.child(0).child(0) instanceof SlotReference + && !computingSlots.contains(e.child(0).child(0))) { + encodeAliases.add((Alias) e); + } + } + return encodeAliases; + } + + /** + * case 1 + * project(encode(A) as B) + * --> any(A) + * => + * project(B) + * -->any(A): push "encode(A) as B" + * + * case 2 + * project(A, encode(A) as B) + * -->any(A) + * => + * project(decode(B) as A, B) + * -->any(A): push "encode(A) as B" + * + * case 3 + * project(A as C, encode(A) as B) + * -->any(A) + * => + * project(decode(B) as C, B) + * -->any(A): push "encode(A) as B" + */ + private LogicalProject rewriteRootProject(LogicalProject project, + List pushedEncodeAlias) { + if (pushedEncodeAlias.isEmpty()) { + return project; + } + Map encodeBodyToEncodeAlias = new HashMap<>(); + for (Alias alias : pushedEncodeAlias) { + Expression encodeBody = alias.child().child(0); + encodeBodyToEncodeAlias.put(encodeBody, alias); + } + List projections = Lists.newArrayListWithCapacity(project.getProjects().size()); + for (NamedExpression e : project.getProjects()) { + if (pushedEncodeAlias.contains(e)) { + // case 1 + projections.add(e.toSlot()); + } else if (encodeBodyToEncodeAlias.containsKey(e)) { + // case 2 + ExprId id = e.getExprId(); + DecodeAsVarchar decode = new DecodeAsVarchar(encodeBodyToEncodeAlias.get(e).toSlot()); + Alias alias = new Alias(id, decode, decode.toSql()); + projections.add(alias); + } else if (e instanceof Alias && encodeBodyToEncodeAlias.containsKey(e.child(0))) { + // case 3 + Alias alias = (Alias) e; + DecodeAsVarchar decode = new DecodeAsVarchar(encodeBodyToEncodeAlias.get(e.child(0)).toSlot()); + Alias newAlias = (Alias) alias.withChildren(decode); + projections.add(newAlias); + } else { + projections.add(e); + } + } + return project.withProjects(projections); + + } + + private LogicalProject pushDownEncodeSlot(LogicalProject project) { + List encodeAliases = collectEncodeAliases(project); + if (encodeAliases.isEmpty()) { + return project; + } + + PushDownContext ctx = new PushDownContext(project, encodeAliases); + ctx.prepare(); + if (ctx.notPushed.size() == encodeAliases.size()) { + return project; + } + Plan child = project.child(); + PushDownContext childContext = new PushDownContext(child, ctx.toBePushedToChild.get(child)); + Plan newChild = child.accept(EncodeSlotPushDownVisitor.INSTANCE, childContext); + List pushed = ctx.toBePused; + if (child != newChild) { + if (newChild instanceof LogicalProject) { + pushed.removeAll(childContext.notPushed); + newChild = ((LogicalProject) newChild).child(); + } + project = (LogicalProject) project.withChildren(newChild); + project = rewriteRootProject(project, pushed); + } + return project; + } + + /** + * push down encode slot context + */ + public static class PushDownContext { + public Plan plan; + + public List encodeAliases; + // encode_as_int(slot1) as slot2 + // replaceMap: + // slot1 -> slot2 + Map replaceMap = new HashMap<>(); + // child plan -> aliases in encodeAliases which can be pushed down to child plan + Map> toBePushedToChild = new HashMap<>(); + List toBePused = new ArrayList<>(); + // the aliases that cannot be pushed down to any child plan + // for example: + // encode(A+B) as x, where plan is a join, and A, B comes from join's left and right child respectively + List notPushed = new ArrayList<>(); + + public PushDownContext(Plan plan, List encodeAliases) { + this.plan = plan; + this.encodeAliases = encodeAliases; + } + + // init replaceMap/toBePushed/notPushed + private void prepare() { + List> childrenPassThroughSlots = + plan.children().stream().map(n -> getPassThroughSlots(n)).collect(Collectors.toList()); + for (int i = 0; i < plan.children().size(); i++) { + Plan child = plan.children().get(i); + if (child instanceof LogicalJoin) { + LogicalJoin join = (LogicalJoin) child; + BiMap compareSlots = EncodeSlotPushDownVisitor + .getEncodeCandidateSlotsAndShouldNotPushSlotsFromJoinCondition(join).first; + childrenPassThroughSlots.get(i).addAll(compareSlots.keySet()); + } + } + for (Alias alias : encodeAliases) { + EncodeString encode = (EncodeString) alias.child(); + Expression strExpr = encode.child(); + boolean pushed = false; + Preconditions.checkArgument(strExpr instanceof SlotReference, + "expect encode_as_xxx(slot), but " + alias); + + for (int i = 0; i < childrenPassThroughSlots.size(); i++) { + if (childrenPassThroughSlots.get(i).contains(strExpr)) { + toBePushedToChild.putIfAbsent(plan.child(i), new ArrayList<>()); + toBePushedToChild.get(plan.child(i)).add(alias); + toBePused.add(alias); + replaceMap.put(alias.child().child(0), (SlotReference) alias.toSlot()); + pushed = true; + break; + } + } + if (!pushed) { + notPushed.add(alias); + } + } + } + + /** + * expandEncodeAliasForJoin + */ + public void expandEncodeAliasForJoin(BiMap equalSlots) { + List expanded = new ArrayList<>(); + for (Alias alias : encodeAliases) { + if (alias.child().child(0) instanceof SlotReference) { + SlotReference slot = (SlotReference) alias.child().child(0); + SlotReference otherHand = equalSlots.get(slot); + if (otherHand != null) { + EncodeString encodeOtherHand = (EncodeString) alias.child().withChildren(otherHand); + Alias encodeOtherHandAlias = new Alias(encodeOtherHand, encodeOtherHand.toSql()); + if (!encodeAliases.contains(encodeOtherHandAlias)) { + expanded.add(encodeOtherHandAlias); + } + } + } + } + encodeAliases.addAll(expanded); + } + + // the child of alias is a slot reference. for example: slotA as B + // + private boolean isSlotAlias(Expression expr) { + return expr instanceof Alias && expr.child(0) instanceof SlotReference; + } + + private Set getPassThroughSlots(Plan plan) { + Set outputSlots = Sets.newHashSet(plan.getOutputSet()); + Set keySlots = Sets.newHashSet(); + for (Expression e : plan.getExpressions()) { + if (!(e instanceof SlotReference) && !isSlotAlias(e)) { + keySlots.addAll(e.getInputSlots()); + } + } + outputSlots.removeAll(keySlots); + return outputSlots; + } + } + + /** + * push down encode slot + */ + public static class EncodeSlotPushDownVisitor extends PlanVisitor { + public static EncodeSlotPushDownVisitor INSTANCE = new EncodeSlotPushDownVisitor(); + + /** + * visitChildren + */ + public Plan visitChildren(Plan plan, PushDownContext ctx) { + ImmutableList.Builder newChildren = ImmutableList.builderWithExpectedSize(plan.arity()); + boolean hasNewChildren = false; + for (Plan child : plan.children()) { + Plan newChild; + if (ctx.toBePushedToChild.containsKey(child)) { + newChild = child.accept(this, new PushDownContext(child, ctx.toBePushedToChild.get(child))); + if (!hasNewChildren && newChild != child) { + hasNewChildren = true; + } + } else { + newChild = child; + } + newChildren.add(newChild); + } + if (hasNewChildren) { + plan = plan.withChildren(newChildren.build()); + } + return plan; + } + + private Plan projectNotPushedAlias(Plan plan, List notPushedAlias) { + if (!notPushedAlias.isEmpty()) { + // project encode expressions if they are not pushed down + // project(encode) + // +--> plan + List projections = + notPushedAlias.stream().map(e -> (NamedExpression) e).collect(Collectors.toList()); + projections.addAll(plan.getOutput()); + plan = new LogicalProject<>(projections, plan); + } + return plan; + } + + @Override + public Plan visit(Plan plan, PushDownContext ctx) { + ctx.prepare(); + plan = visitChildren(plan, ctx); + plan = projectNotPushedAlias(plan, ctx.notPushed); + return plan; + } + + @Override + public Plan visitLogicalRepeat(LogicalRepeat repeat, PushDownContext ctx) { + Plan plan = projectNotPushedAlias(repeat, ctx.encodeAliases); + return plan; + } + + private Optional findEncodeAliasByEncodeSlot(SlotReference slot, List aliases) { + for (Alias alias : aliases) { + if (alias.child().child(0).equals(slot)) { + return Optional.of(alias); + } + } + return Optional.empty(); + } + + @Override + public LogicalProject visitLogicalProject( + LogicalProject project, PushDownContext ctx) { + /* + * case 1 + * push down "encode(v1) as v2 + * project(v1, ...) + * +--->any(v1) + * => + * project(v2, ...) + * +--->any(v1) + * and push down "encode(v1) as v2" to any(v1) + * + * case 2 + * push down "encode(v1) as v2 + * project(k as v1, ...) + * +--->any(k) + * => + * project(v2, ...) + * +--->any(k) + * and push down "encode(k) as v2" to any(v1) + * + * case 3 + * push down "encode(v44) as v307" + * project(decode(v305) as v44) + * +-->agg(v305, groupBy[v305]) + * +--->project(encode(v44) as v305) + * => + * project(v305 as v307) + * +-->agg + * + * case 4 + * push down "encode(v1) as v2 + * project(a + b as v1, ...) + * +--->any(a, b) + * => + * project(encode(a+b) as v2, ...) + * +-->any(a, b) + + * + */ + List projections = Lists.newArrayListWithCapacity(project.getProjects().size()); + List toBePushed = Lists.newArrayList(); + List notPushed = Lists.newArrayList(ctx.encodeAliases); + + for (NamedExpression e : project.getProjects()) { + boolean changed = false; + + if (e instanceof SlotReference) { + Optional encodeAliasOpt = findEncodeAliasByEncodeSlot((SlotReference) e, ctx.encodeAliases); + if (encodeAliasOpt.isPresent()) { + // case 1 + projections.add(encodeAliasOpt.get().toSlot()); + toBePushed.add(encodeAliasOpt.get()); + notPushed.remove(encodeAliasOpt.get()); + changed = true; + } + } else { + // e is Alias + Expression aliasExpr = e.child(0); + if (aliasExpr instanceof SlotReference) { + //case 2 + Optional encodeAliasOpt = findEncodeAliasByEncodeSlot((SlotReference) e.toSlot(), + ctx.encodeAliases); + if (encodeAliasOpt.isPresent()) { + projections.add(encodeAliasOpt.get().toSlot()); + Alias encodeAlias = encodeAliasOpt.get(); + EncodeString encode = (EncodeString) encodeAlias.child(); + SlotReference baseSlot = (SlotReference) aliasExpr; + Alias encodeAliasForChild = (Alias) encodeAlias.withChildren(encode.withChildren(baseSlot)); + toBePushed.add(encodeAliasForChild); + notPushed.remove(encodeAlias); + changed = true; + } + } else { + Optional encodeAliasOpt = findEncodeAliasByEncodeSlot((SlotReference) e.toSlot(), + ctx.encodeAliases); + if (encodeAliasOpt.isPresent()) { + Alias encodeAlias = encodeAliasOpt.get(); + if (aliasExpr instanceof DecodeAsVarchar) { + // case 3 + // push down "encode(v44) as v307" + // project(decode(v305) as v44) + // +-->agg(v305, groupBy[v305]) + // +--->project(encode(v44) as v305) + Expression decodeBody = aliasExpr.child(0); + Alias aliasForProject = (Alias) encodeAlias.withChildren(decodeBody); + projections.add(aliasForProject); + notPushed.remove(encodeAlias); + changed = true; + } else { + // case 4 + EncodeString encode = (EncodeString) encodeAlias.child(); + Alias encodeAliasForProject = (Alias) encodeAlias + .withChildren(encode.withChildren(aliasExpr)); + projections.add(encodeAliasForProject); + notPushed.remove(encodeAlias); + changed = true; + } + } + } + } + if (!changed) { + projections.add(e); + } + } + projections.addAll(notPushed); + + project = project.withProjects(projections); + if (!toBePushed.isEmpty()) { + PushDownContext childContext = new PushDownContext(project.child(), toBePushed); + Plan newChild = project.child().accept(this, childContext); + if (project.child() != newChild) { + project = (LogicalProject) project.withChildren(newChild); + } + } + return project; + } + + private static boolean canBothSidesEncode(ComparisonPredicate compare) { + return compare.left().getDataType() instanceof CharacterType + && ((CharacterType) compare.left().getDataType()).getLen() < 15 + && ((CharacterType) compare.right().getDataType()).getLen() < 15 + && compare.left() instanceof SlotReference && compare.right() instanceof SlotReference; + } + + /** + * getEncodeCandidateSlotsFromJoinCondition + * + */ + public static Pair, Set> + getEncodeCandidateSlotsAndShouldNotPushSlotsFromJoinCondition(LogicalJoin join) { + // T1 join T2 on v1=v2 => v1/v2 can be encoded + // T1 join T2 on v1=v2 and fun(v1) => v1/v2 can not be encoded + BiMap compareSlots = HashBiMap.create(); + List conditions = new ArrayList<>(); + conditions.addAll(join.getHashJoinConjuncts()); + conditions.addAll(join.getOtherJoinConjuncts()); + Set shouldNotPushSlots = Sets.newHashSet(); + for (Expression e : conditions) { + boolean canPush = false; + if (e instanceof ComparisonPredicate) { + ComparisonPredicate compare = (ComparisonPredicate) e; + if (canBothSidesEncode(compare)) { + compareSlots.put((SlotReference) compare.left(), (SlotReference) compare.right()); + canPush = true; + } + } + if (!canPush) { + shouldNotPushSlots.addAll(e.getInputSlots()); + } + } + for (Slot notPushSlot : shouldNotPushSlots) { + if (compareSlots.isEmpty()) { + break; + } + compareSlots.remove((SlotReference) notPushSlot); + compareSlots.inverse().remove((SlotReference) notPushSlot); + } + return Pair.of(compareSlots, shouldNotPushSlots); + } + + @Override + public Plan visitLogicalJoin(LogicalJoin join, PushDownContext ctx) { + List pushLeft = new ArrayList<>(); + List pushRight = new ArrayList<>(); + Pair, Set> pair = + getEncodeCandidateSlotsAndShouldNotPushSlotsFromJoinCondition(join); + BiMap encodeCandidateSlots = pair.first; + Set shouldNotPushSlots = pair.second; + Set leftOutputSlots = join.left().getOutputSet(); + Map replaceMap = new HashMap<>(); + List notPushed = new ArrayList<>(); + for (Alias encodeAlias : ctx.encodeAliases) { + SlotReference encodeSlot = (SlotReference) encodeAlias.child().child(0); + if (encodeCandidateSlots.containsKey(encodeSlot)) { + SlotReference otherHand = encodeCandidateSlots.get(encodeSlot); + Alias otherHandAlias = new Alias(encodeAlias.child().withChildren(otherHand)); + if (leftOutputSlots.contains(encodeSlot)) { + pushLeft.add(encodeAlias); + pushRight.add(otherHandAlias); + } else { + pushRight.add(encodeAlias); + pushLeft.add(otherHandAlias); + } + replaceMap.put(encodeSlot, (SlotReference) encodeAlias.toSlot()); + replaceMap.put(otherHand, (SlotReference) otherHandAlias.toSlot()); + } else if (!shouldNotPushSlots.contains(encodeSlot)) { + if (leftOutputSlots.contains(encodeSlot)) { + pushLeft.add(encodeAlias); + } else { + pushRight.add(encodeAlias); + } + } else { + notPushed.add(encodeAlias); + } + replaceMap.put(encodeSlot, (SlotReference) encodeAlias.toSlot()); + } + List newConjuncts = Lists.newArrayListWithCapacity(join.getOtherJoinConjuncts().size()); + boolean changed = false; + for (Expression e : join.getOtherJoinConjuncts()) { + if (e instanceof ComparisonPredicate) { + ComparisonPredicate compare = (ComparisonPredicate) e; + if (canBothSidesEncode(compare)) { + SlotReference newLeft = replaceMap.get(compare.left()); + SlotReference newRight = replaceMap.get(compare.right()); + if (newLeft != null && newRight != null) { + compare = (ComparisonPredicate) compare.withChildren(newLeft, newRight); + changed = true; + } + Preconditions.checkArgument((newLeft == null) == (newRight == null), + "PushDownEncodeSlot replaceMap is not valid, " + compare); + } + newConjuncts.add(compare); + } else { + newConjuncts.add(e); + } + } + if (changed) { + join = join.withJoinConjuncts(join.getHashJoinConjuncts(), newConjuncts, join.getJoinReorderContext()); + } + Plan newLeft; + if (pushLeft.isEmpty()) { + newLeft = join.left(); + } else { + newLeft = join.left().accept(this, new PushDownContext(join.left(), pushLeft)); + } + Plan newRight; + if (pushRight.isEmpty()) { + newRight = join.right(); + } else { + newRight = join.right().accept(this, new PushDownContext(join.right(), pushRight)); + } + Plan result = join.withChildren(newLeft, newRight); + if (!notPushed.isEmpty()) { + List projections = new ArrayList<>(); + projections.addAll(notPushed); + projections.addAll(join.getOutput()); + result = new LogicalProject(projections, join); + } + return result; + } + + @Override + public Plan visitLogicalSetOperation(LogicalSetOperation op, PushDownContext ctx) { + // push down "encode(v) as x" through + // union(output[v], regular([v1],[v2])) + // -->child1(v1) + // -->child2(v2) + // rewrite union to: union(output[x], regular([x1], [x2])) + // and then push "encode(v1) as x1" to child(v1) + // push "encode(v2) as x2" to child(v2) + + List newOutput = Lists.newArrayListWithCapacity(op.getOutput().size()); + List> newRegularOutputs = Lists.newArrayListWithCapacity(op.getOutput().size()); + for (int cid = 0; cid < op.children().size(); cid++) { + newRegularOutputs.add(Lists.newArrayList(op.getRegularChildOutput(cid))); + } + + for (int oid = 0; oid < op.getOutput().size(); oid++) { + NamedExpression e = op.getOutput().get(oid); + boolean changed = false; + for (Alias alias : ctx.encodeAliases) { + if (alias.child().child(0).equals(e)) { + newOutput.add(alias.toSlot()); + changed = true; + EncodeString encode = (EncodeString) alias.child(); + ctx.toBePused.add(alias); + for (int cid = 0; cid < op.children().size(); cid++) { + Plan child = op.child(cid); + ctx.toBePushedToChild.putIfAbsent(child, new ArrayList<>()); + Alias aliasForChild = new Alias( + encode.withChildren(op.getRegularChildrenOutputs().get(cid).get(oid))); + ctx.toBePushedToChild.get(child).add(aliasForChild); + newRegularOutputs.get(cid).set(oid, (SlotReference) aliasForChild.toSlot()); + } + break; + } + } + if (!changed) { + newOutput.add(e); + } + } + op = op.withNewOutputs(newOutput); + + //rewrite children + List newChildren = Lists.newArrayListWithCapacity(op.children().size()); + for (Plan child : op.children()) { + if (!ctx.toBePushedToChild.get(child).isEmpty()) { + PushDownContext childCtx = new PushDownContext(child, ctx.toBePushedToChild.get(child)); + Plan newChild = child.accept(this, childCtx); + newChildren.add(newChild); + } else { + newChildren.add(child); + } + } + op = op.withChildrenAndTheirOutputs(newChildren, newRegularOutputs); + return op; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java index 38b687ba8387a05..3963c1b651ba285 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java @@ -23,7 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.scalar.DecodeAsVarchar; -import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeStrToInteger; +import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeString; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; @@ -149,7 +149,7 @@ private static Set eliminateDecodeAndEncode(Set expressi } private static Expression eliminateDecodeAndEncode(Expression expression) { - if (expression instanceof DecodeAsVarchar && expression.child(0) instanceof EncodeStrToInteger) { + if (expression instanceof DecodeAsVarchar && expression.child(0) instanceof EncodeString) { return expression.child(0).child(0); } boolean hasNewChild = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyEncodeDecode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyEncodeDecode.java index b6513fc75807079..fd15fae0d07a53d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyEncodeDecode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyEncodeDecode.java @@ -23,7 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.functions.scalar.DecodeAsVarchar; -import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeStrToInteger; +import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeString; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -48,7 +48,7 @@ public List buildRules() { boolean changed = false; for (NamedExpression namedExpression : project.getProjects()) { if (namedExpression instanceof Alias - && namedExpression.child(0) instanceof EncodeStrToInteger + && namedExpression.child(0) instanceof EncodeString && namedExpression.child(0).child(0) instanceof DecodeAsVarchar) { Alias alias = (Alias) namedExpression; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java index 7f26298c7006260..00e7f986ad5826a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java @@ -199,7 +199,7 @@ public String computeToSql() { @Override public String toString() { StringBuilder sb = new StringBuilder(); - sb.append(function).append(" WindowSpec("); + sb.append("WindowExpression(").append(function).append(" spec("); if (!partitionKeys.isEmpty()) { sb.append("PARTITION BY ").append(partitionKeys.stream() .map(Expression::toString) @@ -211,7 +211,7 @@ public String toString() { .collect(Collectors.joining(", ", "", " "))); } windowFrame.ifPresent(wf -> sb.append(wf.toSql())); - return sb.toString().trim() + ")"; + return sb.toString().trim() + "))"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsBigInt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsBigInt.java index 7d798ecf3e8cab9..d3f267efa36d337 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsBigInt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsBigInt.java @@ -33,8 +33,8 @@ /** * ScalarFunction 'EncodeAsBigInt'. */ -public class EncodeAsBigInt extends ScalarFunction - implements ExplicitlyCastableSignature, PropagateNullable, EncodeStrToInteger { +public class EncodeAsBigInt extends EncodeString + implements ExplicitlyCastableSignature, PropagateNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsInt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsInt.java index 5c6382d6ea144da..661f05fe3be82de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsInt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsInt.java @@ -33,8 +33,8 @@ /** * ScalarFunction 'EncodeAsInt'. */ -public class EncodeAsInt extends ScalarFunction - implements ExplicitlyCastableSignature, PropagateNullable, EncodeStrToInteger { +public class EncodeAsInt extends EncodeString + implements ExplicitlyCastableSignature, PropagateNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(IntegerType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsLargeInt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsLargeInt.java index bb30a9a8e8aef53..ee8d723d2b5143a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsLargeInt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsLargeInt.java @@ -33,8 +33,8 @@ /** * ScalarFunction 'EncodeAsLargeInt'. */ -public class EncodeAsLargeInt extends ScalarFunction - implements ExplicitlyCastableSignature, PropagateNullable, EncodeStrToInteger { +public class EncodeAsLargeInt extends EncodeString + implements ExplicitlyCastableSignature, PropagateNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(LargeIntType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsSmallInt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsSmallInt.java index 355a740197c33ec..d0e6a1fa23b1213 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsSmallInt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeAsSmallInt.java @@ -33,8 +33,8 @@ /** * ScalarFunction 'CompressAsSmallInt'. */ -public class EncodeAsSmallInt extends ScalarFunction - implements ExplicitlyCastableSignature, PropagateNullable, EncodeStrToInteger { +public class EncodeAsSmallInt extends EncodeString + implements ExplicitlyCastableSignature, PropagateNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(SmallIntType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeStrToInteger.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeString.java similarity index 71% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeStrToInteger.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeString.java index 87a9c43687d6a3f..778d76c34625087 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeStrToInteger.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/EncodeString.java @@ -17,8 +17,17 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; + /** * Encode_as_XXXInt */ -public interface EncodeStrToInteger { +public abstract class EncodeString extends ScalarFunction implements UnaryExpression { + /** + * constructor with 1 argument. + */ + public EncodeString(String name, Expression arg0) { + super(name, arg0); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java index 11baf2137ab5d81..f243937e33d0847 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java @@ -207,7 +207,8 @@ public String toString() { "outputExpr", outputExpressions, "partitionExpr", partitionExpressions, "topnFilter", topnPushInfo != null, - "topnPushDown", getMutableState(MutableState.KEY_PUSH_TOPN_TO_AGG).isPresent() + "topnPushInfo", getMutableState(MutableState.KEY_PUSH_TOPN_TO_AGG).isPresent() + ? (TopnPushInfo) getMutableState(MutableState.KEY_PUSH_TOPN_TO_AGG).get() : "false" ); } @@ -329,6 +330,15 @@ public TopnPushInfo(List orderkeys, long limit) { this.orderkeys = ImmutableList.copyOf(orderkeys); this.limit = limit; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("["); + builder.append("orderkeys=").append(orderkeys); + builder.append(", limit=").append(limit); + builder.append("]"); + return builder.toString(); + } } public TopnPushInfo getTopnPushInfo() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 25637d1b8166568..813a3edb7d2c312 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -420,7 +420,7 @@ public static Optional extractSlotOrCastOnSlot(Expression expr) { /** * Generate replaceMap Slot -> Expression from NamedExpression[Expression as name] */ - public static Map generateReplaceMap(List namedExpressions) { + public static Map generateReplaceMap(List namedExpressions) { Map replaceMap = Maps.newLinkedHashMapWithExpectedSize(namedExpressions.size()); for (NamedExpression namedExpression : namedExpressions) { if (namedExpression instanceof Alias) { @@ -484,7 +484,7 @@ public static Set replace(Set exprs, /** * Replace expression node in the expression tree by `replaceMap` in top-down manner. */ - public static List replaceNamedExpressions(List namedExpressions, + public static List replaceNamedExpressions(List namedExpressions, Map replaceMap) { Builder replaceExprs = ImmutableList.builderWithExpectedSize(namedExpressions.size()); for (NamedExpression namedExpression : namedExpressions) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java index 0076c232340d29f..28f7cda427ed7fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java @@ -121,8 +121,8 @@ public static List adjustNullableForRepeat( /** * merge childProjects with parentProjects */ - public static List mergeProjections(List childProjects, - List parentProjects) { + public static List mergeProjections(List childProjects, + List parentProjects) { Map replaceMap = ExpressionUtils.generateReplaceMap(childProjects); return ExpressionUtils.replaceNamedExpressions(parentProjects, replaceMap); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java index 446f49c37829022..049b18ff431d4dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java @@ -412,9 +412,9 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve if (!conjuncts.isEmpty()) { output.append(detailPrefix).append("having: ").append(getExplainString(conjuncts)).append("\n"); } - output.append(detailPrefix).append("sortByGroupKey:").append(sortByGroupKey != null).append("\n"); output.append(detailPrefix).append(String.format( "cardinality=%,d", cardinality)).append("\n"); + output.append(detailPrefix).append("sortInfo:").append(sortByGroupKey).append("\n"); return output.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index d62c00534b6d948..4d3ce439a674bb5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2231,13 +2231,13 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { public boolean enableSortSpill = false; @VariableMgr.VarAttr( - name = "ENABLE_COMPRESS_MATERIALIZE", + name = "enable_compress_materialize", description = {"控制是否启用compress materialize。", "enable compress-materialize. "}, needForward = true, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL ) - public boolean enableCompressMaterialize = false; + public boolean enableCompressMaterialize = true; @VariableMgr.VarAttr( name = ENABLE_AGG_SPILL, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java index a91c0dd47126fcf..6462b1da5714a4e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java @@ -101,6 +101,7 @@ protected void runBeforeAll() throws Exception { SSBUtils.createTables(this); createView("CREATE VIEW V1 AS SELECT * FROM part"); createView("CREATE VIEW V2 AS SELECT * FROM part"); + connectContext.getSessionVariable().enableCompressMaterialize = false; } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 45e1190412d0a4f..95e6cf5fe790a54 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -249,6 +249,7 @@ protected void runBeforeAll() throws Exception { + ");\n"); // Should not make scan to empty relation when the table used by materialized view has no data connectContext.getSessionVariable().setDisableNereidsRules("OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION,ELIMINATE_GROUP_BY_KEY_BY_UNIFORM"); + connectContext.getSessionVariable().enableCompressMaterialize = false; } // Test when join both side are all partition table and partition column name is same diff --git a/regression-test/data/nereids_p0/compress_materialize/compress_materialize.out b/regression-test/data/nereids_p0/compress_materialize/compress_materialize.out index eee04795628144d..8922b2155aec2b1 100644 --- a/regression-test/data/nereids_p0/compress_materialize/compress_materialize.out +++ b/regression-test/data/nereids_p0/compress_materialize/compress_materialize.out @@ -53,3 +53,21 @@ a 1 中 8 bb 3 +-- !explain_sort_agg -- +cost = 19.563333333333333 +PhysicalResultSink[294] ( outputExprs=[v1#1] ) ++--PhysicalProject[289]@5 ( stats=1, projects=[v1#1] ) + +--PhysicalQuickSort[284]@4 ( stats=1, orderKeys=[encode_as_bigint(v1)#4 asc null first], phase=MERGE_SORT ) + +--PhysicalDistribute[279]@7 ( stats=1, distributionSpec=DistributionSpecGather ) + +--PhysicalQuickSort[274]@7 ( stats=1, orderKeys=[encode_as_bigint(v1)#4 asc null first], phase=LOCAL_SORT ) + +--PhysicalProject[269]@3 ( stats=1, projects=[decode_as_varchar(encode_as_bigint(v1)#3) AS `v1`#1, encode_as_bigint(decode_as_varchar(encode_as_bigint(v1)#3)) AS `encode_as_bigint(decode_as_varchar(encode_as_bigint(v1)))`#4], multi_proj=l0([encode_as_bigint(v1)#3, decode_as_varchar(encode_as_bigint(v1)#3) AS `decode_as_varchar(encode_as_bigint(v1))`#5])l1([decode_as_varchar(encode_as_bigint(v1))#5 AS `v1`#1, encode_as_bigint(decode_as_varchar(encode_as_bigint(v1))#5) AS `encode_as_bigint(decode_as_varchar(encode_as_bigint(v1)))`#4]) ) + +--PhysicalHashAggregate[264]@2 ( stats=1, aggPhase=GLOBAL, aggMode=BUFFER_TO_RESULT, maybeUseStreaming=false, groupByExpr=[encode_as_bigint(v1)#3], outputExpr=[encode_as_bigint(v1)#3], partitionExpr=Optional[[encode_as_bigint(v1)#3]], topnFilter=false, topnPushDown=false ) + +--PhysicalDistribute[259]@8 ( stats=1, distributionSpec=DistributionSpecHash ( orderedShuffledColumns=[3], shuffleType=EXECUTION_BUCKETED, tableId=-1, selectedIndexId=-1, partitionIds=[], equivalenceExprIds=[[3]], exprIdToEquivalenceSet={3=0} ) ) + +--PhysicalHashAggregate[254]@8 ( stats=1, aggPhase=LOCAL, aggMode=INPUT_TO_BUFFER, maybeUseStreaming=true, groupByExpr=[encode_as_bigint(v1)#3], outputExpr=[encode_as_bigint(v1)#3], partitionExpr=Optional[[encode_as_bigint(v1)#3]], topnFilter=false, topnPushDown=false ) + +--PhysicalProject[249]@1 ( stats=1, projects=[encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#3] ) + +--PhysicalOlapScan[t1]@0 ( stats=1 ) + +-- !exec_sort_agg -- +a +b + diff --git a/regression-test/data/nereids_p0/compress_materialize/pushdown_encode.out b/regression-test/data/nereids_p0/compress_materialize/pushdown_encode.out new file mode 100644 index 000000000000000..da63c94a5abc17b --- /dev/null +++ b/regression-test/data/nereids_p0/compress_materialize/pushdown_encode.out @@ -0,0 +1,36 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !exec_sort_agg -- +a +b + +-- !exec_sort_filter -- +a +b + +-- !exec_agg_join -- +b + +-- !agg_join_2 -- +b 2 + +-- !nlj -- +a 9 +b 7 + +-- !union -- +1 +2 +2 +3 +4 + +-- !intersect -- +2 + +-- !except -- +1 + +-- !agg_sort -- +a +b + diff --git a/regression-test/data/nereids_rules_p0/pull_up_join_from_union/pull_up_join_from_union.out b/regression-test/data/nereids_rules_p0/pull_up_join_from_union/pull_up_join_from_union.out index 10820e9ee484140..ecef42943dcedf8 100644 --- a/regression-test/data/nereids_rules_p0/pull_up_join_from_union/pull_up_join_from_union.out +++ b/regression-test/data/nereids_rules_p0/pull_up_join_from_union/pull_up_join_from_union.out @@ -80,13 +80,11 @@ PhysicalResultSink -- !complex_join_condition1 -- PhysicalResultSink ---PhysicalUnion -----hashJoin[INNER_JOIN] hashCondition=((expr_cast(id as BIGINT) = expr_(cast(id as BIGINT) - 1))) otherCondition=() -------PhysicalOlapScan[table_a] +--hashJoin[INNER_JOIN] hashCondition=((expr_(cast(id as BIGINT) - 1) = expr_cast(id as BIGINT))) otherCondition=() +----PhysicalUnion ------PhysicalOlapScan[table_b] -----hashJoin[INNER_JOIN] hashCondition=((expr_cast(id as BIGINT) = expr_(cast(id as BIGINT) - 1))) otherCondition=() -------PhysicalOlapScan[table_a] ------PhysicalOlapScan[table_c] +----PhysicalOlapScan[table_a] -- !complex_join_condition2 -- PhysicalResultSink diff --git a/regression-test/suites/correctness_p0/test_first_value_window.groovy b/regression-test/suites/correctness_p0/test_first_value_window.groovy index 7c1582e0e61b601..da582fae8bef1ea 100644 --- a/regression-test/suites/correctness_p0/test_first_value_window.groovy +++ b/regression-test/suites/correctness_p0/test_first_value_window.groovy @@ -16,6 +16,7 @@ // under the License. suite("test_first_value_window") { + // errror sql """ set enable_nereids_planner = true; """ sql """ set enable_fallback_to_original_planner = false; """ diff --git a/regression-test/suites/mv_p0/test_substr/test_substr.groovy b/regression-test/suites/mv_p0/test_substr/test_substr.groovy index 7d6ae82634fd8c5..0bbd8b64c61bc79 100644 --- a/regression-test/suites/mv_p0/test_substr/test_substr.groovy +++ b/regression-test/suites/mv_p0/test_substr/test_substr.groovy @@ -18,8 +18,10 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite ("test_substr") { - sql """set enable_nereids_planner=true""" - sql """SET enable_fallback_to_original_planner=false""" + sql "set enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + sql "set enable_compress_materialize=false;" + sql """ drop table if exists dwd;""" sql """ diff --git a/regression-test/suites/nereids_p0/compress_materialize/compress_materialize.groovy b/regression-test/suites/nereids_p0/compress_materialize/compress_materialize.groovy index 8489de2aa2a1fd1..3599220209c503b 100644 --- a/regression-test/suites/nereids_p0/compress_materialize/compress_materialize.groovy +++ b/regression-test/suites/nereids_p0/compress_materialize/compress_materialize.groovy @@ -193,6 +193,5 @@ suite("compress_materialize") { qt_sort "select * from compressSort order by k desc, v"; qt_sort "select * from compressSort order by k desc nulls last"; qt_sort "select * from compressSort order by k desc nulls last, v limit 3"; - } diff --git a/regression-test/suites/nereids_p0/compress_materialize/pushdown_encode.groovy b/regression-test/suites/nereids_p0/compress_materialize/pushdown_encode.groovy new file mode 100644 index 000000000000000..265799339877b2f --- /dev/null +++ b/regression-test/suites/nereids_p0/compress_materialize/pushdown_encode.groovy @@ -0,0 +1,292 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("pushdown_encode") { +// push down encode slot + sql """ + drop table if exists t1; + CREATE TABLE t1 ( + `k1` int NOT NULL, + `v1` char(5) NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + + insert into t1 values (1, "a"), (2, "b"); + + drop table if exists t2; + CREATE TABLE t2 ( + `k2` int NOT NULL, + `v2` char(5) NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k2`) + DISTRIBUTED BY HASH(`k2`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + + insert into t2 values (3, "c"), (4, "d"), (2, "b"); + + set disable_join_reorder=true; + """ + + explain{ + sql """ + physical plan + select v1 + from (select sum(k1) as k, v1 from t1 group by v1) t + order by v1; + """ + contains("orderKeys=[encode_as_bigint(v1)#4 asc null first]") + contains("projects=[decode_as_varchar(encode_as_bigint(v1)#3) AS `v1`#1, encode_as_bigint(v1)#3 AS `encode_as_bigint(v1)`#4]") + contains("groupByExpr=[encode_as_bigint(v1)#3]") + contains("projects=[encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#3]") + } + + order_qt_exec_sort_agg """ + select v1 + from (select sum(k1) as k, v1 from t1 group by v1) t + order by v1; + """ + + explain{ + sql """ + physical plan + select v1 + from t1 + where k1 > 0 + order by v1; + """ + contains("orderKeys=[encode_as_bigint(v1)#2 asc null first]") + contains("projects=[decode_as_varchar(encode_as_bigint(v1#1)) AS `decode_as_varchar(encode_as_bigint(v1))`#1, encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#2]") + } + + order_qt_exec_sort_filter """ + select v1 + from t1 + where k1 > 0 + order by v1; + """ + + explain{ + sql """ + physical plan + select v1 + from t1 join t2 on v1=v2 + group by v1; + """ + contains("groupByExpr=[encode_as_bigint(v1)#4]") + contains("projects=[encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#4]") + } + + order_qt_exec_agg_join""" + select v1 + from t1 join t2 on v1=v2 + group by v1; + """ + + explain { + sql""" + physical plan + select v1 + from t1 join t2 on v1=v2 + group by v1; + """ + contains("groupByExpr=[encode_as_bigint(v1)#4]") + contains("projects=[encode_as_bigint(v1)#4]") + contains("hashCondition=[(encode_as_bigint(v1)#4 = encode_as_bigint(v2)#5)]") + contains("projects=[encode_as_bigint(v2#3) AS `encode_as_bigint(v2)`#5]") + contains("projects=[encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#4]") + } + + explain { + // because of "length(v1)>0", encode not pushed down through join + sql """ + physical plan + select v1 + from t1 left join t2 on v1=v2 and length(v1)>0 + group by v1; + """ + contains("hashCondition=[(v1#1 = v2#3)], otherCondition=[(length(v1#1) > 0)]") + } + + order_qt_agg_join_2 """ + select v1, sum(k2) + from t1 join t2 on v1=v2 + group by v1;""" + + explain { + sql """physical plan + select v1, sum(k2) + from t1 join t2 on v1=v2 + group by v1;""" + contains("projects=[decode_as_varchar(encode_as_bigint(v1)#5) AS `v1`#1, sum(k2)#4]") + contains("groupByExpr=[encode_as_bigint(v1)#5]") + contains("projects=[encode_as_bigint(v1)#5, k2#2]") + contains("hashCondition=[(encode_as_bigint(v1)#5 = encode_as_bigint(v2)#6)]") + contains("projects=[encode_as_bigint(v2#3) AS `encode_as_bigint(v2)`#6, k2#2]") + contains("projects=[encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#5]") + } + + explain { + sql """ + physical plan + select v1, sum(k2) + from t1 right outer join t2 on v1 < v2 + group by v1; + """ + contains("groupByExpr=[encode_as_bigint(v1)#5]") + contains("projects=[encode_as_bigint(v1)#5, k2#2]") + contains("otherCondition=[(encode_as_bigint(v1)#5 < encode_as_bigint(v2)#6)]") + contains("projects=[encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#5]") + contains("projects=[encode_as_bigint(v2#3) AS `encode_as_bigint(v2)`#6, k2#2]") + } + + + explain { + sql """ + physical plan + select v1, sum(k2) + from + (select t1.k1, t1.v1 from t1 join t2 on v1 < concat(v2,'a')) t3 + join t2 on t3.k1=t2.k2 + group by v1; + """ + contains("otherCondition=[(v1#1 < concat(v2, 'a')#8)]") + contains("projects=[k1#0, encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#7]") + + // +--PhysicalHashJoin[730]@7 ( stats=3, type=INNER_JOIN, hashCondition=[(k1#0 = k2#4)], otherCondition=[], markCondition=[], runtimeFilters=[RF0[k2#4->[k1#0](ndv/size = 3/4) , RF1[k2#4->[k1#0](ndv/size = 3/4) ] ) + // |--PhysicalProject[714]@4 ( stats=3, projects=[k1#0, encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#7] ) + // | +--PhysicalNestedLoopJoin[709]@3 ( stats=3, type=INNER_JOIN, hashCondition=[], otherCondition=[(v1#1 < concat(v2, 'a')#8)], markCondition=[] ) + // | |--PhysicalProject[698]@2 ( stats=3, projects=[concat(v2#3, 'a') AS `concat(v2, 'a')`#8] ) + // | | +--PhysicalOlapScan[t2]@1 ( stats=3 ) + // | +--PhysicalDistribute[704]@0 ( stats=2, distributionSpec=DistributionSpecReplicated ) + // | +--PhysicalOlapScan[t1]@0 ( stats=2, RFs= RF0 RF1 ) + // +--PhysicalDistribute[725]@6 ( stats=3, distributionSpec=DistributionSpecReplicated ) + // +--PhysicalProject[720]@6 ( stats=3, projects=[k2#4] ) + // +--PhysicalOlapScan[t2]@5 ( stats=3 ) + } + + + order_qt_nlj """ + select v1, sum(k2) + from t1 right outer join t2 on v1 < v2 and v2>"abc" + group by v1; + """ + + // not push through join, because v2>"abc" + sql """ + select v1, sum(k2) + from t1 right outer join t2 on v1 = v2 and (k1!=k2 or v2>"abc") + group by v1; + """ + explain { + sql """ + shape plan + select v1, sum(k2) + from t1 right outer join t2 on v1 = v2 and (k1!=k2 or v2>"abc") + group by v1; + """ + contains "hashCondition=((t1.v1 = t2.v2))" + } + + explain { + sql """ + physical plan + select k + from ( + (select k1 as k, v1 as v from t1) + union all + (select k2 as k, v2 as v from t2) + ) T + order by v; + """ + contains("orderKeys=[encode_as_bigint(v)#10 asc null first]") + contains("outputs=[k#8, encode_as_bigint(v)#10], regularChildrenOutputs=[[k#4, encode_as_bigint(v)#11], [k#6, encode_as_bigint(v)#12]]") + contains("projects=[k1#0 AS `k`#4, encode_as_bigint(v1#1) AS `encode_as_bigint(v)`#11]") + contains("projects=[k2#2 AS `k`#6, encode_as_bigint(v2#3) AS `encode_as_bigint(v)`#12]") + } + + order_qt_union """ + select k + from ( + (select k1 as k, v1 as v from t1) + union all + (select k2 as k, v2 as v from t2) + ) T + order by v; + """ + order_qt_intersect """ + select k + from ( + (select k1 as k, v1 as v from t1) + intersect + (select k2 as k, v2 as v from t2) + ) T + order by v; + """ + + order_qt_except """ + select k + from ( + (select k1 as k, v1 as v from t1) + except + (select k2 as k, v2 as v from t2) + ) T + order by v; + """ + + order_qt_agg_sort """ + select v1 + from (select v1 from t1 where k1 > 0 order by v1 limit 10) t group by v1 + """ + + explain{ + sql """ + physical plan + select v1 + from (select v1 from t1 where k1 > 0 order by v1 limit 10) t group by v1 + """ + contains("projects=[decode_as_varchar(encode_as_bigint(v1#1)) AS `decode_as_varchar(encode_as_bigint(v1))`#1, encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#3]") + } + + // if encodeBody is used in windowExpression, do not push encode down + + sql """ + SELECT v1, k1, k2 + , sum(k2) OVER (PARTITION BY v1 ORDER BY k1) AS w_sum + FROM t1 + JOIN t2 ON k1 = k2 - 2 + ORDER BY k1, v1, w_sum; + """ + + explain { + sql """ + physical plan + SELECT v1, k1, k2 + , sum(k2) OVER (PARTITION BY v1 ORDER BY k1) AS w_sum + FROM t1 + JOIN t2 ON k1 = k2 - 2 + ORDER BY k1, v1, w_sum; + """ + contains("orderKeys=[k1#0 asc null first, encode_as_bigint(v1)#5 asc null first, w_sum#4 asc null first]") + } + +} \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy index 9139fea3c9db4a9..342bb7484d259f2 100644 --- a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy +++ b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_gby_key.groovy @@ -64,7 +64,7 @@ suite("eliminate_gby_key") { select t2_c1 from temp; """) - contains("groupByExpr=[c1#13, c3#18], outputExpr=[c1#13, c3#18]") + contains("groupByExpr=[c1#14, c3#19], outputExpr=[c1#14, c3#19]") } explain { @@ -84,7 +84,7 @@ suite("eliminate_gby_key") { select t2_c2 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[encode_as_int(substring(c2, 1, 3))#20, c1#14, c3#19], outputExpr=[encode_as_int(substring(c2, 1, 3))#20, c1#14, c3#19]") } explain { @@ -104,7 +104,7 @@ suite("eliminate_gby_key") { select c3 from temp; """) - contains("groupByExpr=[c1#13, c3#18], outputExpr=[c1#13, c3#18]") + contains("groupByExpr=[c1#14, c3#19], outputExpr=[c1#14, c3#19]") } explain { @@ -124,7 +124,7 @@ suite("eliminate_gby_key") { select cnt from temp; """) - contains("groupByExpr=[c1#13, c3#18], outputExpr=[c1#13, c3#18") + contains("groupByExpr=[c1#14, c3#19], outputExpr=[c1#14, c3#19") } explain { @@ -144,7 +144,7 @@ suite("eliminate_gby_key") { select t2_c2, t2_c1 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[encode_as_int(substring(c2, 1, 3))#20, c1#14, c3#19], outputExpr=[encode_as_int(substring(c2, 1, 3))#20, c1#14, c3#19]") } explain { @@ -164,7 +164,7 @@ suite("eliminate_gby_key") { select c3, t2_c1 from temp; """) - contains("groupByExpr=[c1#13, c3#18], outputExpr=[c1#13, c3#18]") + contains("groupByExpr=[c1#14, c3#19], outputExpr=[c1#14, c3#19]") } explain { @@ -184,7 +184,7 @@ suite("eliminate_gby_key") { select c3, t2_c2 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[encode_as_int(substring(c2, 1, 3))#20, c1#14, c3#19], outputExpr=[encode_as_int(substring(c2, 1, 3))#20, c1#14, c3#19]") } explain { @@ -204,7 +204,7 @@ suite("eliminate_gby_key") { select t2_c1, cnt from temp; """) - contains("groupByExpr=[c1#13, c3#18], outputExpr=[c1#13, c3#18") + contains("groupByExpr=[c1#14, c3#19], outputExpr=[c1#14, c3#19,") } explain { @@ -224,7 +224,7 @@ suite("eliminate_gby_key") { select c3, cnt from temp; """) - contains("groupByExpr=[c1#13, c3#18], outputExpr=[c1#13, c3#18") + contains("groupByExpr=[c1#14, c3#19], outputExpr=[c1#14, c3#19") } explain { @@ -244,7 +244,7 @@ suite("eliminate_gby_key") { select t2_c1, c3, cnt from temp; """) - contains("groupByExpr=[c1#13, c3#18], outputExpr=[c1#13, c3#18") + contains("groupByExpr=[c1#14, c3#19], outputExpr=[c1#14, c3#19") } explain { @@ -264,7 +264,7 @@ suite("eliminate_gby_key") { select t2_c2, c3, t2_c1 from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18]") + contains("groupByExpr=[encode_as_int(substring(c2, 1, 3))#20, c1#14, c3#19], outputExpr=[encode_as_int(substring(c2, 1, 3))#20, c1#14, c3#19]") } explain { @@ -284,6 +284,6 @@ suite("eliminate_gby_key") { select t2_c2, c3, t2_c1, cnt from temp; """) - contains("groupByExpr=[t2_c2#19, c1#13, c3#18], outputExpr=[t2_c2#19, c1#13, c3#18,") + contains("groupByExpr=[encode_as_int(substring(c2, 1, 3))#20, c1#14, c3#19], outputExpr=[encode_as_int(substring(c2, 1, 3))#20, c1#14, c3#19,") } } diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index 5b8aa6e00d23080..33ffb02a47a7921 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -19,6 +19,7 @@ suite("aggregate_with_roll_up") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" sql "set runtime_filter_mode=OFF"; + sql "set enable_compress_materialize=false;" sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" sql """ diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/any_value_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/any_value_roll_up.groovy index 44815c8a1427f07..dcb966161e33c73 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/any_value_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/any_value_roll_up.groovy @@ -21,6 +21,7 @@ suite("any_value_roll_up") { sql "use ${db}" sql "set runtime_filter_mode=OFF"; sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + sql "set enable_compress_materialize=false;" sql """ drop table if exists orders_2 diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy index fd41789d02661ed..9d889452ea17c10 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy @@ -21,6 +21,7 @@ This file is used specifically to test the presence of AGGs under joins. suite("dimension_2_join_agg_replenish") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" + sql "set enable_compress_materialize=false;" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy index 71154fbeb276cf8..9896804f2d92d42 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy @@ -21,6 +21,8 @@ suite("outer_join_dphyp") { sql "set runtime_filter_mode=OFF"; sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" sql "SET enable_dphyp_optimizer = true" + sql "set enable_compress_materialize=false;" + sql """ drop table if exists orders """ diff --git a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy index cdc4632ff7379b2..68817a34c5b7cd2 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy @@ -20,6 +20,7 @@ suite("outer_join") { sql "use ${db}" sql "set runtime_filter_mode=OFF"; sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + sql "set enable_compress_materialize=false;" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy b/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy index 5ae587910b6ce50..51dce141de5836b 100644 --- a/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy +++ b/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy @@ -26,12 +26,12 @@ suite("push_topn_to_agg") { sql "set push_topn_to_agg=false" explain{ sql "select o_custkey, sum(o_shippriority) from orders group by o_custkey limit 4;" - multiContains ("sortByGroupKey:false", 2) + multiContains ("sortInfo:[o_custkey]", 2) } sql "set push_topn_to_agg=true" explain{ sql "select o_custkey, sum(o_shippriority) from orders group by o_custkey limit 4;" - multiContains ("sortByGroupKey:true", 2) + multiContains ("sortInfo:[o_custkey]", 2) notContains("STREAMING") } @@ -39,14 +39,14 @@ suite("push_topn_to_agg") { // limit -> proj -> agg, explain{ sql "select sum(c_custkey), c_name from customer group by c_name limit 6;" - multiContains ("sortByGroupKey:true", 2) + multiContains ("sortInfo:[c_name]", 2) notContains("STREAMING") } // topn -> agg explain{ sql "select o_custkey, sum(o_shippriority) from orders group by o_custkey order by o_custkey limit 8;" - multiContains ("sortByGroupKey:true", 2) + multiContains ("sortInfo:[o_custkey]", 2) notContains("STREAMING") } @@ -55,9 +55,9 @@ suite("push_topn_to_agg") { // 2. append o_custkey to order key explain{ sql "select sum(o_shippriority) from orders group by o_custkey, o_clerk order by o_clerk limit 11;" - contains("sortByGroupKey:true") - contains("group by: o_clerk[#10], o_custkey[#9]") contains("order by: o_clerk[#18] ASC, o_custkey[#19] ASC") + contains("group by: o_clerk[#12], o_custkey[#13]") + contains("sortInfo:[o_clerk, o_custkey]") } @@ -66,7 +66,8 @@ suite("push_topn_to_agg") { sql "select sum(distinct o_shippriority) from orders group by o_orderkey limit 13; " contains("VTOP-N") contains("order by: o_orderkey") - multiContains("sortByGroupKey:true", 1) + multiContains("sortInfo:[o_orderkey]", 1) // the upper agg + contains("sortInfo:null") // the middle and bottom agg } // multi distinct @@ -74,31 +75,32 @@ suite("push_topn_to_agg") { sql "select count(distinct o_clerk), sum(distinct o_shippriority) from orders group by o_orderkey limit 14; " contains("VTOP-N") contains("order by: o_orderkey") - multiContains("sortByGroupKey:true", 2) + multiContains("sortInfo:[o_orderkey]", 3) } // use group key as sort key to enable topn-push opt explain { sql "select sum(o_shippriority) from orders group by o_clerk limit 14; " - contains("sortByGroupKey:true") + contains("sortInfo:[o_clerk]") } // group key is expression explain { sql "select sum(o_shippriority), o_clerk+1 from orders group by o_clerk+1 limit 15; " - contains("sortByGroupKey:true") + contains("order by: (cast(o_clerk as DOUBLE) + cast(1 as DOUBLE))") + contains("group by: (cast(o_clerk as DOUBLE) + cast(1 as DOUBLE))") + contains("sortInfo:[(cast(o_clerk as DOUBLE) + cast(1 as DOUBLE))]") } // order key is not part of group key explain { sql "select o_custkey, sum(o_shippriority) from orders group by o_custkey order by o_custkey+1 limit 16; " - contains("sortByGroupKey:false") - notContains("sortByGroupKey:true") + contains("sortInfo:null") } // topn + one phase agg explain { sql "select sum(ps_availqty), ps_partkey, ps_suppkey from partsupp group by ps_partkey, ps_suppkey order by ps_partkey, ps_suppkey limit 18;" - contains("sortByGroupKey:true") + contains("sortInfo:[ps_partkey, ps_suppkey]") } } \ No newline at end of file diff --git a/regression-test/suites/query_p0/limit/test_group_by_limit.groovy b/regression-test/suites/query_p0/limit/test_group_by_limit.groovy index d6f49f9d7a397f4..076f3e203f83fd0 100644 --- a/regression-test/suites/query_p0/limit/test_group_by_limit.groovy +++ b/regression-test/suites/query_p0/limit/test_group_by_limit.groovy @@ -17,105 +17,122 @@ suite("test_group_by_limit", "query") { -sql 'set enable_agg_spill=false' + sql 'set enable_agg_spill=false' -sql 'set enable_force_spill=false' + sql 'set enable_force_spill=false' -sql 'set topn_opt_limit_threshold=10' + sql 'set topn_opt_limit_threshold=10' -sql "set experimental_ENABLE_COMPRESS_MATERIALIZE=true;" + // different types + qt_select1 """ select sum(orderkey), count(partkey), shipmode from tpch_tiny_lineitem group by shipmode limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), shipmode from tpch_tiny_lineitem group by shipmode limit 3; " + contains("VTOP-N") + contains("sortInfo:[encode_as_largeint(shipmode)]") + } -// different types -qt_select1 """ select sum(orderkey), count(partkey), shipmode from tpch_tiny_lineitem group by shipmode limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), shipmode from tpch_tiny_lineitem group by shipmode limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} + qt_select2 """ select sum(orderkey), count(partkey), linenumber from tpch_tiny_lineitem group by linenumber limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), linenumber from tpch_tiny_lineitem group by linenumber limit 3; " + contains("VTOP-N") + contains("sortInfo:[linenumber]") + } -qt_select2 """ select sum(orderkey), count(partkey), linenumber from tpch_tiny_lineitem group by linenumber limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), linenumber from tpch_tiny_lineitem group by linenumber limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} + qt_select3 """ select sum(orderkey), count(partkey), tax from tpch_tiny_lineitem group by tax limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), tax from tpch_tiny_lineitem group by tax limit 3; " + contains("VTOP-N") + contains("sortInfo:[tax]") + } -qt_select3 """ select sum(orderkey), count(partkey), tax from tpch_tiny_lineitem group by tax limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), tax from tpch_tiny_lineitem group by tax limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} + qt_select4 """ select sum(orderkey), count(partkey), commitdate from tpch_tiny_lineitem group by commitdate limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), commitdate from tpch_tiny_lineitem group by commitdate limit 3; " + contains("VTOP-N") + contains("sortInfo:[commitdate]") + } -qt_select4 """ select sum(orderkey), count(partkey), commitdate from tpch_tiny_lineitem group by commitdate limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), commitdate from tpch_tiny_lineitem group by commitdate limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} + // group by functions + qt_select5 """ select sum(orderkey), count(partkey), cast(commitdate as datetime) from tpch_tiny_lineitem group by cast(commitdate as datetime) limit 3; """ + explain { + sql " select sum(orderkey), count(partkey), cast(commitdate as datetime) from tpch_tiny_lineitem group by cast(commitdate as datetime) limit 3; " + contains("VTOP-N") + contains("sortInfo:[cast(commitdate as DATETIMEV2(0))]") + } -// group by functions -qt_select5 """ select sum(orderkey), count(partkey), cast(commitdate as datetime) from tpch_tiny_lineitem group by cast(commitdate as datetime) limit 3; """ -explain { - sql " select sum(orderkey), count(partkey), cast(commitdate as datetime) from tpch_tiny_lineitem group by cast(commitdate as datetime) limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} + qt_select6 """ select sum(orderkey), count(partkey), month(commitdate) from tpch_tiny_lineitem group by month(commitdate) limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), month(commitdate) from tpch_tiny_lineitem group by month(commitdate) limit 3; " + contains("VTOP-N") + contains("sortInfo:[month(commitdate)]") + } -qt_select6 """ select sum(orderkey), count(partkey), month(commitdate) from tpch_tiny_lineitem group by month(commitdate) limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), month(commitdate) from tpch_tiny_lineitem group by month(commitdate) limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} + // mutli column + qt_select7 """ select sum(orderkey), count(partkey), shipmode, linenumber from tpch_tiny_lineitem group by shipmode, linenumber limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), shipmode, linenumber from tpch_tiny_lineitem group by shipmode, linenumber limit 3; " + contains("VTOP-N") + contains("sortInfo:[encode_as_largeint(shipmode), linenumber]") + } + qt_select8 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax limit 3; " + contains("VTOP-N") + contains("sortInfo:[encode_as_largeint(shipmode), linenumber, tax]") + } + qt_select9 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax , commitdate from tpch_tiny_lineitem group by shipmode, linenumber, tax, commitdate limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax , commitdate from tpch_tiny_lineitem group by shipmode, linenumber, tax, commitdate limit 3; " + contains("VTOP-N") + contains("sortInfo:[encode_as_largeint(shipmode), linenumber, tax, commitdate]") + } -// mutli column -qt_select7 """ select sum(orderkey), count(partkey), shipmode, linenumber from tpch_tiny_lineitem group by shipmode, linenumber limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), shipmode, linenumber from tpch_tiny_lineitem group by shipmode, linenumber limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} -qt_select8 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} -qt_select9 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax , commitdate from tpch_tiny_lineitem group by shipmode, linenumber, tax, commitdate limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax , commitdate from tpch_tiny_lineitem group by shipmode, linenumber, tax, commitdate limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} + // group by + order by -// group by + order by + // group by columns eq order by columns + qt_select10 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode, linenumber, tax limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode, linenumber, tax limit 3; " + contains("VTOP-N") + contains("sortInfo:[encode_as_largeint(shipmode), linenumber, tax]") + } + // group by columns contains order by columns + qt_select11 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode limit 3; " + contains("VTOP-N") + contains("sortInfo:[encode_as_largeint(shipmode), linenumber, tax]") + } + // desc order by column + qt_select12 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax limit 3; " + contains("VTOP-N") + contains("sortInfo:[encode_as_largeint(shipmode), linenumber, tax]") + } + qt_select13 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax desc limit 3; """ + explain{ + sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax desc limit 3; " + contains("VTOP-N") + contains("sortInfo:[encode_as_largeint(shipmode), linenumber, tax]") + } -// group by columns eq order by columns -qt_select10 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode, linenumber, tax limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode, linenumber, tax limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} -// group by columns contains order by columns -qt_select11 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} -// desc order by column -qt_select12 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} -qt_select13 """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax desc limit 3; """ -explain{ - sql " select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax desc limit 3; " - contains("VTOP-N") - contains("sortByGroupKey:true") -} + explain { + sql """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by rollup(shipmode, linenumber, tax) order by shipmode desc, linenumber, tax desc limit 3; """ + contains("sortInfo:null") + } + + explain { + sql """ select sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by rollup(shipmode, linenumber, tax) limit 3; """ + notContains("VTOP-N") + contains("sortInfo:null") + } + + explain { + sql """ select sum(orderkey), count(partkey) from tpch_tiny_lineitem limit 3; """ + notContains("VTOP-N") + contains("sortInfo:null") + } + + }