From 37f318284d43b6e1747eee44c6de4fe44e53bb5b Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 29 Nov 2024 19:15:35 +0800 Subject: [PATCH] sort-proj --- .../rules/rewrite/PushDownEncodeSlot.java | 37 ++++++++++--- .../trees/plans/logical/LogicalOlapScan.java | 2 +- .../compress_materialize.groovy | 52 +++++++++++++++++++ 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownEncodeSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownEncodeSlot.java index c97fd876962b2a8..750a18ead7cdb63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownEncodeSlot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownEncodeSlot.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.util.PlanUtils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.util.ArrayList; @@ -91,11 +92,24 @@ private List collectEncodeAlias(LogicalProject project) { return encodeAlias; } + private LogicalProject rewriteRootProject(LogicalProject project, + List encodeAlias) { + List projections = project.getProjects().stream().map( + e -> encodeAlias.contains(e) ? e.toSlot() : e) + .collect(Collectors.toList()); + return project.withProjects(projections); + + } + private LogicalProject pushDownEncodeSlot(LogicalProject project) { List encodeAlias = collectEncodeAlias(project); - LogicalProject result = (LogicalProject) - project.accept(EncodeSlotPushDownVisitor.INSTANCE, encodeAlias); - return result; + project = rewriteRootProject(project, encodeAlias); + Plan child = project.child(); + Plan newChild = child.accept(EncodeSlotPushDownVisitor.INSTANCE, encodeAlias); + if (child.equals(newChild)) { + project = (LogicalProject) project.withChildren(newChild); + } + return project; } /** @@ -104,7 +118,8 @@ private LogicalProject pushDownEncodeSlot(LogicalProject> { public static EncodeSlotPushDownVisitor INSTANCE = new EncodeSlotPushDownVisitor(); - public LogicalProject replaceProjectsEncodeSlot(LogicalProject project, Map replaceMap) { + public LogicalProject replaceProjectsEncodeSlot(LogicalProject project, + Map replaceMap) { List newProjections = new ArrayList<>(); boolean changed = false; for (NamedExpression expr : project.getProjects()) { @@ -149,7 +164,7 @@ public Plan visit(Plan plan, List encodeAlias) { // rewrite plan according to encode expression // for example: project(encode_as_int(slot1) as slot2) // 1. rewrite project's expressions: project(slot2), - // 2. push encode_as_int(slot1) as slot2 down to project.child() + // 2. push "encode_as_int(slot1) as slot2" down to project.child() // rewrite expressions if (plan instanceof LogicalProject) { plan = replaceProjectsEncodeSlot((LogicalProject)plan, replaceMap); @@ -191,7 +206,17 @@ public Plan visit(Plan plan, List encodeAlias) { private Set getPassThroughSlots(Plan plan) { Set outputSlots = Sets.newHashSet(plan.getOutputSet()); - outputSlots.removeAll(plan.getInputSlots()); + Set keySlots = Sets.newHashSet(); + if (plan instanceof LogicalProject) { + for (NamedExpression e : ((LogicalProject) plan).getProjects()) { + if (!(e instanceof SlotReference)) { + keySlots.addAll(e.getInputSlots()); + } + } + } else { + keySlots = plan.getInputSlots(); + } + outputSlots.removeAll(keySlots); return outputSlots; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index b7b9f155b910603..90ceb24231340c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -217,7 +217,7 @@ public OlapTable getTable() { @Override public String toString() { return Utils.toSqlString("LogicalOlapScan", - "name", table.getName(), + "qualified", qualifiedName(), "indexName", getSelectedMaterializedIndexName().orElse(""), "selectedIndexId", selectedIndexId, "preAgg", preAggStatus diff --git a/regression-test/suites/nereids_p0/compress_materialize/compress_materialize.groovy b/regression-test/suites/nereids_p0/compress_materialize/compress_materialize.groovy index 8489de2aa2a1fd1..6da6174279e879e 100644 --- a/regression-test/suites/nereids_p0/compress_materialize/compress_materialize.groovy +++ b/regression-test/suites/nereids_p0/compress_materialize/compress_materialize.groovy @@ -193,6 +193,58 @@ suite("compress_materialize") { qt_sort "select * from compressSort order by k desc, v"; qt_sort "select * from compressSort order by k desc nulls last"; qt_sort "select * from compressSort order by k desc nulls last, v limit 3"; + + // push down encode slot + sql """ + drop table if exists t1; + CREATE TABLE t1 ( + `k1` int NOT NULL, + `v1` char(5) NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + + insert into t1 values (1, "a"), (2, "b"); + + drop table if exists t2; + CREATE TABLE t2 ( + `k2` int NOT NULL, + `v2` char(5) NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k2`) + DISTRIBUTED BY HASH(`k2`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + insert into t2 values (3, "c"), (4, "d"); + """ + + qt_explain_sort_agg """ + explain physical plan + select v1 + from (select sum(k1) as k, v1 from t1 group by v1) t + order by v1; + """ + + qt_exec_sort_agg """ + select v1 + from (select sum(k1) as k, v1 from t1 group by v1) t + order by v1; + """ + + qt_sort_proj + """ + explain physical plan + select v1 + from t1 + where k1 > 0 + order by v1; + """ + + // TODO: select k, v as x from (select k1 as k, v1 as v from t1) A union all (select k2 as k, v2 as v from t2) order by x; -- 下推 union }