From c2a377b487171a1f2b1f15cde17b38715a4572c8 Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 2 Nov 2023 21:45:20 +0800 Subject: [PATCH] [planner](poc)Poc --- .../apache/doris/nereids/CascadesContext.java | 12 + .../apache/doris/nereids/NereidsPlanner.java | 14 +- .../cascades/OptimizeGroupExpressionJob.java | 18 +- .../jobs/joinorder/hypergraph/Node.java | 5 + .../org/apache/doris/nereids/memo/Group.java | 12 + .../org/apache/doris/nereids/memo/Memo.java | 17 + .../apache/doris/nereids/rules/RuleSet.java | 9 + .../apache/doris/nereids/rules/RuleType.java | 18 + .../rewrite/DeferMaterializeTopNResult.java | 2 +- ...AbstractMaterializedViewAggregateRule.java | 24 ++ .../mv/AbstractMaterializedViewJoinRule.java | 157 ++++++++ .../mv/AbstractMaterializedViewRule.java | 367 ++++++++++++++++++ .../nereids/rules/rewrite/mv/Mapping.java | 117 ++++++ .../rewrite/mv/MaterializationContext.java | 68 ++++ .../mv/MaterializedProjectFilterRule.java | 34 ++ .../mv/MaterializedViewAggregateRule.java | 33 ++ .../mv/MaterializedViewProjectJoinRule.java | 46 +++ .../nereids/rules/rewrite/mv/StructInfo.java | 105 +++++ .../nereids/trees/expressions/Alias.java | 3 +- .../trees/expressions/ArrayItemReference.java | 3 +- .../trees/expressions/SlotReference.java | 54 ++- .../visitor/ExpressionVisitors.java | 40 ++ .../trees/metadata/EquivalenceClass.java | 122 ++++++ .../trees/metadata/PlanMetadataQuery.java | 61 +++ .../nereids/trees/metadata/Predicates.java | 120 ++++++ .../plans/logical/LogicalCatalogRelation.java | 2 +- .../trees/plans/logical/LogicalOlapScan.java | 4 +- .../plans/logical/LogicalTVFRelation.java | 2 +- .../physical/PhysicalCatalogRelation.java | 2 +- .../plans/physical/PhysicalTVFRelation.java | 2 +- .../trees/plans/visitor/PlanVisitors.java | 188 +++++++++ .../doris/nereids/util/ExpressionUtils.java | 60 +++ .../rewrite/mv/MaterializedViewTest.java | 103 +++++ .../doris/nereids/util/PlanChecker.java | 45 ++- 34 files changed, 1828 insertions(+), 41 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewAggregateRule.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewJoinRule.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewRule.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/Mapping.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializationContext.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedProjectFilterRule.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewAggregateRule.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewProjectJoinRule.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/StructInfo.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/EquivalenceClass.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/PlanMetadataQuery.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/Predicates.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitors.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index ebef71feb850b00..1c5440897443707 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -44,6 +44,7 @@ import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; +import org.apache.doris.nereids.rules.rewrite.mv.MaterializationContext; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -112,6 +113,8 @@ public class CascadesContext implements ScheduleContext { private final Optional currentTree; private final Optional parent; + private List materializationContexts; + /** * Constructor of OptimizerContext. * @@ -133,6 +136,7 @@ private CascadesContext(Optional parent, Optional curren this.currentJobContext = new JobContext(this, requireProperties, Double.MAX_VALUE); this.subqueryExprIsAnalyzed = new HashMap<>(); this.runtimeFilterContext = new RuntimeFilterContext(getConnectContext().getSessionVariable()); + this.materializationContexts = new ArrayList<>(); } /** @@ -309,6 +313,14 @@ public void setOuterScope(@Nullable Scope outerScope) { this.outerScope = Optional.ofNullable(outerScope); } + public List getMaterializationContexts() { + return materializationContexts; + } + + public void addMaterializationContext(MaterializationContext materializationContext) { + this.materializationContexts.add(materializationContext); + } + /** * getAndCacheSessionVariable */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index c9c9e7b67b77bf0..e931fb312d011cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -243,9 +243,12 @@ private LogicalPlan preprocess(LogicalPlan logicalPlan) { } private void initCascadesContext(LogicalPlan plan, PhysicalProperties requireProperties) { - cascadesContext = CascadesContext.initContext(statementContext, plan, requireProperties); - if (statementContext.getConnectContext().getTables() != null) { - cascadesContext.setTables(statementContext.getConnectContext().getTables()); + // should call statementContext.getConnectContext().getEnv().getMgr + if (cascadesContext == null) { + cascadesContext = CascadesContext.initContext(statementContext, plan, requireProperties); + if (statementContext.getConnectContext().getTables() != null) { + cascadesContext.setTables(statementContext.getConnectContext().getTables()); + } } } @@ -463,6 +466,11 @@ public CascadesContext getCascadesContext() { return cascadesContext; } + @VisibleForTesting + public void setCascadesContext(CascadesContext cascadesContext) { + this.cascadesContext = cascadesContext; + } + public static PhysicalProperties buildInitRequireProperties() { return PhysicalProperties.GATHER; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java index 38c0c4484bc6099..48026f1ba432c0d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java @@ -23,7 +23,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.Rule; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; /** @@ -76,22 +76,26 @@ private List getExplorationRules() { || (groupExpression.getOwnerGroup() != null && !groupExpression.getOwnerGroup().isStatsReliable()); int joinNumBushyTree = context.getCascadesContext().getConnectContext() .getSessionVariable().getMaxJoinNumBushyTree(); + + List exploreRules = new ArrayList<>(getRuleSet().getMaterializedViewRules()); + if (isDisableJoinReorder) { - return Collections.emptyList(); + //todo } else if (isDpHyp) { if (isOtherJoinReorder) { - return getRuleSet().getDPHypReorderRules(); + exploreRules.addAll(getRuleSet().getDPHypReorderRules()); } else { - return Collections.emptyList(); + //todo } } else if (isLeftZigZagTree) { return getRuleSet().getLeftZigZagTreeJoinReorder(); } else if (isEnableBushyTree) { - return getRuleSet().getBushyTreeJoinReorder(); + exploreRules.addAll(getRuleSet().getBushyTreeJoinReorder()); } else if (context.getCascadesContext().getStatementContext().getMaxNAryInnerJoin() <= joinNumBushyTree) { - return getRuleSet().getBushyTreeJoinReorder(); + exploreRules.addAll(getRuleSet().getBushyTreeJoinReorder()); } else { - return getRuleSet().getZigZagTreeJoinReorder(); + exploreRules.addAll(getRuleSet().getZigZagTreeJoinReorder()); } + return exploreRules; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java index f8a9bafe3c7b920..6b162cf0afae870 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java @@ -28,6 +28,11 @@ /** * HyperGraph Node. + * Jc + * \ + * F + * \ + * JC */ public class Node { private final int index; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index 8009156e1340638..347a479a3ce7a23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.cost.Cost; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.rewrite.mv.StructInfo; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; @@ -74,6 +75,8 @@ public class Group { private int chosenGroupExpressionId = -1; + private Optional structInfo = Optional.empty(); + /** * Constructor for Group. * @@ -152,6 +155,7 @@ public GroupExpression logicalExpressionsAt(int index) { * @return the first logical group expression in this group */ public GroupExpression getLogicalExpression() { + // poc tmp Preconditions.checkArgument(logicalExpressions.size() == 1, "There should be only one Logical Expression in Group"); return logicalExpressions.get(0); @@ -532,4 +536,12 @@ public String treeString() { return TreeStringUtils.treeString(this, toString, getChildren, getExtraPlans, displayExtraPlan); } + + public Optional getStructInfo() { + return structInfo; + } + + public void setStructInfo(StructInfo structInfo) { + this.structInfo = Optional.ofNullable(structInfo); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index 5b75cb2d76e1934..9e0419fae3b27dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -314,6 +314,7 @@ private Group init(Plan plan) { plan = replaceChildrenToGroupPlan(plan, childrenGroups); GroupExpression newGroupExpression = new GroupExpression(plan, childrenGroups); Group group = new Group(groupIdGenerator.getNextId(), newGroupExpression, plan.getLogicalProperties()); + // PoC add struct info to group groups.put(group.getGroupId(), group); if (groupExpressions.containsKey(newGroupExpression)) { @@ -323,6 +324,22 @@ private Group init(Plan plan) { return group; } + /** initPoC */ + public Group initPoC(Plan plan) { + Preconditions.checkArgument(!(plan instanceof GroupPlan), "Cannot init memo by a GroupPlan"); + + /* initialize children recursively */ + List childrenGroups = new ArrayList<>(plan.arity()); + for (Plan child : plan.children()) { + childrenGroups.add(initPoC(child)); + } + + plan = replaceChildrenToGroupPlan(plan, childrenGroups); + GroupExpression newGroupExpression = new GroupExpression(plan, childrenGroups); + Group group = new Group(groupIdGenerator.getNextId(), newGroupExpression, plan.getLogicalProperties()); + return group; + } + /** * add or replace the plan into the target group. *

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 80163663184227b..b22b7c8152d6f74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -92,6 +92,7 @@ import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughWindow; import org.apache.doris.nereids.rules.rewrite.PushdownJoinOtherCondition; import org.apache.doris.nereids.rules.rewrite.PushdownProjectThroughLimit; +import org.apache.doris.nereids.rules.rewrite.mv.MaterializedViewProjectJoinRule; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -220,6 +221,10 @@ public class RuleSet { .add(JoinCommute.BUSHY.build()) .build(); + public static final List MATERIALIZED_VIEW_RULES = planRuleFactories() + .add(MaterializedViewProjectJoinRule.INSTANCE) + .build(); + public List getDPHypReorderRules() { return DPHYP_REORDER_RULES; } @@ -240,6 +245,10 @@ public List getImplementationRules() { return IMPLEMENTATION_RULES; } + public List getMaterializedViewRules() { + return MATERIALIZED_VIEW_RULES; + } + public static RuleFactories planRuleFactories() { return new RuleFactories(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 94ccef528fd996b..edad74e2e198040 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -225,6 +225,24 @@ public enum RuleType { MATERIALIZED_INDEX_PROJECT_SCAN(RuleTypeClass.REWRITE), MATERIALIZED_INDEX_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE), MATERIALIZED_INDEX_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE), + + MATERIALIZED_VIEW_PROJECT_JOIN(RuleTypeClass.REWRITE), + MATERIALIZED_VIEW_FILTER_JOIN(RuleTypeClass.REWRITE), + MATERIALIZED_VIEW_PROJECT_FILTER_JOIN(RuleTypeClass.REWRITE), + MATERIALIZED_VIEW_FILTER_PROJECT_JOIN(RuleTypeClass.REWRITE), + MATERIALIZED_VIEW_ONLY_JOIN(RuleTypeClass.REWRITE), + + MATERIALIZED_VIEW_PROJECT_AGGREGATE(RuleTypeClass.REWRITE), + MATERIALIZED_VIEW_FILTER_AGGREGATE(RuleTypeClass.REWRITE), + MATERIALIZED_VIEW_PROJECT_FILTER_AGGREGATE(RuleTypeClass.REWRITE), + MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE(RuleTypeClass.REWRITE), + MATERIALIZED_VIEW_ONLY_AGGREGATE(RuleTypeClass.REWRITE), + + MATERIALIZED_VIEW_FILTER_SCAN(RuleTypeClass.REWRITE), + MATERIALIZED_VIEW_PROJECT_SCAN(RuleTypeClass.REWRITE), + MATERIALIZED_VIEW_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE), + MATERIALIZED_VIEW_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE), + OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE), FILE_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE), PUSH_CONJUNCTS_INTO_JDBC_SCAN(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java index 15516e0501f490b..381f66fb6395c6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java @@ -83,7 +83,7 @@ private Plan deferMaterialize(LogicalResultSink logicalResultSin LogicalTopN logicalTopN, Optional> logicalFilter, LogicalOlapScan logicalOlapScan) { Column rowId = new Column(Column.ROWID_COL, Type.STRING, false, null, false, "", "rowid column"); - SlotReference columnId = SlotReference.fromColumn(rowId, logicalOlapScan.getQualifier()); + SlotReference columnId = SlotReference.fromColumn(rowId, logicalOlapScan.getQualifier(), null); Set deferredMaterializedExprIds = Sets.newHashSet(logicalOlapScan.getOutputExprIdSet()); logicalFilter.ifPresent(filter -> filter.getConjuncts() .forEach(e -> deferredMaterializedExprIds.removeAll(e.getInputSlotExprIds()))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewAggregateRule.java new file mode 100644 index 000000000000000..1027b83093011b6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewAggregateRule.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.mv; + +/** + * AbstractMaterializedViewAggregateRule + * */ +public abstract class AbstractMaterializedViewAggregateRule extends AbstractMaterializedViewRule { +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewJoinRule.java new file mode 100644 index 000000000000000..f23e874630cd6d7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewJoinRule.java @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.mv; + +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitors; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitors.SlotReferenceReplacer.ExprReplacer; + +import com.google.common.collect.BiMap; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * AbstractMaterializedViewJoinRule + */ +public abstract class AbstractMaterializedViewJoinRule extends AbstractMaterializedViewRule { + + @Override + protected Plan rewriteView(MatchMode matchMode, + StructInfo queryStructInfo, + StructInfo viewStructInfo, + BiMap queryToViewTableMappings, + Plan tempRewritedPlan) { + + List expressions = rewriteExpression(queryStructInfo.getTopExpressions(), + queryStructInfo, + viewStructInfo, + queryToViewTableMappings, + tempRewritedPlan + ); + if (expressions == null) { + return queryStructInfo.getPlan(); + } + // PoC Generate mapping from query slot reference to mv slot reference, note: clone + // if any slot can not map then bail out + // simplfy implement + Set querySlotSet = new HashSet<>(); + queryStructInfo.getPlan().accept(PlanVisitors.SLOT_REFERENCE_COLLECTOR, querySlotSet); + + Set viewSlotSet = new HashSet<>(); + viewStructInfo.getPlan().accept(PlanVisitors.SLOT_REFERENCE_COLLECTOR, viewSlotSet); + + Map queryToViewSlotMapping = new HashMap<>(); + for (SlotReference querySlot : querySlotSet) { + for (SlotReference viewSlot : viewSlotSet) { + if (Objects.equals(querySlot.getName(), viewSlot.getName()) + && Objects.equals(querySlot.getQualifier(), viewSlot.getQualifier())) { + queryToViewSlotMapping.put(querySlot, viewSlot); + } + } + } + // PoC Generate mapping from mv sql output to mv scan out put + Map mvToMvScanMapping = new HashMap<>(); + List mvScanSlotList = tempRewritedPlan.getOutput(); + List mvSlotList = viewStructInfo.getPlan().getOutput(); + for (int i = 0; i < mvSlotList.size(); i++) { + mvToMvScanMapping.put((SlotReference) mvSlotList.get(i), (SlotReference) mvScanSlotList.get(i)); + } + + // TODO check if the query expr can derive from the view + // PoC If the query expression can get from mv sql, so replace the mv scan slot reference + // PoC according to the mapping above. Simplify implement + Map mvScanToQueryMapping = new HashMap<>(); + List output = queryStructInfo.getPlan().getOutput(); + for (Slot querySlot : output) { + Slot mvSlot = queryToViewSlotMapping.get(querySlot); + if (mvSlot == null) { + return null; + } + SlotReference mvScanSlot = mvToMvScanMapping.get(mvSlot); + if (mvScanSlot == null) { + return null; + } + mvScanToQueryMapping.put(mvScanSlot, querySlot); + } + // Replace the mv scan output with query slot, lazy before add filter and other project + + // tempRewritedPlan.accept(SlotReferenceReplacer.INSTANCE, mvScanToQueryMapping); + + tempRewritedPlan.getOutput().stream() + .forEach(slot -> slot.accept(ExprReplacer.INSTANCE, mvScanToQueryMapping)); + LogicalProject planLogicalProject = new LogicalProject<>( + output.stream().map(NamedExpression.class::cast).collect(Collectors.toList()), + tempRewritedPlan); + return planLogicalProject; + } + + protected boolean isPatternSupport(LogicalProject topProject, Plan plan) { + if (topProject != null) { + return PatternChecker.INSTANCE.visit(topProject, null); + } + return PatternChecker.INSTANCE.visit(plan, null); + } + + static class PatternChecker extends DefaultPlanVisitor { + public static final PatternChecker INSTANCE = new PatternChecker(); + + @Override + public Boolean visit(Plan plan, Void context) { + if (plan == null) { + return true; + } + if (!(plan instanceof LogicalPlan)) { + return false; + } + if (!(plan instanceof LogicalRelation) + && !(plan instanceof LogicalProject) + && !(plan instanceof LogicalFilter) + && !(plan instanceof LogicalJoin)) { + return false; + } + super.visit(plan, context); + return true; + } + + @Override + public Boolean visitGroupPlan(GroupPlan groupPlan, Void context) { + Group group = groupPlan.getGroup(); + return group.getLogicalExpressions().stream() + .anyMatch(groupExpression -> groupExpression.getPlan().accept(this, context)); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewRule.java new file mode 100644 index 000000000000000..59eefa20b7e0084 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewRule.java @@ -0,0 +1,367 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.mv; + +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.metadata.EquivalenceClass; +import org.apache.doris.nereids.trees.metadata.PlanMetadataQuery; +import org.apache.doris.nereids.trees.metadata.Predicates; +import org.apache.doris.nereids.trees.metadata.Predicates.SplitPredicate; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.algebra.Project; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitors; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * AbstractMaterializedViewRule + */ +public abstract class AbstractMaterializedViewRule { + + protected List rewrite(LogicalProject queryTopProject, Plan queryPlan, CascadesContext cascadesContext) { + List materializationContexts = cascadesContext.getMaterializationContexts(); + List rewriteResults = new ArrayList<>(); + if (materializationContexts.isEmpty()) { + return rewriteResults; + } + // check query queryPlan + if (!isPatternSupport(queryTopProject, queryPlan)) { + return rewriteResults; + } + StructInfo queryStructInfo = extractStructInfo(queryTopProject, queryPlan, cascadesContext); + if (!checkStructInfo(queryStructInfo)) { + return rewriteResults; + } + + // PoC hyper graph query + // HyperGraph queryHyperGraph = new HyperGraph(); + // Plan plan = queryStructInfo.getPlan(); + // plan = plan.accept(new PlanVisitors.GroupPlanRemover(), null); + // Group poCGroup = cascadesContext.getMemo().initPoC(plan); + // JoinOrderJob joinOrderJob = new JoinOrderJob(poCGroup, cascadesContext.getCurrentJobContext()); + // joinOrderJob.buildGraph(poCGroup, queryHyperGraph); + + for (MaterializationContext materializationContext : materializationContexts) { + Plan mvPlan = materializationContext.getMvPlan(); + LogicalProject viewTopProject; + Plan viewPlan; + // TODO get table and scan from materialization context + // poc child(0) remove the logical result sink + Plan viewScanNode = materializationContext.getScanPlan().child(0); + if (mvPlan instanceof Project) { + viewTopProject = (LogicalProject) mvPlan; + viewPlan = mvPlan.child(0); + } else { + viewTopProject = null; + viewPlan = mvPlan; + } + // TODO Normalize to remove resultSink + if (viewPlan instanceof LogicalResultSink) { + viewPlan = (Plan) ((LogicalResultSink) viewPlan).child(); + } + if (!isPatternSupport(viewTopProject, viewPlan)) { + continue; + } + StructInfo viewStructInfo = extractStructInfo(viewTopProject, viewPlan, cascadesContext); + if (!checkStructInfo(viewStructInfo)) { + continue; + } + + // Poc Hyper graph view + HyperGraph viewHyperGraph = new HyperGraph(); + Plan view = viewStructInfo.getPlan(); + view = view.accept(new PlanVisitors.GroupPlanRemover(), null); + Group poCViewGroup = cascadesContext.getMemo().initPoC(view); + JoinOrderJob viewJoinOrderJob = new JoinOrderJob(poCViewGroup, cascadesContext.getCurrentJobContext()); + viewJoinOrderJob.buildGraph(poCViewGroup, viewHyperGraph); + + MatchMode matchMode = decideMatchMode(queryStructInfo, viewStructInfo); + if (MatchMode.NOT_MATCH == matchMode) { + continue; + } + List> queryToViewTableMappings = + generateRelationMap(queryStructInfo, viewStructInfo); + for (BiMap queryToViewTableMapping : queryToViewTableMappings) { + Expression compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo, + queryToViewTableMapping); + if (compensatePredicates == null) { + continue; + } + Plan rewritedPlan; + if (compensatePredicates instanceof BooleanLiteral + && ((BooleanLiteral) compensatePredicates).getValue()) { + rewritedPlan = viewScanNode; + } else { + // try to compensate predicates by using mv scan, Poc is always true + List rewriteCompensatePredicates = rewriteExpression( + ImmutableList.of(compensatePredicates), + queryStructInfo, + viewStructInfo, + queryToViewTableMapping.inverse(), + viewScanNode); + if (rewriteCompensatePredicates.isEmpty()) { + continue; + } + rewritedPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), viewScanNode); + } + rewritedPlan = rewriteView(matchMode, queryStructInfo, viewStructInfo, + queryToViewTableMapping, rewritedPlan); + if (rewritedPlan == null) { + continue; + } + rewriteResults.add(rewritedPlan); + } + } + return rewriteResults; + } + + // Aggregate rewriting(roll up) or join rewriting according to different inherit class implement + protected Plan rewriteView(MatchMode matchMode, + StructInfo queryStructInfo, + StructInfo viewStructInfo, + BiMap queryToViewTableMappings, + Plan temporaryRewrite) { + return temporaryRewrite; + } + + protected SplitPredicate rewriteExpression(SplitPredicate splitPredicate, + StructInfo sourceStructInfo, + StructInfo targetStructInfo, + BiMap sourceToTargetMapping, + Plan targetScanNode) { + // call below rewriteExpression + return null; + } + + // Use target targetScanNode out to represent the source expression + protected List rewriteExpression(List sourceExpression, + StructInfo sourceStructInfo, + StructInfo targetStructInfo, + BiMap sourceToTargetMapping, + Plan targetScanNode) { + // Firstly, rewrite the target plan output slot using query with inverse mapping + // and record the position(another way, according to unify bottom mapping and up to bottom mapping separately, + // then represent query using target) + // then rewrite the sourceExpression to use the viewScanNode with above map + // source target + // project (slot 1, 2) project(slot 3, 2, 1) + // scan(table) scan(table) + // + // transform to: + // project (slot 2, 1) + // target + // + return ImmutableList.of(BooleanLiteral.of(true)); + } + + protected Expression predicatesCompensate( + StructInfo queryStructInfo, + StructInfo viewStructInfo, + BiMap queryToViewTableMapping + ) { + // TODO Predicate compensate should be common and move to util + return BooleanLiteral.of(true); + // Equal predicate compensate + // EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass(); + // EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass(); + // if (queryEquivalenceClass.isEmpty() + // && !viewEquivalenceClass.isEmpty()) { + // return null; + // } + // // TODO Equals compare the exprId, which should be the absolutely table and column. + // Mapping mapping = queryEquivalenceClass.generateMapping(viewEquivalenceClass); + // if (mapping == null) { + // return null; + // } + // List> queryValues = queryEquivalenceClass.getEquivalenceValues(); + // List> viewValues = viewEquivalenceClass.getEquivalenceValues(); + // Expression compensatePredicate = BooleanLiteral.of(true); + // for (int i = 0; i < queryValues.size(); i++) { + // List targets = mapping.getTargetBy(i); + // if (!targets.isEmpty()) { + // // Add only predicates that are not there + // for (int j : targets) { + // Set difference = new HashSet<>(queryValues.get(i)); + // difference.removeAll(viewValues.get(j)); + // for (SlotReference e : difference) { + // Expression equals = new EqualTo(e, viewValues.get(j).iterator().next()); + // compensatePredicate = new And(compensatePredicate, equals); + // } + // } + // } else { + // // Add all predicates + // Iterator querySlotIterator = queryValues.get(i).iterator(); + // SlotReference first = querySlotIterator.next(); + // while (querySlotIterator.hasNext()) { + // Expression equals = new EqualTo(first, querySlotIterator.next()); + // compensatePredicate = new And(compensatePredicate, equals); + // } + // } + // // TODO RangePredicates and ResidualPredicates + // } + // return compensatePredicate; + } + + // Generate table mapping between query table to view table + protected List> generateRelationMap( + StructInfo queryStructInfo, + StructInfo viewStrutInfo) { + List queryRelations = new ArrayList<>(); + PlanVisitors.TABLE_COLLECTOR_INSTANCE.visit(queryStructInfo.getPlan(), queryRelations); + + List viewRelations = new ArrayList<>(); + PlanVisitors.TABLE_COLLECTOR_INSTANCE.visit(viewStrutInfo.getPlan(), viewRelations); + + Multimap queryTableRelationIdMap = ArrayListMultimap.create(); + for (CatalogRelation relation : queryRelations) { + queryTableRelationIdMap.put(relation.getTable(), relation.getRelationId()); + } + Multimap viewTableRelationIdMap = ArrayListMultimap.create(); + for (CatalogRelation relation : viewRelations) { + viewTableRelationIdMap.put(relation.getTable(), relation.getRelationId()); + } + // TODO Just support 1:1 + BiMap mappingMap = HashBiMap.create(); + for (Map.Entry queryEntry : queryTableRelationIdMap.entries()) { + Collection viewTableIds = viewTableRelationIdMap.get(queryEntry.getKey()); + mappingMap.put(queryEntry.getValue(), viewTableIds.iterator().next()); + } + return ImmutableList.of(mappingMap); + } + + protected MatchMode decideMatchMode(StructInfo queryStructInfo, StructInfo viewStructInfo) { + List queryTableRefs = queryStructInfo.getRelations() + .stream() + .map(CatalogRelation::getTable).collect( + Collectors.toList()); + List viewTableRefs = viewStructInfo.getRelations() + .stream() + .map(CatalogRelation::getTable).collect( + Collectors.toList()); + boolean sizeSame = viewTableRefs.size() == queryTableRefs.size(); + boolean queryPartial = viewTableRefs.containsAll(queryTableRefs); + boolean viewPartial = queryTableRefs.containsAll(viewTableRefs); + if (sizeSame && queryPartial && viewPartial) { + return MatchMode.COMPLETE; + } + if (!sizeSame && queryPartial) { + return MatchMode.QUERY_PARTIAL; + } + if (!sizeSame && viewPartial) { + return MatchMode.VIEW_PARTIAL; + } + return MatchMode.NOT_MATCH; + } + + protected boolean checkStructInfo(StructInfo info) { + if (info.getRelations().isEmpty()) { + return false; + } + if (!info.getPredicates().getCanNotPulledUpPredicates().isEmpty()) { + return false; + } + return true; + } + + protected StructInfo extractStructInfo(LogicalProject topProject, Plan topInput, CascadesContext cascadesContext) { + + Plan query = topProject == null ? topInput : topProject; + if (query.getGroupExpression().isPresent() + && query.getGroupExpression().get().getOwnerGroup().getStructInfo().isPresent()) { + Group belongGroup = query.getGroupExpression().get().getOwnerGroup(); + return belongGroup.getStructInfo().get(); + } else { + // PoC build hyper graph and set into group + HyperGraph hyperGraph = new HyperGraph(); + query = query.accept(new PlanVisitors.GroupPlanRemover(), null); + Group poCGroup = cascadesContext.getMemo().initPoC(query); + JoinOrderJob joinOrderJob = new JoinOrderJob(poCGroup, cascadesContext.getCurrentJobContext()); + joinOrderJob.buildGraph(poCGroup, hyperGraph); + + // Support rewrite TableType.OLAP currently + // get tables + List relations = + PlanMetadataQuery.getTables(topInput, Sets.newHashSet(TableType.OLAP)); + + Predicates predicates = PlanMetadataQuery.getPredicates(topInput); + + // get predicates (include join condition and filter) + Expression composedExpressions = predicates.composedExpression(); + SplitPredicate splitPredicate = Predicates.splitPredicates(composedExpressions); + + // construct equivalenceClass according to equals predicates + final EquivalenceClass equivalenceClass = new EquivalenceClass(); + List equalPredicates = + ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicates()); + for (Expression expression : equalPredicates) { + EqualTo equalTo = (EqualTo) expression; + equivalenceClass.addEquivalenceClass( + (SlotReference) equalTo.getArguments().get(0), + (SlotReference) equalTo.getArguments().get(1)); + } + + // set on current group and mv scan not set + StructInfo structInfo = StructInfo.of(relations, predicates, equivalenceClass, + topProject, topInput, hyperGraph); + if (query.getGroupExpression().isPresent()) { + query.getGroupExpression().get().getOwnerGroup().setStructInfo(structInfo); + } + return structInfo; + } + } + + protected boolean isPatternSupport(LogicalProject topProject, Plan plan) { + return false; + } + + /** + * MatchMode + */ + protected enum MatchMode { + COMPLETE, + VIEW_PARTIAL, + QUERY_PARTIAL, + NOT_MATCH + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/Mapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/Mapping.java new file mode 100644 index 000000000000000..59d3653db1eaac9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/Mapping.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.mv; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * mapping query to view transversely + * and mapping current node to input node vertically + */ +public class Mapping { + + private final int sourceCount; + private final int targetCount; + // TODO more efficient implement + private final List mappings; + + public Mapping(int sourceCount, int targetCount, List mappings) { + this.sourceCount = sourceCount; + this.targetCount = targetCount; + this.mappings = mappings == null ? new ArrayList<>() : mappings; + } + + public int getSourceCount() { + return sourceCount; + } + + public int getTargetCount() { + return targetCount; + } + + public List getMappings() { + return mappings; + } + + /** + * Mapping + */ + public static Mapping of(int sourceCount, int targetCount, List mappings) { + return new Mapping(sourceCount, targetCount, mappings); + } + + public static Mapping of(int sourceCount, int targetCount) { + return new Mapping(sourceCount, targetCount, new ArrayList<>()); + } + + public static Mapping empty() { + return new Mapping(0, 0, ImmutableList.of()); + } + + public void addMapping(IntPair pair) { + this.mappings.add(pair); + } + + public boolean isEmpty() { + return sourceCount == 0 && targetCount == 0 && this.mappings.isEmpty(); + } + + public List getTargetBy(int source) { + return mappings.stream() + .filter(pair -> pair.getSource() == source) + .map(IntPair::getTarget) + .collect(Collectors.toList()); + } + + public List getTarget() { + return mappings.stream().map(IntPair::getTarget).collect(Collectors.toList()); + } + + public List getSource() { + return mappings.stream().map(IntPair::getSource).collect(Collectors.toList()); + } + + /** + * IntPair + */ + public static final class IntPair { + public final int source; + public final int target; + + public IntPair(int source, int target) { + this.source = source; + this.target = target; + } + + public static IntPair of(int left, int right) { + return new IntPair(left, right); + } + + public int getSource() { + return source; + } + + public int getTarget() { + return target; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializationContext.java new file mode 100644 index 000000000000000..07d9907570f9069 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializationContext.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.mv; + +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.View; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.memo.GroupId; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * MaterializationContext + */ +public class MaterializationContext { + + // TODO: 2023/11/1 add MaterializedView class + private final Plan mvPlan; + private final CascadesContext context; + private final List baseTables; + private final List baseViews; + // Group ids that are rewritten by this mv to reduce rewrite times + private final Set matchedGroups = new HashSet<>(); + private final Plan scanPlan; + + public MaterializationContext(Plan mvPlan, CascadesContext context, + List
baseTables, List baseViews, Plan scanPlan) { + this.mvPlan = mvPlan; + this.context = context; + this.baseTables = baseTables; + this.baseViews = baseViews; + this.scanPlan = scanPlan; + } + + public Set getMatchedGroups() { + return matchedGroups; + } + + public void addMatchedGroup(GroupId groupId) { + matchedGroups.add(groupId); + } + + public Plan getMvPlan() { + return mvPlan; + } + + public Plan getScanPlan() { + return scanPlan; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedProjectFilterRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedProjectFilterRule.java new file mode 100644 index 000000000000000..76d870637570a4c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedProjectFilterRule.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.mv; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; + +import java.util.List; + +/** + * MaterializedProjectFilterRule + * */ +public class MaterializedProjectFilterRule extends AbstractMaterializedViewRule implements RewriteRuleFactory { + + @Override + public List buildRules() { + return null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewAggregateRule.java new file mode 100644 index 000000000000000..0c450e55408d7d0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewAggregateRule.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.mv; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; + +import java.util.List; + +/** + * MaterializedViewAggregateRule + * */ +public class MaterializedViewAggregateRule extends AbstractMaterializedViewAggregateRule implements RewriteRuleFactory { + @Override + public List buildRules() { + return null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewProjectJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewProjectJoinRule.java new file mode 100644 index 000000000000000..e5305e2971916da --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewProjectJoinRule.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.mv; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RulePromise; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * MaterializedViewJoinRule + * */ +public class MaterializedViewProjectJoinRule extends AbstractMaterializedViewJoinRule implements RewriteRuleFactory { + + public static final MaterializedViewProjectJoinRule INSTANCE = new MaterializedViewProjectJoinRule(); + + @Override + public List buildRules() { + return ImmutableList.of(logicalProject(logicalJoin(any(), any())).thenApplyMulti(ctx -> { + LogicalProject> root = ctx.root; + return rewrite(root, root.child(), ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_ONLY_JOIN, RulePromise.EXPLORE)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/StructInfo.java new file mode 100644 index 000000000000000..63362bab8d9363d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/StructInfo.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.mv; + +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.metadata.EquivalenceClass; +import org.apache.doris.nereids.trees.metadata.Predicates; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.algebra.Project; + +import java.util.List; + +/** + * StructInfo + */ +public class StructInfo { + private final List relations; + private final Predicates predicates; + private final EquivalenceClass equivalenceClass; + private final Project topProject; + private final Plan topInput; + private final HyperGraph hyperGraph; + + private StructInfo(List relations, + Predicates predicates, + EquivalenceClass equivalenceClass, + Project topProject, + Plan topInput, + HyperGraph hyperGraph) { + this.relations = relations; + this.predicates = predicates; + this.equivalenceClass = equivalenceClass; + this.topProject = topProject; + this.topInput = topInput; + this.hyperGraph = hyperGraph; + } + + public static StructInfo of(List relations, + Predicates predicates, + EquivalenceClass equivalenceClass, + Project topProject, + Plan topInput, + HyperGraph hyperGraph) { + return new StructInfo(relations, predicates, equivalenceClass, topProject, topInput, hyperGraph); + } + + public List getRelations() { + return relations; + } + + public Predicates getPredicates() { + return predicates; + } + + public EquivalenceClass getEquivalenceClass() { + return equivalenceClass; + } + + public Plan getTopInput() { + return topInput; + } + + public Project getTopProject() { + return topProject; + } + + public HyperGraph getHyperGraph() { + return hyperGraph; + } + + public List getTopExpressions() { + return getTopProject() == null ? extractReferences(topInput) : + getTopProject().getProjects(); + } + + public Plan getPlan() { + return getTopProject() == null ? getTopInput() : (Plan) getTopProject(); + } + + /** + * It returns a list of references to all columns in the node. + * If the node is an Aggregate, it returns only the list of references to the grouping columns. + * The returned list is immutable. + */ + private List extractReferences(Plan plan) { + return null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java index 877f792e5019edb..86c89efdc9a03ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java @@ -75,7 +75,8 @@ public Slot toSlot() throws UnboundException { child() instanceof SlotReference ? ((SlotReference) child()).getColumn().orElse(null) : null, - nameFromChild ? Optional.of(child().toString()) : Optional.of(name)); + nameFromChild ? Optional.of(child().toString()) : Optional.of(name), + Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java index 226e8d1f9b25380..a7f1fe171ede293 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java @@ -142,7 +142,8 @@ public static class ArrayItemSlot extends SlotReference implements SlotNotFromCh * @param nullable true if nullable */ public ArrayItemSlot(ExprId exprId, String name, DataType dataType, boolean nullable) { - super(exprId, name, dataType, nullable, ImmutableList.of(), null, Optional.empty()); + super(exprId, name, dataType, nullable, ImmutableList.of(), null, Optional.empty(), + Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index db0f4e6635980dc..831bec709880e7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.DataType; import com.google.common.base.Preconditions; @@ -33,7 +34,7 @@ * Reference to slot in expression. */ public class SlotReference extends Slot { - protected final ExprId exprId; + protected ExprId exprId; protected final String name; protected final DataType dataType; protected final boolean nullable; @@ -43,29 +44,31 @@ public class SlotReference extends Slot { // different SlotReference will have different internalName // TODO: remove this member variable after mv selection is refactored protected final Optional internalName; - private final Column column; + private final Optional relationId; public SlotReference(String name, DataType dataType) { - this(StatementScopeIdGenerator.newExprId(), name, dataType, true, ImmutableList.of(), null, Optional.empty()); + this(StatementScopeIdGenerator.newExprId(), name, dataType, true, ImmutableList.of(), null, Optional.empty(), + Optional.empty()); } public SlotReference(String name, DataType dataType, boolean nullable) { - this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, ImmutableList.of(), - null, Optional.empty()); + this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, ImmutableList.of(), null, + Optional.empty(), Optional.empty()); } public SlotReference(String name, DataType dataType, boolean nullable, List qualifier) { - this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, qualifier, null, Optional.empty()); + this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, qualifier, null, Optional.empty(), + Optional.empty()); } public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, List qualifier) { - this(exprId, name, dataType, nullable, qualifier, null, Optional.empty()); + this(exprId, name, dataType, nullable, qualifier, null, Optional.empty(), Optional.empty()); } - public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, - List qualifier, @Nullable Column column) { - this(exprId, name, dataType, nullable, qualifier, column, Optional.empty()); + public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, List qualifier, + @Nullable Column column) { + this(exprId, name, dataType, nullable, qualifier, column, Optional.empty(), Optional.empty()); } /** @@ -79,8 +82,8 @@ public SlotReference(ExprId exprId, String name, DataType dataType, boolean null * @param column the column which this slot come from * @param internalName the internalName of this slot */ - public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, - List qualifier, @Nullable Column column, Optional internalName) { + public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, List qualifier, + @Nullable Column column, Optional internalName, Optional relationId) { this.exprId = exprId; this.name = name; this.dataType = dataType; @@ -88,22 +91,27 @@ public SlotReference(ExprId exprId, String name, DataType dataType, boolean null this.nullable = nullable; this.column = column; this.internalName = internalName.isPresent() ? internalName : Optional.of(name); + this.relationId = relationId; } public static SlotReference of(String name, DataType type) { return new SlotReference(name, type); } - public static SlotReference fromColumn(Column column, List qualifier) { + public static SlotReference fromColumn(Column column, List qualifier, RelationId relationId) { DataType dataType = DataType.fromCatalogType(column.getType()); return new SlotReference(StatementScopeIdGenerator.newExprId(), column.getName(), dataType, - column.isAllowNull(), qualifier, column, Optional.empty()); + column.isAllowNull(), qualifier, column, Optional.empty(), Optional.ofNullable(relationId)); } - public static SlotReference fromColumn(Column column, String name, List qualifier) { + public static SlotReference fromColumn(Column column, String name, List qualifier, RelationId relationId) { DataType dataType = DataType.fromCatalogType(column.getType()); - return new SlotReference(StatementScopeIdGenerator.newExprId(), name, dataType, - column.isAllowNull(), qualifier, column, Optional.empty()); + return new SlotReference(StatementScopeIdGenerator.newExprId(), name, dataType, column.isAllowNull(), qualifier, + column, Optional.empty(), Optional.ofNullable(relationId)); + } + + public void changeExprId(ExprId id) { + this.exprId = id; } @Override @@ -188,6 +196,10 @@ public Optional getColumn() { return Optional.ofNullable(column); } + public Optional getRelationId() { + return relationId; + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitSlotReference(this, context); @@ -204,22 +216,22 @@ public SlotReference withNullable(boolean newNullable) { if (this.nullable == newNullable) { return this; } - return new SlotReference(exprId, name, dataType, newNullable, qualifier, column, internalName); + return new SlotReference(exprId, name, dataType, newNullable, qualifier, column, internalName, relationId); } @Override public SlotReference withQualifier(List qualifier) { - return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName); + return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, relationId); } @Override public SlotReference withName(String name) { - return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName); + return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, relationId); } @Override public SlotReference withExprId(ExprId exprId) { - return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName); + return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, relationId); } public boolean isVisible() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java index 513da0e93d91120..715e7ac6e3f4b95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java @@ -17,9 +17,16 @@ package org.apache.doris.nereids.trees.expressions.visitor; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.metadata.Predicates; +import org.apache.doris.nereids.util.ExpressionUtils; + +import java.util.ArrayList; +import java.util.List; /** * This is the factory for all ExpressionVisitor instance. @@ -54,4 +61,37 @@ public Boolean visitAggregateFunction(AggregateFunction aggregateFunction, Void return true; } } + + /** + * Split the expression to + * Should new instance when used. + */ + public static class PredicatesSpliter extends DefaultExpressionVisitor { + + private List equalPredicates = new ArrayList<>(); + private List rangePredicates = new ArrayList<>(); + private List residualPredicates = new ArrayList<>(); + + @Override + public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Void context) { + // TODO Smallest implement, complete later + if (comparisonPredicate instanceof EqualTo) { + Expression argument0 = comparisonPredicate.getArgument(0); + Expression argument1 = comparisonPredicate.getArgument(1); + if (argument0.isSlot() && argument1.isSlot()) { + equalPredicates.add(comparisonPredicate); + } else { + rangePredicates.add(comparisonPredicate); + } + } + return super.visit(comparisonPredicate, context); + } + + public Predicates.SplitPredicate getSplitPredicate() { + return Predicates.SplitPredicate.of( + equalPredicates.isEmpty() ? null : ExpressionUtils.and(equalPredicates), + rangePredicates.isEmpty() ? null : ExpressionUtils.and(rangePredicates), + residualPredicates.isEmpty() ? null : ExpressionUtils.and(residualPredicates)); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/EquivalenceClass.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/EquivalenceClass.java new file mode 100644 index 000000000000000..cc81ed21f9d305f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/EquivalenceClass.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.metadata; + +import org.apache.doris.nereids.rules.rewrite.mv.Mapping; +import org.apache.doris.nereids.rules.rewrite.mv.Mapping.IntPair; +import org.apache.doris.nereids.trees.expressions.SlotReference; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * EquivalenceClass + */ +public class EquivalenceClass { + + private final Map> equivalenceSlotMap = new LinkedHashMap<>(); + + public EquivalenceClass() { + } + + /** + * EquivalenceClass + */ + public void addEquivalenceClass(SlotReference slot0, SlotReference slot1) { + + Set slot0Sets = equivalenceSlotMap.get(slot0); + Set slot1Sets = equivalenceSlotMap.get(slot1); + if (slot0Sets != null && slot1Sets != null) { + // Both present, we need to merge + if (slot0Sets.size() < slot1Sets.size()) { + // We swap them to merge + Set tmp = slot1Sets; + slot1Sets = slot0Sets; + slot0Sets = tmp; + } + for (SlotReference newRef : slot1Sets) { + slot0Sets.add(newRef); + equivalenceSlotMap.put(newRef, slot0Sets); + } + } else if (slot0Sets != null) { + // p1 present, we need to merge into it + slot0Sets.add(slot1); + equivalenceSlotMap.put(slot1, slot0Sets); + } else if (slot1Sets != null) { + // p2 present, we need to merge into it + slot1Sets.add(slot0); + equivalenceSlotMap.put(slot0, slot1Sets); + } else { + // None are present, add to same equivalence class + Set equivalenceClass = new LinkedHashSet<>(); + equivalenceClass.add(slot0); + equivalenceClass.add(slot1); + equivalenceSlotMap.put(slot0, equivalenceClass); + equivalenceSlotMap.put(slot1, equivalenceClass); + } + } + + public Map> getEquivalenceSlotMap() { + return equivalenceSlotMap; + } + + public boolean isEmpty() { + return equivalenceSlotMap.isEmpty(); + } + + /** + * EquivalenceClass + */ + public List> getEquivalenceValues() { + List> values = new ArrayList<>(); + equivalenceSlotMap.values().forEach(each -> values.add(each)); + return values; + } + + /** + * EquivalenceClass + */ + public Mapping generateMapping(EquivalenceClass target) { + + List> sourceEquivalenceValues = this.getEquivalenceValues(); + List> targetEquivalenceValues = target.getEquivalenceValues(); + Mapping mapping = Mapping.of(sourceEquivalenceValues.size(), targetEquivalenceValues.size()); + + for (int i = 0; i < targetEquivalenceValues.size(); i++) { + boolean foundQueryEquivalenceClass = false; + final Set viewEquivalenceClass = targetEquivalenceValues.get(i); + for (int j = 0; j < sourceEquivalenceValues.size(); j++) { + final Set queryEquivalenceClass = sourceEquivalenceValues.get(j); + if (queryEquivalenceClass.containsAll(viewEquivalenceClass)) { + mapping.addMapping(IntPair.of(j, i)); + foundQueryEquivalenceClass = true; + break; + } + } // end for + if (!foundQueryEquivalenceClass) { + // Target equivalence class not found in source equivalence class + return null; + } + } + return mapping; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/PlanMetadataQuery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/PlanMetadataQuery.java new file mode 100644 index 000000000000000..2c2676c3f99ad8f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/PlanMetadataQuery.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.metadata; + +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitors; + +import com.google.common.collect.ImmutableSet; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * PlanMetadataQuery + */ +public class PlanMetadataQuery { + + private PlanMetadataQuery() { + } + + // Replace the slot in expression with the lineage identifier from specified + // baseTable sets or target table types + // Note: Maybe unnecessary + public static Expression shuttleExpressionWithLineage(Plan plan, Expression expression, + Set targetTypes, + Set tableIdentifiers) { + return null; + } + + public static List getTables(Plan plan, Set targetTypes) { + List relations = new ArrayList<>(); + PlanVisitors.TABLE_COLLECTOR_INSTANCE.visit(plan, relations); + return relations; + } + + public static Predicates getPredicates(Plan plan) { + List expressions = new ArrayList<>(); + plan.accept(PlanVisitors.PREDICATES_COLLECTOR_INSTANCE, expressions); + return Predicates.of(new HashSet<>(expressions), ImmutableSet.of()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/Predicates.java new file mode 100644 index 000000000000000..56042b832eb3512 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/Predicates.java @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.metadata; + +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors.PredicatesSpliter; +import org.apache.doris.nereids.util.ExpressionUtils; + +import java.util.Set; + +/** + * Predicates + * */ +public class Predicates { + + // Predicates that can be pulled up + private final Set pulledUpPredicates; + // Record the predicates that can not pulled up from outer join or other join + private final Set canNotPulledUpPredicates; + + public Predicates(Set pulledUpPredicates, Set canNotPulledUpPredicates) { + this.pulledUpPredicates = pulledUpPredicates; + this.canNotPulledUpPredicates = canNotPulledUpPredicates; + } + + public static Predicates of(Set pulledUpPredicates, + Set canNotPulledUpPredicates) { + return new Predicates(pulledUpPredicates, canNotPulledUpPredicates); + } + + public Set getPulledUpPredicates() { + return pulledUpPredicates; + } + + public Set getCanNotPulledUpPredicates() { + return canNotPulledUpPredicates; + } + + public Expression composedExpression() { + return ExpressionUtils.and(ExpressionUtils.and(pulledUpPredicates), + ExpressionUtils.and(canNotPulledUpPredicates)); + } + + /** + * SplitPredicate + * */ + public static SplitPredicate splitPredicates(Expression expression) { + PredicatesSpliter predicatesSplit = new PredicatesSpliter(); + expression.accept(predicatesSplit, null); + return predicatesSplit.getSplitPredicate(); + } + + /** + * SplitPredicate + * */ + public static final class SplitPredicate { + private final Expression equalPredicates; + private final Expression rangePredicates; + private final Expression residualPredicates; + + public SplitPredicate(Expression equalPredicates, Expression rangePredicates, Expression residualPredicates) { + this.equalPredicates = equalPredicates; + this.rangePredicates = rangePredicates; + this.residualPredicates = residualPredicates; + } + + public Expression getEqualPredicates() { + return equalPredicates; + } + + public Expression getRangePredicates() { + return rangePredicates; + } + + public Expression getResidualPredicates() { + return residualPredicates; + } + + public static SplitPredicate empty() { + return new SplitPredicate(null, null, null); + } + + /** + * SplitPredicate + * */ + public static SplitPredicate of(Expression equalPredicates, + Expression rangePredicates, + Expression residualPredicates) { + return new SplitPredicate(equalPredicates, rangePredicates, residualPredicates); + } + + /** + * isEmpty + * */ + public boolean isEmpty() { + return equalPredicates == null + && rangePredicates == null + && residualPredicates == null; + } + + public Expression composedExpression() { + return ExpressionUtils.and(equalPredicates, rangePredicates, residualPredicates); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index 4dd5121ef83e5f8..122d298ea29e57a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -86,7 +86,7 @@ public DatabaseIf getDatabase() throws AnalysisException { public List computeOutput() { return table.getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, qualified())) + .map(col -> SlotReference.fromColumn(col, qualified(), relationId)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 69166318989ddf6..227bdc1d19bb941 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -305,7 +305,7 @@ public List computeOutput() { if (cacheSlotWithSlotName.containsKey(Pair.of(selectedIndexId, col.getName()))) { return cacheSlotWithSlotName.get(Pair.of(selectedIndexId, col.getName())); } - Slot slot = SlotReference.fromColumn(col, qualified()); + Slot slot = SlotReference.fromColumn(col, qualified(), relationId); cacheSlotWithSlotName.put(Pair.of(selectedIndexId, col.getName()), slot); return slot; }).collect(ImmutableList.toImmutableList()); @@ -343,7 +343,7 @@ private Slot generateUniqueSlot(Column column, boolean isBaseIndex, long indexId if (cacheSlotWithSlotName.containsKey(Pair.of(indexId, name))) { return cacheSlotWithSlotName.get(Pair.of(indexId, name)); } - Slot slot = SlotReference.fromColumn(column, name, qualified()); + Slot slot = SlotReference.fromColumn(column, name, qualified(), relationId); cacheSlotWithSlotName.put(Pair.of(indexId, name), slot); return slot; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java index 4527ffa31626ca1..750357fa20eabcb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java @@ -98,7 +98,7 @@ public String toString() { public List computeOutput() { return function.getTable().getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, qualifier)) + .map(col -> SlotReference.fromColumn(col, qualifier, relationId)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java index 57e3b942212726b..67e65c302c82226 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java @@ -101,7 +101,7 @@ public DatabaseIf getDatabase() throws AnalysisException { public List computeOutput() { return table.getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, qualified())) + .map(col -> SlotReference.fromColumn(col, qualified(), relationId)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java index 955ea2f45da522d..9ba4098bb2c631e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java @@ -107,7 +107,7 @@ public String toString() { public List computeOutput() { return function.getTable().getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, ImmutableList.of())) + .map(col -> SlotReference.fromColumn(col, ImmutableList.of(), relationId)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitors.java new file mode 100644 index 000000000000000..adf3d91f612bc88 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitors.java @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.visitor; + +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; +import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.util.ExpressionUtils; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * This is the facade and factory for common plan visitor. + */ +public class PlanVisitors { + + public static final TableScanCollector TABLE_COLLECTOR_INSTANCE = new TableScanCollector(); + public static final PredicatesCollector PREDICATES_COLLECTOR_INSTANCE = new PredicatesCollector(); + public static final SlotReferenceCollector SLOT_REFERENCE_COLLECTOR = new SlotReferenceCollector(); + + /** + * Collect the table in plan + * Note: will not get table if table is eliminated by EmptyRelation in rewrite. + */ + public static class TableScanCollector extends DefaultPlanVisitor> { + + @Override + public Void visit(Plan plan, List collectedRelations) { + if (plan instanceof CatalogRelation) { + CatalogRelation catalogRelation = (CatalogRelation) plan; + collectedRelations.add(catalogRelation); + } + return super.visit(plan, collectedRelations); + } + + @Override + public Void visitGroupPlan(GroupPlan groupPlan, List context) { + Group group = groupPlan.getGroup(); + // TODO Should record the struct info on the group? + return group.getLogicalExpressions().get(0).getPlan().accept(this, context); + } + } + + /** + * Collect the predicates in plan + */ + public static class PredicatesCollector extends DefaultPlanVisitor> { + + @Override + public Void visitLogicalFilter(LogicalFilter filter, List context) { + Set conjuncts = filter.getConjuncts(); + if (conjuncts == null) { + return super.visit(filter, context); + } + for (Expression conjunct : conjuncts) { + context.addAll(ExpressionUtils.decomposeAnd(conjunct)); + } + return super.visit(filter, context); + } + + @Override + public Void visitLogicalJoin(LogicalJoin join, List context) { + List conjuncts = join.getHashJoinConjuncts(); + // TODO Check getOtherJoinConjuncts method predicates + if (conjuncts == null) { + return super.visit(join, context); + } + for (Expression conjunct : conjuncts) { + context.addAll(ExpressionUtils.decomposeAnd(conjunct)); + } + return super.visit(join, context); + } + + @Override + public Void visitGroupPlan(GroupPlan groupPlan, List context) { + Group group = groupPlan.getGroup(); + // TODO Should record the struct info on the group? + return group.getLogicalExpressions().get(0).getPlan().accept(this, context); + } + } + + /** + * SlotReferenceCollector + */ + public static class SlotReferenceCollector + extends DefaultPlanVisitor> { + @Override + public Void visit(Plan plan, Set collectedExpressions) { + List expressions = plan.getExpressions(); + if (expressions.isEmpty()) { + return super.visit(plan, collectedExpressions); + } + expressions.forEach(expression -> { + if (expression instanceof SlotReference && ((SlotReference) expression).getRelationId() + .isPresent()) { + collectedExpressions.add((SlotReference) expression); + } + }); + return super.visit(plan, collectedExpressions); + } + + @Override + public Void visitGroupPlan(GroupPlan groupPlan, Set context) { + Group group = groupPlan.getGroup(); + // TODO Should record the struct info on the group? + return group.getLogicalExpressions().get(0).getPlan().accept(this, context); + } + } + + /** + * GroupPlanRemover + */ + public static class GroupPlanRemover + extends DefaultPlanRewriter { + + @Override + public Plan visitGroupPlan(GroupPlan groupPlan, Void context) { + Group group = groupPlan.getGroup(); + return group.getLogicalExpressions().get(0).getPlan(); + } + } + + /** + * SlotReferenceReplacer + */ + public static class SlotReferenceReplacer + extends DefaultPlanVisitor> { + + public static final SlotReferenceReplacer INSTANCE = new SlotReferenceReplacer(); + + @Override + public Plan visit(Plan plan, Map mvScanToQueryMapping) { + List slots = plan.getOutput(); + if (slots.isEmpty()) { + return super.visit(plan, mvScanToQueryMapping); + } + slots.forEach(slot -> slot.accept(ExprReplacer.INSTANCE, mvScanToQueryMapping)); + return super.visit(plan, mvScanToQueryMapping); + } + + @Override + public Plan visitGroupPlan(GroupPlan groupPlan, Map mvScanToQueryMapping) { + Group group = groupPlan.getGroup(); + // TODO Should record the struct info on the group? + return group.getLogicalExpressions().get(0).getPlan().accept(this, mvScanToQueryMapping); + } + + /** + * ExprReplacer + */ + public static class ExprReplacer extends DefaultExpressionVisitor> { + public static final ExprReplacer INSTANCE = new ExprReplacer(); + + @Override + public Void visitSlotReference(SlotReference slot, Map context) { + Slot mappedSlot = context.get(slot); + if (mappedSlot != null) { + slot.changeExprId(mappedSlot.getExprId()); + } + return super.visit(slot, context); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 0c5faa209576667..d4bf3084abc00e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -42,12 +42,14 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -196,6 +198,64 @@ public static Expression combine(Class type, Collection decomposeAnd(Expression expression) { + List conjuncts = new ArrayList<>(); + if (expression == null) { + return conjuncts; + } + extractExpressionBy(expression, conjuncts, And.class); + return conjuncts; + } + + private static void extractExpressionBy(Expression target, + List expressions, + Class clazz) { + if (!(target instanceof CompoundPredicate)) { + expressions.add(target); + return; + } + CompoundPredicate compoundPredicate = (CompoundPredicate) target; + if (!compoundPredicate.getClass().isAssignableFrom(clazz)) { + expressions.add(compoundPredicate); + return; + } + extractExpressionBy(compoundPredicate.getArgument(0), expressions, clazz); + extractExpressionBy(compoundPredicate.getArgument(1), expressions, clazz); + } + + /** + * Returns a condition decomposed by OR. + */ + public static List decomposeOr(Expression expression) { + List disConjunct = new ArrayList<>(); + if (expression == null) { + return disConjunct; + } + extractExpressionBy(expression, disConjunct, Or.class); + return disConjunct; + } + + /** + * Given an expression, it will swap the table relation contained in its + * using the contents in the map. + */ + public static Expression swapTableRelation(Expression expression, + BiMap tableRelationMapping) { + return null; + } + + /** + * Given an expression, it will permute the column contained in its + * using the contents in the map. + */ + public static Expression permuteColumn(Expression expression, + BiMap columnMapping) { + return null; + } + /** * Choose the minimum slot from input parameter. */ diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewTest.java new file mode 100644 index 000000000000000..517810d3e1e72aa --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewTest.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.mv; + +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class MaterializedViewTest extends TestWithFeService { + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("mv_poc"); + useDatabase("mv_poc"); + + createTable("CREATE TABLE lineitem (\n" + + " l_shipdate DATE NOT NULL,\n" + + " l_orderkey bigint NOT NULL,\n" + + " l_linenumber int not null,\n" + + " l_partkey int NOT NULL,\n" + + " l_suppkey int not null,\n" + + " l_quantity decimal(15, 2) NOT NULL,\n" + + " l_extendedprice decimal(15, 2) NOT NULL,\n" + + " l_discount decimal(15, 2) NOT NULL,\n" + + " l_tax decimal(15, 2) NOT NULL,\n" + + " l_returnflag VARCHAR(1) NOT NULL,\n" + + " l_linestatus VARCHAR(1) NOT NULL,\n" + + " l_commitdate DATE NOT NULL,\n" + + " l_receiptdate DATE NOT NULL,\n" + + " l_shipinstruct VARCHAR(25) NOT NULL,\n" + + " l_shipmode VARCHAR(10) NOT NULL,\n" + + " l_comment VARCHAR(44) NOT NULL\n" + + ")ENGINE=OLAP\n" + + "DUPLICATE KEY(`l_shipdate`, `l_orderkey`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 32\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\",\n" + + " \"colocate_with\" = \"lineitem_orders\"\n" + + ");"); + + createTable("CREATE TABLE orders (\n" + + " o_orderkey bigint NOT NULL,\n" + + " o_orderdate DATE NOT NULL,\n" + + " o_custkey int NOT NULL,\n" + + " o_orderstatus VARCHAR(1) NOT NULL,\n" + + " o_totalprice decimal(15, 2) NOT NULL,\n" + + " o_orderpriority VARCHAR(15) NOT NULL,\n" + + " o_clerk VARCHAR(15) NOT NULL,\n" + + " o_shippriority int NOT NULL,\n" + + " o_comment VARCHAR(79) NOT NULL\n" + + ")ENGINE=OLAP\n" + + "DUPLICATE KEY(`o_orderkey`, `o_orderdate`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 32\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\",\n" + + " \"colocate_with\" = \"lineitem_orders\"\n" + + ");"); + + createTableAsSelect("CREATE TABLE mv PROPERTIES(\"replication_num\" = \"1\") " + + "as select l_shipdate, l_linenumber from lineitem inner join orders on l_orderkey = o_orderkey;"); + } + + @Test + public void testInnerJoin() { + + connectContext.getSessionVariable().enableNereidsTimeout = false; + connectContext.getSessionVariable().enableDPHypOptimizer = true; + // query only l_orderkey from join(lineitem, orders) will output l_orderkey and o_orderkey + // PoC just use lineitem's field + PlanChecker.from(connectContext) + .checkMVRewrite( + "select l_shipdate from lineitem inner join orders on l_orderkey = o_orderkey", + "select l_shipdate, l_linenumber from lineitem inner join orders on l_orderkey = o_orderkey", + "select l_shipdate, l_linenumber from mv", + (queryPlanner, mvPlanner) -> { + PhysicalPlan physicalPlan = queryPlanner.getPhysicalPlan(); + Assertions.assertTrue( + ((PhysicalResultSink) physicalPlan).toJson().toString().contains("mv")); + } + ); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 9274fdd0abd4dca..6d5a8b0ba843763 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -50,6 +50,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.rules.rewrite.mv.MaterializationContext; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; @@ -73,6 +74,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Supplier; @@ -167,7 +169,7 @@ public PlanChecker applyTopDown(RuleFactory ruleFactory) { public PlanChecker applyTopDown(List rule) { Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext, - ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteTopDownJob::new, true))) + ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteTopDownJob::new, true))) .execute(); cascadesContext.toMemo(); MemoValidator.validate(cascadesContext.getMemo()); @@ -579,6 +581,47 @@ public PlanChecker checkPlannerResult(String sql) { }); } + public PlanChecker checkMVRewrite(String sql, String mvSql, String mvScanSql, + BiConsumer consumer) { + + PhysicalProperties physicalProperties = NereidsPlanner.buildInitRequireProperties(); + // Mock materialized view define sql + LogicalPlan mvUnboundPlan = new NereidsParser().parseSingle(mvSql); + NereidsPlanner mvPlanner = new NereidsPlanner( + new StatementContext(connectContext, new OriginStatement(mvSql, 0))); + mvPlanner.plan(mvUnboundPlan, physicalProperties, ExplainLevel.ALL_PLAN); + + // mock the mv scan, this should be from materializedView + LogicalPlan mvScanUnboundPlan = new NereidsParser().parseSingle(mvScanSql); + NereidsPlanner mvScanPlanner = new NereidsPlanner( + new StatementContext(connectContext, new OriginStatement(mvScanSql, 0))); + mvScanPlanner.plan(mvScanUnboundPlan, physicalProperties, ExplainLevel.ALL_PLAN); + + // mock the mv context and query rewrite, should call actual materialized view instead + LogicalPlan queryUnboundPlan = new NereidsParser().parseSingle(sql); + StatementContext queryStmtContext = new StatementContext(connectContext, new OriginStatement(sql, 0)); + NereidsPlanner queryPlanner = new NereidsPlanner(queryStmtContext); + CascadesContext queryCascadesContext = + CascadesContext.initContext(queryStmtContext, queryUnboundPlan, physicalProperties); + if (queryStmtContext.getConnectContext().getTables() != null) { + queryCascadesContext.setTables(queryStmtContext.getConnectContext().getTables()); + } + + MaterializationContext mvContext = new MaterializationContext( + mvPlanner.getRewrittenPlan(), + queryCascadesContext, + ImmutableList.of(), + ImmutableList.of(), + mvScanPlanner.getRewrittenPlan() + ); + queryCascadesContext.addMaterializationContext(mvContext); + queryPlanner.setCascadesContext(queryCascadesContext); + + queryPlanner.plan(LogicalPlanAdapter.of(queryUnboundPlan)); + consumer.accept(queryPlanner, mvPlanner); + return this; + } + public CascadesContext getCascadesContext() { return cascadesContext; }