From e9a0f1b60ff24e8df1bfaef3ae8f12c52043d526 Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 2 Nov 2023 21:45:20 +0800 Subject: [PATCH] [feature](nereids) Support inner join query rewrite by materialized view --- .../java/org/apache/doris/catalog/MTMV.java | 11 + .../MaterializedViewException.java} | 32 +- .../apache/doris/mtmv/MTMVCacheManager.java | 6 + .../java/org/apache/doris/mtmv/MVCache.java | 62 ++++ .../apache/doris/nereids/NereidsPlanner.java | 14 +- .../org/apache/doris/nereids/PlannerHook.java | 31 ++ .../cascades/OptimizeGroupExpressionJob.java | 6 +- .../jobs/joinorder/hypergraph/HyperGraph.java | 3 +- .../org/apache/doris/nereids/memo/Group.java | 14 +- .../mv/AbstractMaterializedViewJoinRule.java | 47 ++- .../mv/AbstractMaterializedViewRule.java | 232 ++++++++++--- .../exploration/mv/EquivalenceClass.java | 54 ++- .../mv/InitMaterializationContextHook.java | 111 +++++++ .../mv/MaterializationContext.java | 109 +++++- .../rules/exploration/mv/Predicates.java | 95 +++--- .../exploration/mv/PredicatesSplitter.java | 111 +++++++ .../rules/exploration/mv/RelationMapping.java | 63 ---- .../rules/exploration/mv/StructInfo.java | 210 ++++++++++-- .../mapping/EquivalenceClassSetMapping.java | 66 ++++ .../mv/mapping/ExpressionMapping.java | 98 ++++++ .../exploration/mv/{ => mapping}/Mapping.java | 61 ++-- .../mv/mapping/RelationMapping.java | 126 +++++++ .../exploration/mv/mapping/SlotMapping.java | 98 ++++++ .../apache/doris/nereids/trees/TreeNode.java | 14 + .../nereids/trees/expressions/ExprId.java | 10 + .../visitor/ExpressionVisitors.java | 49 --- .../trees/plans/commands/ExplainCommand.java | 4 + .../visitor/ExpressionLineageReplacer.java | 160 +++++++++ .../trees/plans/visitor/TableCollector.java | 2 +- .../doris/nereids/util/ExpressionUtils.java | 106 ++++-- .../apache/doris/planner/OriginalPlanner.java | 4 + .../org/apache/doris/planner/Planner.java | 3 + .../org/apache/doris/qe/StmtExecutor.java | 5 +- .../rules/exploration/mv/MappingTest.java | 309 ++++++++++++++++++ .../ExpressionRewriteTestHelper.java | 2 +- .../expression/PredicatesSplitterTest.java | 109 ++++++ .../nereids/util/ExpressionUtilsTest.java | 106 +++++- .../doris/nereids/util/PlanChecker.java | 2 +- .../data/nereids_rules_p0/mv/inner_join.out | 9 + .../nereids_rules_p0/mv/inner_join.groovy | 163 +++++++++ 40 files changed, 2372 insertions(+), 345 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/{nereids/rules/exploration/mv/SlotMapping.java => common/MaterializedViewException.java} (50%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/{ => mapping}/Mapping.java (69%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java create mode 100644 regression-test/data/nereids_rules_p0/mv/inner_join.out create mode 100644 regression-test/suites/nereids_rules_p0/mv/inner_join.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 0c59f56db5c8317..af0691d94fbecc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -30,6 +30,7 @@ import org.apache.doris.mtmv.MTMVRefreshInfo; import org.apache.doris.mtmv.MTMVRelation; import org.apache.doris.mtmv.MTMVStatus; +import org.apache.doris.mtmv.MVCache; import org.apache.doris.persist.gson.GsonUtils; import com.google.gson.annotations.SerializedName; @@ -61,6 +62,8 @@ public class MTMV extends OlapTable { private Map mvProperties; @SerializedName("r") private MTMVRelation relation; + // Should update after every fresh + private MVCache mvCache; // For deserialization public MTMV() { @@ -116,6 +119,14 @@ public MTMVRelation getRelation() { return relation; } + public MVCache getMvCache() { + return mvCache; + } + + public void setMvCache(MVCache mvCache) { + this.mvCache = mvCache; + } + public MTMVRefreshInfo alterRefreshInfo(MTMVRefreshInfo newRefreshInfo) { return refreshInfo.updateNotNull(newRefreshInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java b/fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java similarity index 50% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java rename to fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java index 7c50d79c6a23c27..27f2adb78b61afb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java @@ -15,35 +15,31 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.rules.exploration.mv; - -import com.google.common.collect.BiMap; +package org.apache.doris.common; /** - * SlotMapping, this is open generated from relationMapping + * MaterializedViewException */ -public class SlotMapping extends Mapping { - - private final BiMap relationSlotMap; +public class MaterializedViewException extends UserException { - public SlotMapping(BiMap relationSlotMap) { - this.relationSlotMap = relationSlotMap; + public MaterializedViewException(String msg, Throwable cause) { + super(msg, cause); } - public BiMap getRelationSlotMap() { - return relationSlotMap; + public MaterializedViewException(Throwable cause) { + super(cause); } - public SlotMapping inverse() { - return SlotMapping.of(relationSlotMap.inverse()); + public MaterializedViewException(String msg, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(msg, cause, enableSuppression, writableStackTrace); } - public static SlotMapping of(BiMap relationSlotMap) { - return new SlotMapping(relationSlotMap); + public MaterializedViewException(String msg) { + super(msg); } - public static SlotMapping generate(RelationMapping relationMapping) { - // TODO implement - return SlotMapping.of(null); + public MaterializedViewException(InternalErrorCode errCode, String msg) { + super(errCode, msg); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCacheManager.java index 847f85b90749958..950fc79ef7da894 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCacheManager.java @@ -48,6 +48,7 @@ import org.apache.doris.persist.AlterMTMV; import org.apache.doris.qe.ConnectContext; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; @@ -70,6 +71,11 @@ public Set getMtmvsByBaseTable(BaseTableInfo table) { return tableMTMVs.get(table); } + // TODO Implement the method which getting materialized view by tables + public List getAvailableMaterializedView(List tables) { + return ImmutableList.of(); + } + public boolean isAvailableMTMV(MTMV mtmv, ConnectContext ctx) throws AnalysisException, DdlException { // check session variable if enable rewrite if (!ctx.getSessionVariable().isEnableMvRewrite()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java new file mode 100644 index 000000000000000..7490566a7f475c3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; + +/**The cache for materialized view cache */ +public class MVCache { + + // the materialized view plan which should be optimized by the same rules to query + private final Plan logicalPlan; + // this should be shuttle expression with lineage + private final List mvOutputExpressions; + // the context when parse, analyze, optimize the mv logical plan + private final CascadesContext context; + + public MVCache(MTMV materializedView, Plan logicalPlan, List mvOutputExpressions, + CascadesContext context) { + this.logicalPlan = logicalPlan; + this.mvOutputExpressions = mvOutputExpressions; + this.context = context; + } + + public Plan getLogicalPlan() { + return logicalPlan; + } + + public List getMvOutputExpressions() { + return mvOutputExpressions; + } + + public MVCache(Plan logicalPlan, List mvOutputExpressions, CascadesContext context) { + this.logicalPlan = logicalPlan; + this.mvOutputExpressions = mvOutputExpressions; + this.context = context; + } + + public static MVCache from(MTMV mtmv, Plan logicalPlan, + List mvOutputExpressions, CascadesContext context) { + return new MVCache(mtmv, logicalPlan, mvOutputExpressions, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index d50d2d445c97ca5..fa1e00c37042ac7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -89,6 +89,7 @@ public class NereidsPlanner extends Planner { private PhysicalPlan physicalPlan; // The cost of optimized plan private double cost = 0; + private List hooks = new ArrayList<>(); public NereidsPlanner(StatementContext statementContext) { this.statementContext = statementContext; @@ -261,15 +262,12 @@ private void initCascadesContext(LogicalPlan plan, PhysicalProperties requirePro if (statementContext.getConnectContext().getTables() != null) { cascadesContext.setTables(statementContext.getConnectContext().getTables()); } - if (statementContext.getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite()) { - // TODO Pre handle materialized view to materializationContext and - // call cascadesContext.addMaterializationContext() to add it - } } private void analyze() { LOG.info("Start analyze plan"); cascadesContext.newAnalyzer().analyze(); + getHooks().forEach(hook -> hook.afterAnalyze(this)); NereidsTracer.logImportantTime("EndAnalyzePlan"); LOG.info("End analyze plan"); } @@ -526,4 +524,12 @@ public Plan getOptimizedPlan() { public PhysicalPlan getPhysicalPlan() { return physicalPlan; } + + public List getHooks() { + return hooks; + } + + public void addHook(PlannerHook hook) { + this.hooks.add(hook); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java new file mode 100644 index 000000000000000..0fb95a28e5582c8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids; + +/** + * optimize plan process has some phase, such as analyze, rewrite, optimize, generate physical plan + * and so on, this hook give a chance to do something in the planning process. + * For example: after analyze plan when query or explain, we should generate materialization context. + */ +public interface PlannerHook { + default void beforeAnalyze(NereidsPlanner planner) { + } + + default void afterAnalyze(NereidsPlanner planner) { + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java index 08b7731fd0915dd..72426f0fa15dc53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java @@ -22,7 +22,9 @@ import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.qe.ConnectContext; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -46,7 +48,9 @@ public void execute() { countJobExecutionTimesOfGroupExpressions(groupExpression); List implementationRules = getRuleSet().getImplementationRules(); List explorationRules = getExplorationRules(); - if (context.getCascadesContext().getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite()) { + ConnectContext connectContext = context.getCascadesContext().getConnectContext(); + if (connectContext.getSessionVariable().isEnableMaterializedViewRewrite()) { + explorationRules = new ArrayList<>(explorationRules); explorationRules.addAll(getRuleSet().getMaterializedViewRules()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java index 3ffd159e14b5dbe..e8e5acd2cddeca3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java @@ -431,14 +431,13 @@ private void addStructInfo(HyperGraph other) { private Pair buildStructInfo(Plan plan) { if (plan instanceof GroupPlan) { Group group = ((GroupPlan) plan).getGroup(); - buildStructInfo(group.getLogicalExpressions().get(0).getPlan()); List childGraphs = ((GroupPlan) plan).getGroup().getHyperGraphs(); if (childGraphs.size() != 0) { int idx = addStructInfoNode(childGraphs); return Pair.of(new BitSet(), LongBitmap.newBitmap(idx)); } GroupExpression groupExpression = group.getLogicalExpressions().get(0); - buildStructInfo(groupExpression.getPlan() + return buildStructInfo(groupExpression.getPlan() .withChildren( groupExpression.children().stream().map(GroupPlan::new).collect(Collectors.toList()))); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index 3f20fcc391590ab..2a0e83c72a4df18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -76,7 +76,7 @@ public class Group { private int chosenGroupExpressionId = -1; - private Optional structInfo = Optional.empty(); + private List structInfos = new ArrayList<>(); /** * Constructor for Group. @@ -541,11 +541,15 @@ public String treeString() { return TreeStringUtils.treeString(this, toString, getChildren, getExtraPlans, displayExtraPlan); } - public Optional getStructInfo() { - return structInfo; + public List getStructInfos() { + return structInfos; } - public void setStructInfo(StructInfo structInfo) { - this.structInfo = Optional.ofNullable(structInfo); + public void addStructInfo(StructInfo structInfo) { + this.structInfos.add(structInfo); + } + + public void addStructInfo(List structInfos) { + this.structInfos.addAll(structInfos); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 17df9ef88ed77e6..4bd21474cb79da4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -17,10 +17,21 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.Edge; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.AbstractNode; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.util.ExpressionUtils; +import com.google.common.collect.Sets; + +import java.util.HashSet; import java.util.List; /** @@ -33,21 +44,28 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInfo, StructInfo viewStructInfo, - RelationMapping queryToViewTableMappings, - Plan tempRewritedPlan) { + SlotMapping queryToViewSlotMappings, + Plan tempRewritedPlan, + MaterializationContext materializationContext) { + List queryShuttleExpression = ExpressionUtils.shuttleExpressionWithLineage( + queryStructInfo.getExpressions(), + queryStructInfo.getOriginalPlan()); // Rewrite top projects, represent the query projects by view List expressions = rewriteExpression( - queryStructInfo.getExpressions(), - queryStructInfo, - viewStructInfo, - queryToViewTableMappings, + queryShuttleExpression, + materializationContext.getViewExpressionIndexMapping(), + queryToViewSlotMappings, tempRewritedPlan ); // Can not rewrite, bail out if (expressions == null) { return null; } + if (queryStructInfo.getOriginalPlan().getGroupExpression().isPresent()) { + materializationContext.addMatchedGroup( + queryStructInfo.getOriginalPlan().getGroupExpression().get().getOwnerGroup().getGroupId()); + } return new LogicalProject<>(expressions, tempRewritedPlan); } @@ -56,7 +74,20 @@ protected Plan rewriteQueryByView(MatchMode matchMode, // join condition should be slot reference equals currently @Override protected boolean checkPattern(StructInfo structInfo) { - // TODO Should get struct info from hyper graph and check - return false; + HyperGraph hyperGraph = structInfo.getHyperGraph(); + HashSet requiredJoinType = Sets.newHashSet(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN); + for (AbstractNode node : hyperGraph.getNodes()) { + StructInfoNode structInfoNode = (StructInfoNode) node; + if (!structInfoNode.getPlan().accept(StructInfo.JOIN_PATTERN_CHECKER, + requiredJoinType)) { + return false; + } + for (Edge edge : hyperGraph.getEdges()) { + if (!edge.getJoin().accept(StructInfo.JOIN_PATTERN_CHECKER, requiredJoinType)) { + return false; + } + } + } + return true; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index c52acdb3fdada1f..d4eae8a1b646062 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -19,21 +19,30 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.memo.Group; -import org.apache.doris.nereids.rules.exploration.mv.Mapping.ExpressionIndexMapping; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; +import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.util.ExpressionUtils; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -51,15 +60,28 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { if (materializationContexts.isEmpty()) { return rewriteResults; } - StructInfo queryStructInfo = extractStructInfo(queryPlan, cascadesContext); - // Check query queryPlan + + List queryStructInfos = extractStructInfo(queryPlan, cascadesContext); + // TODO Just Check query queryPlan firstly, support multi later. + StructInfo queryStructInfo = queryStructInfos.get(0); if (!checkPattern(queryStructInfo)) { return rewriteResults; } for (MaterializationContext materializationContext : materializationContexts) { - Plan mvPlan = materializationContext.getMvPlan(); - StructInfo viewStructInfo = extractStructInfo(mvPlan, cascadesContext); + // already rewrite, bail out + if (queryPlan.getGroupExpression().isPresent() + && materializationContext.alreadyRewrite( + queryPlan.getGroupExpression().get().getOwnerGroup().getGroupId())) { + continue; + } + Plan mvPlan = materializationContext.getMtmv().getMvCache().getLogicalPlan(); + List viewStructInfos = extractStructInfo(mvPlan, cascadesContext); + if (viewStructInfos.size() > 1) { + // view struct info should only have one + return rewriteResults; + } + StructInfo viewStructInfo = viewStructInfos.get(0); if (!checkPattern(viewStructInfo)) { continue; } @@ -67,29 +89,35 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { continue; } MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations()); - if (MatchMode.NOT_MATCH == matchMode) { + if (MatchMode.COMPLETE != matchMode) { continue; } List queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(), viewStructInfo.getRelations()); + if (queryToViewTableMappings == null) { + return rewriteResults; + } for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { + SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); + if (queryToViewSlotMapping == null) { + continue; + } SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo, - queryToViewTableMapping); + queryToViewSlotMapping); // Can not compensate, bail out - if (compensatePredicates == null || compensatePredicates.isEmpty()) { + if (compensatePredicates.isEmpty()) { continue; } Plan rewritedPlan; - Plan mvScan = materializationContext.getScanPlan(); + Plan mvScan = materializationContext.getMvScanPlan(); if (compensatePredicates.isAlwaysTrue()) { rewritedPlan = mvScan; } else { // Try to rewrite compensate predicates by using mv scan List rewriteCompensatePredicates = rewriteExpression( compensatePredicates.toList(), - queryStructInfo, - viewStructInfo, - queryToViewTableMapping, + materializationContext.getViewExpressionIndexMapping(), + queryToViewSlotMapping, mvScan); if (rewriteCompensatePredicates.isEmpty()) { continue; @@ -97,8 +125,12 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { rewritedPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan); } // Rewrite query by view - rewritedPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo, - queryToViewTableMapping, rewritedPlan); + rewritedPlan = rewriteQueryByView(matchMode, + queryStructInfo, + viewStructInfo, + queryToViewSlotMapping, + rewritedPlan, + materializationContext); if (rewritedPlan == null) { continue; } @@ -108,22 +140,26 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { return rewriteResults; } - /**Rewrite query by view, for aggregate or join rewriting should be different inherit class implementation*/ + /** + * Rewrite query by view, for aggregate or join rewriting should be different inherit class implementation + */ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInfo, StructInfo viewStructInfo, - RelationMapping queryToViewTableMappings, - Plan tempRewritedPlan) { + SlotMapping queryToViewSlotMappings, + Plan tempRewritedPlan, + MaterializationContext materializationContext) { return tempRewritedPlan; } - /**Use target output expression to represent the source expression*/ - protected List rewriteExpression(List sourceExpressions, - StructInfo sourceStructInfo, - StructInfo targetStructInfo, - RelationMapping sourceToTargetMapping, + /** + * Use target output expression to represent the source expression + */ + protected List rewriteExpression( + List sourceExpressions, + ExpressionMapping expressionMapping, + SlotMapping sourceToTargetMapping, Plan targetScanNode) { - // TODO represent the sourceExpressions by using target scan node // Firstly, rewrite the target plan output expression using query with inverse mapping // then try to use the mv expression to represent the query. if any of source expressions // can not be represented by mv, return null @@ -136,17 +172,32 @@ protected List rewriteExpression(List sou // transform source to: // project(slot 2, 1) // target - List targetTopExpressions = targetStructInfo.getExpressions(); - List shuttledTargetExpressions = ExpressionUtils.shuttleExpressionWithLineage( - targetTopExpressions, targetStructInfo.getOriginalPlan(), Sets.newHashSet(), Sets.newHashSet()); - SlotMapping sourceToTargetSlotMapping = SlotMapping.generate(sourceToTargetMapping); - // mv sql plan expressions transform to query based - List queryBasedExpressions = ExpressionUtils.permute(shuttledTargetExpressions, - sourceToTargetSlotMapping.inverse()); - // mv sql query based expression and index mapping - ExpressionIndexMapping.generate(queryBasedExpressions); - // TODO visit source expression and replace the expression with expressionIndexMapping - return ImmutableList.of(); + // generate mvSql to mvScan expressionMapping, and change mv sql expression to query based + ExpressionMapping expressionMappingKeySourceBased = + expressionMapping.keyPermute(sourceToTargetMapping.inverse()); + List> flattenExpressionMap = + expressionMappingKeySourceBased.flattenMap(); + // view to view scan expression is 1:1 so get first element + Map mvSqlToMvScanMappingQueryBased = flattenExpressionMap.get(0); + + List rewrittenExpressions = new ArrayList<>(); + for (Expression expressionToRewrite : sourceExpressions) { + if (expressionToRewrite instanceof BooleanLiteral + && ((BooleanLiteral) expressionToRewrite).getValue()) { + continue; + } + final Set slotsToRewrite = + expressionToRewrite.collectToSet(expression -> expression instanceof Slot); + Expression replacedExpression = ExpressionUtils.replace(expressionToRewrite, + mvSqlToMvScanMappingQueryBased, + true); + if (replacedExpression.anyMatch(slotsToRewrite::contains)) { + // if contains any slot to rewrite, which means can not be rewritten by target, bail out + return null; + } + rewrittenExpressions.add((NamedExpression) replacedExpression); + } + return rewrittenExpressions; } /** @@ -158,21 +209,101 @@ protected List rewriteExpression(List sou protected SplitPredicate predicatesCompensate( StructInfo queryStructInfo, StructInfo viewStructInfo, - RelationMapping queryToViewTableMapping + SlotMapping queryToViewSlotMapping ) { - // TODO Equal predicate compensate EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass(); EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass(); + // viewEquivalenceClass to query based + Map viewToQuerySlotMapping = queryToViewSlotMapping.inverse() + .toSlotReferenceMap(); + EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMapping); + final List equalCompensateConjunctions = new ArrayList<>(); + if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) { + equalCompensateConjunctions.add(BooleanLiteral.of(true)); + } if (queryEquivalenceClass.isEmpty() && !viewEquivalenceClass.isEmpty()) { - return null; + return SplitPredicate.empty(); + } + EquivalenceClassSetMapping queryToViewEquivalenceMapping = + EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased); + // can not map all target equivalence class, can not compensate + if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size() + < viewEquivalenceClass.getEquivalenceSetList().size()) { + return SplitPredicate.empty(); } - // TODO range predicates and residual predicates compensate - return SplitPredicate.empty(); + // do equal compensate + Set> mappedQueryEquivalenceSet = + queryToViewEquivalenceMapping.getEquivalenceClassSetMap().keySet(); + queryEquivalenceClass.getEquivalenceSetList().forEach( + queryEquivalenceSet -> { + // compensate the equivalence in query but not in view + if (!mappedQueryEquivalenceSet.contains(queryEquivalenceSet)) { + Iterator iterator = queryEquivalenceSet.iterator(); + SlotReference first = iterator.next(); + while (iterator.hasNext()) { + Expression equals = new EqualTo(first, iterator.next()); + equalCompensateConjunctions.add(equals); + } + } else { + // compensate the equivalence both in query and view, but query has more equivalence + Set viewEquivalenceSet = + queryToViewEquivalenceMapping.getEquivalenceClassSetMap().get(queryEquivalenceSet); + Set copiedQueryEquivalenceSet = new HashSet<>(queryEquivalenceSet); + copiedQueryEquivalenceSet.removeAll(viewEquivalenceSet); + SlotReference first = viewEquivalenceSet.iterator().next(); + for (SlotReference slotReference : copiedQueryEquivalenceSet) { + Expression equals = new EqualTo(first, slotReference); + equalCompensateConjunctions.add(equals); + } + } + } + ); + // TODO range predicates and residual predicates compensate, Simplify implementation. + SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate(); + SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate(); + + // range compensate + List rangeCompensate = new ArrayList<>(); + Expression queryRangePredicate = querySplitPredicate.getRangePredicate(); + Expression viewRangePredicate = viewSplitPredicate.getRangePredicate(); + Expression viewRangePredicateQueryBased = + ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping); + + Set queryRangeSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate)); + Set viewRangeQueryBasedSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased)); + if (!queryRangeSet.containsAll(viewRangeQueryBasedSet)) { + return SplitPredicate.empty(); + } + queryRangeSet.removeAll(viewRangeQueryBasedSet); + rangeCompensate.addAll(queryRangeSet); + + // residual compensate + List residualCompensate = new ArrayList<>(); + Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate(); + Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate(); + Expression viewResidualPredicateQueryBased = + ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping); + Set queryResidualSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate)); + Set viewResidualQueryBasedSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased)); + if (!queryResidualSet.containsAll(viewResidualQueryBasedSet)) { + return SplitPredicate.empty(); + } + queryResidualSet.removeAll(viewResidualQueryBasedSet); + residualCompensate.addAll(queryResidualSet); + + return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions), + rangeCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(rangeCompensate), + residualCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(residualCompensate)); } /** * Decide the match mode + * * @see MatchMode */ private MatchMode decideMatchMode(List queryRelations, List viewRelations) { @@ -202,20 +333,17 @@ private MatchMode decideMatchMode(List queryRelations, List extractStructInfo(Plan plan, CascadesContext cascadesContext) { if (plan.getGroupExpression().isPresent() - && plan.getGroupExpression().get().getOwnerGroup().getStructInfo().isPresent()) { - Group belongGroup = plan.getGroupExpression().get().getOwnerGroup(); - return belongGroup.getStructInfo().get(); + && !plan.getGroupExpression().get().getOwnerGroup().getStructInfos().isEmpty()) { + return plan.getGroupExpression().get().getOwnerGroup().getStructInfos(); } else { - // TODO build graph from plan and extract struct from graph and set to group if exist - // Should get structInfo from hyper graph and add into current group - StructInfo structInfo = StructInfo.of(plan); + // build struct info and add them to current group + List structInfos = StructInfo.of(plan); if (plan.getGroupExpression().isPresent()) { - plan.getGroupExpression().get().getOwnerGroup().setStructInfo(structInfo); + plan.getGroupExpression().get().getOwnerGroup().addStructInfo(structInfos); } - return structInfo; + return structInfos; } } @@ -226,7 +354,7 @@ protected boolean checkPattern(StructInfo structInfo) { if (structInfo.getRelations().isEmpty()) { return false; } - return false; + return true; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java index d140582aa645237..6c3d447baf4fb4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java @@ -20,6 +20,8 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -31,11 +33,16 @@ */ public class EquivalenceClass { - private final Map> equivalenceSlotMap = new LinkedHashMap<>(); + private Map> equivalenceSlotMap = new LinkedHashMap<>(); + private List> equivalenceSlotList; public EquivalenceClass() { } + public EquivalenceClass(Map> equivalenceSlotMap) { + this.equivalenceSlotMap = equivalenceSlotMap; + } + /** * EquivalenceClass */ @@ -81,12 +88,47 @@ public boolean isEmpty() { return equivalenceSlotMap.isEmpty(); } + /**EquivalenceClass permute*/ + public EquivalenceClass permute(Map mapping) { + + Map> permutedEquivalenceSlotMap = new HashMap<>(); + for (Map.Entry> slotReferenceSetEntry : equivalenceSlotMap.entrySet()) { + SlotReference mappedSlotReferenceKey = mapping.get(slotReferenceSetEntry.getKey()); + if (mappedSlotReferenceKey == null) { + // can not permute then need to return null + return null; + } + Set equivalenceValueSet = slotReferenceSetEntry.getValue(); + final Set mappedSlotReferenceSet = new HashSet<>(); + for (SlotReference target : equivalenceValueSet) { + SlotReference mappedSlotReferenceValue = mapping.get(target); + if (mappedSlotReferenceValue == null) { + return null; + } + mappedSlotReferenceSet.add(mappedSlotReferenceValue); + } + permutedEquivalenceSlotMap.put(mappedSlotReferenceKey, mappedSlotReferenceSet); + } + return new EquivalenceClass(permutedEquivalenceSlotMap); + } + /** - * EquivalenceClass + * Return the list of equivalence set, remove duplicate */ - public List> getEquivalenceValues() { - List> values = new ArrayList<>(); - equivalenceSlotMap.values().forEach(each -> values.add(each)); - return values; + public List> getEquivalenceSetList() { + + if (equivalenceSlotList != null) { + return equivalenceSlotList; + } + List> equivalenceSets = new ArrayList<>(); + Set> visited = new HashSet<>(); + equivalenceSlotMap.values().forEach(slotSet -> { + if (!visited.contains(slotSet)) { + equivalenceSets.add(slotSet); + } + visited.add(slotSet); + }); + this.equivalenceSlotList = equivalenceSets; + return this.equivalenceSlotList; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java new file mode 100644 index 000000000000000..24dad225e999f13 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVCacheManager; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.PlannerHook; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.visitor.TableCollector; +import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** If enable query rewrite with mv, should init materialization context after analyze*/ +public class InitMaterializationContextHook implements PlannerHook { + + public static final Logger LOG = LogManager.getLogger(InitMaterializationContextHook.class); + public static final InitMaterializationContextHook INSTANCE = new InitMaterializationContextHook(); + + @Override + public void afterAnalyze(NereidsPlanner planner) { + initMaterializationContext(planner.getCascadesContext()); + } + + private void initMaterializationContext(CascadesContext cascadesContext) { + + Plan rewritePlan = cascadesContext.getRewritePlan(); + TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet()); + rewritePlan.accept(TableCollector.INSTANCE, collectorContext); + List collectedTables = collectorContext.getCollectedTables(); + if (collectedTables.isEmpty()) { + return; + } + List baseTableUsed = + collectedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList()); + // TODO the logic should be move to MTMVCacheManager later when getAvailableMaterializedView is ready in + // MV Cache manager + Env env = cascadesContext.getConnectContext().getEnv(); + MTMVCacheManager cacheManager = env.getMtmvService().getCacheManager(); + Set materializedViews = new HashSet<>(); + for (BaseTableInfo baseTableInfo : baseTableUsed) { + Set mtmvsByBaseTable = cacheManager.getMtmvsByBaseTable(baseTableInfo); + if (mtmvsByBaseTable == null || mtmvsByBaseTable.isEmpty()) { + continue; + } + materializedViews.addAll(mtmvsByBaseTable); + } + if (materializedViews.isEmpty()) { + return; + } + materializedViews.forEach(mvBaseTableInfo -> { + try { + MTMV materializedView = (MTMV) Env.getCurrentInternalCatalog() + .getDbOrMetaException(mvBaseTableInfo.getDbId()) + .getTableOrMetaException(mvBaseTableInfo.getTableId(), TableType.MATERIALIZED_VIEW); + + String qualifiedName = materializedView.getQualifiedName(); + // generate outside, maybe add partition filter in the future + Plan mvScan = new LogicalOlapScan(cascadesContext.getStatementContext().getNextRelationId(), + (OlapTable) materializedView, + ImmutableList.of(qualifiedName), + Lists.newArrayList(materializedView.getId()), + Lists.newArrayList(), + Optional.empty()); + List mvProjects = mvScan.getOutput().stream().map(NamedExpression.class::cast) + .collect(Collectors.toList()); + mvScan = new LogicalProject(mvProjects, mvScan); + cascadesContext.addMaterializationContext( + MaterializationContext.fromMaterializedView(materializedView, mvScan, cascadesContext)); + } catch (MetaNotFoundException metaNotFoundException) { + LOG.error(mvBaseTableInfo.toString() + " can not find corresponding materialized view."); + } + }); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index b0de1ccfa469bb7..8b1e75017caaafc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -17,52 +17,131 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.View; +import org.apache.doris.mtmv.MVCache; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.memo.GroupId; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.qe.OriginStatement; + +import com.google.common.collect.ImmutableList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * Maintain the context for query rewrite by materialized view */ public class MaterializationContext { - // TODO add MaterializedView class - private final Plan mvPlan; - private final CascadesContext context; + private MTMV mtmv; + // Should use stmt id generator in query context + private final Plan mvScanPlan; private final List baseTables; - private final List baseViews; + private final List
baseViews; // Group ids that are rewritten by this mv to reduce rewrite times private final Set matchedGroups = new HashSet<>(); - private final Plan scanPlan; + // generate form mv scan plan + private ExpressionMapping viewExpressionMapping; - public MaterializationContext(Plan mvPlan, CascadesContext context, - List
baseTables, List baseViews, Plan scanPlan) { - this.mvPlan = mvPlan; - this.context = context; + /** + * MaterializationContext, this contains necessary info for query rewriting by mv + */ + public MaterializationContext(MTMV mtmv, + Plan mvScanPlan, + CascadesContext cascadesContext, + List
baseTables, + List
baseViews) { + this.mtmv = mtmv; + this.mvScanPlan = mvScanPlan; this.baseTables = baseTables; this.baseViews = baseViews; - this.scanPlan = scanPlan; + MVCache mvCache = mtmv.getMvCache(); + // TODO This logic should move to materialized view cache manager + if (mvCache == null) { + LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql()); + StatementContext mvSqlStatementContext = new StatementContext( + cascadesContext.getConnectContext(), + new OriginStatement(mtmv.getQuerySql(), 0)); + NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); + + planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + Plan mvAnalyzedPlan = planner.getAnalyzedPlan(); + Plan mvRewrittenPlan = planner.getRewrittenPlan(); + mvCache = MVCache.from(mtmv, mvRewrittenPlan instanceof LogicalResultSink + ? (Plan) ((LogicalResultSink) mvRewrittenPlan).child() : mvRewrittenPlan, + mvAnalyzedPlan.getExpressions().stream().map(NamedExpression.class::cast) + .collect(Collectors.toList()), + planner.getCascadesContext()); + mtmv.setMvCache(mvCache); + } + List mvOutputExpressions = mvCache.getMvOutputExpressions(); + // mv output expression shuttle, this will be used to expression rewrite + mvOutputExpressions = + ExpressionUtils.shuttleExpressionWithLineage(mvOutputExpressions, mvCache.getLogicalPlan()).stream() + .map(NamedExpression.class::cast) + .collect(Collectors.toList()); + this.viewExpressionMapping = ExpressionMapping.generate( + mvOutputExpressions, + mvScanPlan.getExpressions()); } public Set getMatchedGroups() { return matchedGroups; } + public boolean alreadyRewrite(GroupId groupId) { + return this.matchedGroups.contains(groupId); + } + public void addMatchedGroup(GroupId groupId) { matchedGroups.add(groupId); } - public Plan getMvPlan() { - return mvPlan; + public MTMV getMtmv() { + return mtmv; + } + + public Plan getMvScanPlan() { + return mvScanPlan; + } + + public List
getBaseTables() { + return baseTables; + } + + public List
getBaseViews() { + return baseViews; + } + + public ExpressionMapping getViewExpressionIndexMapping() { + return viewExpressionMapping; } - public Plan getScanPlan() { - return scanPlan; + /** + * MaterializationContext fromMaterializedView + */ + public static MaterializationContext fromMaterializedView(MTMV materializedView, + Plan mvScanPlan, + CascadesContext cascadesContext) { + return new MaterializationContext( + materializedView, + mvScanPlan, + cascadesContext, + ImmutableList.of(), + ImmutableList.of()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index 40b91994be76e9e..4f116b7aed0a60f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -19,71 +19,81 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors.PredicatesSpliter; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; -import java.util.Set; +import java.util.Optional; +import java.util.stream.Collectors; /** * This record the predicates which can be pulled up or some other type predicates - * */ + */ public class Predicates { // Predicates that can be pulled up - private final Set pulledUpPredicates; + private final List pulledUpPredicates = new ArrayList<>(); - public Predicates(Set pulledUpPredicates) { - this.pulledUpPredicates = pulledUpPredicates; + private Predicates() { } - public static Predicates of(Set pulledUpPredicates) { - return new Predicates(pulledUpPredicates); + public static Predicates of() { + return new Predicates(); } - public Set getPulledUpPredicates() { + public static Predicates of(List pulledUpPredicates) { + Predicates predicates = new Predicates(); + pulledUpPredicates.forEach(predicates::addPredicate); + return predicates; + } + + public List getPulledUpPredicates() { return pulledUpPredicates; } + public void addPredicate(Expression expression) { + this.pulledUpPredicates.add(expression); + } + public Expression composedExpression() { - return ExpressionUtils.and(pulledUpPredicates); + return ExpressionUtils.and(pulledUpPredicates.stream().map(Expression.class::cast) + .collect(Collectors.toList())); } /** * Split the expression to equal, range and residual predicate. - * */ + */ public static SplitPredicate splitPredicates(Expression expression) { - PredicatesSpliter predicatesSplit = new PredicatesSpliter(expression); - expression.accept(predicatesSplit, null); + PredicatesSplitter predicatesSplit = new PredicatesSplitter(expression); return predicatesSplit.getSplitPredicate(); } /** * The split different representation for predicate expression, such as equal, range and residual predicate. - * */ + */ public static final class SplitPredicate { - private final Expression equalPredicates; - private final Expression rangePredicates; - private final Expression residualPredicates; + private Optional equalPredicates; + private Optional rangePredicates; + private Optional residualPredicates; public SplitPredicate(Expression equalPredicates, Expression rangePredicates, Expression residualPredicates) { - this.equalPredicates = equalPredicates; - this.rangePredicates = rangePredicates; - this.residualPredicates = residualPredicates; + this.equalPredicates = Optional.ofNullable(equalPredicates); + this.rangePredicates = Optional.ofNullable(rangePredicates); + this.residualPredicates = Optional.ofNullable(residualPredicates); } - public Expression getEqualPredicates() { - return equalPredicates; + public Expression getEqualPredicate() { + return equalPredicates.orElse(BooleanLiteral.TRUE); } - public Expression getRangePredicates() { - return rangePredicates; + public Expression getRangePredicate() { + return rangePredicates.orElse(BooleanLiteral.TRUE); } - public Expression getResidualPredicates() { - return residualPredicates; + public Expression getResidualPredicate() { + return residualPredicates.orElse(BooleanLiteral.TRUE); } public static SplitPredicate empty() { @@ -92,7 +102,7 @@ public static SplitPredicate empty() { /** * SplitPredicate construct - * */ + */ public static SplitPredicate of(Expression equalPredicates, Expression rangePredicates, Expression residualPredicates) { @@ -101,31 +111,32 @@ public static SplitPredicate of(Expression equalPredicates, /** * isEmpty - * */ + */ public boolean isEmpty() { - return equalPredicates == null - && rangePredicates == null - && residualPredicates == null; - } - - public Expression composedExpression() { - return ExpressionUtils.and(equalPredicates, rangePredicates, residualPredicates); + return !equalPredicates.isPresent() + && !rangePredicates.isPresent() + && !residualPredicates.isPresent(); } public List toList() { - return ImmutableList.of(equalPredicates, rangePredicates, residualPredicates); + return ImmutableList.of(equalPredicates.orElse(BooleanLiteral.TRUE), + rangePredicates.orElse(BooleanLiteral.TRUE), + residualPredicates.orElse(BooleanLiteral.TRUE)); } /** * Check the predicates in SplitPredicate is whether all true or not */ public boolean isAlwaysTrue() { - return equalPredicates instanceof BooleanLiteral - && rangePredicates instanceof BooleanLiteral - && residualPredicates instanceof BooleanLiteral - && ((BooleanLiteral) equalPredicates).getValue() - && ((BooleanLiteral) rangePredicates).getValue() - && ((BooleanLiteral) residualPredicates).getValue(); + Expression equalExpr = equalPredicates.orElse(BooleanLiteral.TRUE); + Expression rangeExpr = rangePredicates.orElse(BooleanLiteral.TRUE); + Expression residualExpr = residualPredicates.orElse(BooleanLiteral.TRUE); + return equalExpr instanceof BooleanLiteral + && rangeExpr instanceof BooleanLiteral + && residualExpr instanceof BooleanLiteral + && ((BooleanLiteral) equalExpr).getValue() + && ((BooleanLiteral) rangeExpr).getValue() + && ((BooleanLiteral) residualExpr).getValue(); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java new file mode 100644 index 000000000000000..2975b0c61aaf47f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.CompoundPredicate; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; +import org.apache.doris.nereids.util.ExpressionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * Split the expression to equal, range and residual predicate. + * Should instance when used. + * TODO support complex predicate split + */ +public class PredicatesSplitter { + + private List equalPredicates = new ArrayList<>(); + private List rangePredicates = new ArrayList<>(); + private List residualPredicates = new ArrayList<>(); + private List conjunctExpressions; + + private final PredicateExtract instance = new PredicateExtract(); + + public PredicatesSplitter(Expression target) { + this.conjunctExpressions = ExpressionUtils.extractConjunction(target); + for (Expression expression : conjunctExpressions) { + expression.accept(instance, expression); + } + } + + /**PredicateExtract*/ + public class PredicateExtract extends DefaultExpressionVisitor { + + @Override + public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Expression sourceExpression) { + Expression leftArg = comparisonPredicate.getArgument(0); + Expression rightArg = comparisonPredicate.getArgument(1); + boolean leftArgOnlyContainsColumnRef = containOnlyColumnRef(leftArg, true); + boolean rightArgOnlyContainsColumnRef = containOnlyColumnRef(rightArg, true); + if (comparisonPredicate instanceof EqualTo || comparisonPredicate instanceof NullSafeEqual) { + if (leftArgOnlyContainsColumnRef && rightArgOnlyContainsColumnRef) { + equalPredicates.add(comparisonPredicate); + return null; + } else { + residualPredicates.add(comparisonPredicate); + } + } else if ((leftArgOnlyContainsColumnRef && rightArg instanceof Literal) + || (rightArgOnlyContainsColumnRef && leftArg instanceof Literal)) { + rangePredicates.add(comparisonPredicate); + } else { + residualPredicates.add(comparisonPredicate); + } + return null; + } + + @Override + public Void visitCompoundPredicate(CompoundPredicate compoundPredicate, Expression context) { + if (compoundPredicate instanceof Or) { + residualPredicates.add(compoundPredicate); + return null; + } + return super.visitCompoundPredicate(compoundPredicate, context); + } + } + + public Predicates.SplitPredicate getSplitPredicate() { + return Predicates.SplitPredicate.of( + equalPredicates.isEmpty() ? null : ExpressionUtils.and(equalPredicates), + rangePredicates.isEmpty() ? null : ExpressionUtils.and(rangePredicates), + residualPredicates.isEmpty() ? null : ExpressionUtils.and(residualPredicates)); + } + + private static boolean containOnlyColumnRef(Expression expression, boolean allowCast) { + if (expression instanceof SlotReference && ((SlotReference) expression).isColumnFromTable()) { + return true; + } + if (allowCast && expression instanceof Cast) { + return containOnlyColumnRef(((Cast) expression).child(), true); + } + if (expression instanceof Alias) { + return containOnlyColumnRef(((Alias) expression).child(), true); + } + return false; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java deleted file mode 100644 index ea91104a246e096..000000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.catalog.TableIf; -import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; - -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.BiMap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Multimap; - -import java.util.List; - -/** - * Relation mapping - * such as query pattern is a1 left join a2 left join b - * view pattern is a1 left join a2 left join b. the mapping will be - * [{a1:a1, a2:a2, b:b}, {a1:a2, a2:a1, b:b}] - */ -public class RelationMapping extends Mapping { - - private final BiMap mappedRelationMap; - - public RelationMapping(BiMap mappedRelationMap) { - this.mappedRelationMap = mappedRelationMap; - } - - public BiMap getMappedRelationMap() { - return mappedRelationMap; - } - - /** - * Generate mapping according to source and target relation - */ - public static List generate(List source, List target) { - Multimap queryTableRelationIdMap = ArrayListMultimap.create(); - for (CatalogRelation relation : source) { - queryTableRelationIdMap.put(relation.getTable(), relation); - } - Multimap viewTableRelationIdMap = ArrayListMultimap.create(); - for (CatalogRelation relation : target) { - viewTableRelationIdMap.put(relation.getTable(), relation); - } - // todo generate relation map - return ImmutableList.of(); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 141b8a98bccfdbf..46be35992c54a55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -23,51 +23,142 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.algebra.Filter; +import org.apache.doris.nereids.trees.plans.algebra.Join; +import org.apache.doris.nereids.trees.plans.algebra.Project; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.util.ExpressionUtils; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nullable; /** * StructInfo */ public class StructInfo { - private final List relations; - private final Predicates predicates; - // Used by predicate compensation - private final EquivalenceClass equivalenceClass; + public static final JoinPatternChecker JOIN_PATTERN_CHECKER = new JoinPatternChecker(); + private static final RelationCollector RELATION_COLLECTOR = new RelationCollector(); + private static final PredicateCollector PREDICATE_COLLECTOR = new PredicateCollector(); + // struct info splitter + private static final PlanSplitter PLAN_SPLITTER = new PlanSplitter(); + // source data private final Plan originalPlan; private final HyperGraph hyperGraph; + private boolean valid = true; + // derived data following + // top plan which may include project or filter, except for join and scan + private Plan topPlan; + // bottom plan which top plan only contain join or scan. this is needed by hyper graph + private Plan bottomPlan; + private final List relations = new ArrayList<>(); + private Predicates predicates; + private SplitPredicate splitPredicate; + private EquivalenceClass equivalenceClass; - private StructInfo(List relations, - Predicates predicates, - Plan originalPlan, - HyperGraph hyperGraph) { - this.relations = relations; - this.predicates = predicates; + private StructInfo(Plan originalPlan, @Nullable Plan topPlan, @Nullable Plan bottomPlan, HyperGraph hyperGraph) { this.originalPlan = originalPlan; this.hyperGraph = hyperGraph; + this.topPlan = topPlan; + this.bottomPlan = bottomPlan; + init(); + } + + private void init() { + + if (topPlan == null || bottomPlan == null) { + List topPlans = new ArrayList<>(); + this.bottomPlan = originalPlan.accept(PLAN_SPLITTER, topPlans); + this.topPlan = topPlans.get(0); + } + + this.predicates = Predicates.of(); + // Collect predicate from join condition in hyper graph + this.hyperGraph.getEdges().forEach(edge -> { + List hashJoinConjuncts = edge.getHashJoinConjuncts(); + hashJoinConjuncts.forEach(this.predicates::addPredicate); + List otherJoinConjuncts = edge.getOtherJoinConjuncts(); + if (!otherJoinConjuncts.isEmpty()) { + this.valid = false; + } + }); + if (!this.isValid()) { + return; + } + + // Collect predicate from filter node in hyper graph + this.hyperGraph.getNodes().forEach(node -> { + // plan relation collector + Plan nodePlan = node.getPlan(); + nodePlan.accept(RELATION_COLLECTOR, this.relations); + // if inner join add where condition + Set predicates = new HashSet<>(); + nodePlan.accept(PREDICATE_COLLECTOR, predicates); + predicates.forEach(this.predicates::addPredicate); + }); + + // TODO Collect predicate from top plan not in hyper graph, should optimize, twice now + Set topPlanPredicates = new HashSet<>(); + topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates); + topPlanPredicates.forEach(this.predicates::addPredicate); + // construct equivalenceClass according to equals predicates this.equivalenceClass = new EquivalenceClass(); - SplitPredicate splitPredicate = Predicates.splitPredicates(predicates.composedExpression()); - for (Expression expression : ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicates())) { - EqualTo equalTo = (EqualTo) expression; - equivalenceClass.addEquivalenceClass( - (SlotReference) equalTo.getArguments().get(0), - (SlotReference) equalTo.getArguments().get(1)); + List shuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( + this.predicates.getPulledUpPredicates(), originalPlan).stream() + .map(Expression.class::cast) + .collect(Collectors.toList()); + SplitPredicate splitPredicate = Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression)); + this.splitPredicate = splitPredicate; + for (Expression expression : ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicate())) { + if (expression instanceof BooleanLiteral && ((BooleanLiteral) expression).getValue()) { + continue; + } + if (expression instanceof EqualTo) { + EqualTo equalTo = (EqualTo) expression; + equivalenceClass.addEquivalenceClass( + (SlotReference) equalTo.getArguments().get(0), + (SlotReference) equalTo.getArguments().get(1)); + } } } - public static StructInfo of(Plan originalPlan) { - // TODO build graph from original plan and get relations and predicates from graph - return new StructInfo(null, null, originalPlan, null); + /** + * Build Struct info from plan. + * Maybe return multi structInfo when original plan already be rewritten by mv + */ + public static List of(Plan originalPlan) { + // TODO only consider the inner join currently, Should support outer join + // Split plan by the boundary which contains multi child + List topPlans = new ArrayList<>(); + Plan bottomPlan = originalPlan.accept(PLAN_SPLITTER, topPlans); + Plan topPlan = topPlans.get(0); + + List structInfos = HyperGraph.toStructInfo(bottomPlan); + return structInfos.stream() + .map(hyperGraph -> new StructInfo(originalPlan, topPlan, bottomPlan, hyperGraph)) + .collect(Collectors.toList()); } + /** + * Build Struct info from group. + * Maybe return multi structInfo when original plan already be rewritten by mv + */ public static StructInfo of(Group group) { // TODO build graph from original plan and get relations and predicates from graph - return new StructInfo(null, null, group.getLogicalExpression().getPlan(), null); + return null; } public List getRelations() { @@ -90,6 +181,14 @@ public HyperGraph getHyperGraph() { return hyperGraph; } + public SplitPredicate getSplitPredicate() { + return splitPredicate; + } + + public boolean isValid() { + return valid; + } + public List getExpressions() { return originalPlan instanceof LogicalProject ? ((LogicalProject) originalPlan).getProjects() : originalPlan.getOutput(); @@ -99,8 +198,75 @@ public List getExpressions() { * Judge the source graph logical is whether the same as target * For inner join should judge only the join tables, * for other join type should also judge the join direction, it's input filter that can not be pulled up etc. - * */ + */ public static boolean isGraphLogicalEquals(HyperGraph source, HyperGraph target) { - return false; + // TODO: if not inner join, should check the join graph logical equivalence + return true; + } + + private static class RelationCollector extends DefaultPlanVisitor> { + @Override + public Void visit(Plan plan, List collectedRelations) { + if (plan instanceof CatalogRelation) { + collectedRelations.add((CatalogRelation) plan); + } + return super.visit(plan, collectedRelations); + } + } + + private static class PredicateCollector extends DefaultPlanVisitor> { + @Override + public Void visit(Plan plan, Set predicates) { + if (plan instanceof LogicalFilter) { + predicates.add(((LogicalFilter) plan).getPredicate()); + } + return super.visit(plan, predicates); + } + } + + private static class PlanSplitter extends DefaultPlanRewriter> { + + @Override + public Plan visitLogicalRelation(LogicalRelation relation, List topPlans) { + return relation; + } + + @Override + public Plan visit(Plan plan, List topPlans) { + if (plan instanceof Join || plan instanceof SetOperation) { + return plan; + } else { + if (topPlans.isEmpty()) { + topPlans.add(plan); + } + return plan.children().get(0).accept(this, topPlans); + } + } + } + + /** + * JoinPatternChecker + */ + public static class JoinPatternChecker extends DefaultPlanVisitor> { + @Override + public Boolean visit(Plan plan, Set requiredJoinType) { + super.visit(plan, requiredJoinType); + if (!(plan instanceof Filter) + && !(plan instanceof Project) + && !(plan instanceof CatalogRelation) + && !(plan instanceof Join)) { + return false; + } + if (plan instanceof Join) { + Join join = (Join) plan; + if (!requiredJoinType.contains(join.getJoinType())) { + return false; + } + if (!join.getOtherJoinConjuncts().isEmpty()) { + return false; + } + } + return true; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java new file mode 100644 index 000000000000000..f5153a20d9307fd --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv.mapping; + +import org.apache.doris.nereids.rules.exploration.mv.EquivalenceClass; +import org.apache.doris.nereids.trees.expressions.SlotReference; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * EquivalenceClassSetMapping + */ +public class EquivalenceClassSetMapping extends Mapping { + + private final Map, Set> equivalenceClassSetMap; + + public EquivalenceClassSetMapping(Map, + Set> equivalenceClassSetMap) { + this.equivalenceClassSetMap = equivalenceClassSetMap; + } + + public static EquivalenceClassSetMapping of(Map, Set> equivalenceClassSetMap) { + return new EquivalenceClassSetMapping(equivalenceClassSetMap); + } + + /** + * source equivalence set map to target equivalence set + */ + public static EquivalenceClassSetMapping generate(EquivalenceClass source, EquivalenceClass target) { + + Map, Set> equivalenceClassSetMap = new HashMap<>(); + List> sourceSets = source.getEquivalenceSetList(); + List> targetSets = target.getEquivalenceSetList(); + + for (Set sourceSet : sourceSets) { + for (Set targetSet : targetSets) { + if (sourceSet.containsAll(targetSet)) { + equivalenceClassSetMap.put(sourceSet, targetSet); + } + } + } + return EquivalenceClassSetMapping.of(equivalenceClassSetMap); + } + + public Map, Set> getEquivalenceClassSetMap() { + return equivalenceClassSetMap; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java new file mode 100644 index 000000000000000..45d5bac03610575 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv.mapping; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Expression and it's index mapping + */ +public class ExpressionMapping extends Mapping { + private final Multimap expressionMapping; + + public ExpressionMapping(Multimap expressionMapping) { + this.expressionMapping = expressionMapping; + } + + public Multimap getExpressionMapping() { + return expressionMapping; + } + + /** + * ExpressionMapping flatten + */ + public List> flattenMap() { + List>> tmpExpressionPairs = new ArrayList<>(this.expressionMapping.size()); + Map> map = expressionMapping.asMap(); + for (Map.Entry> entry : map.entrySet()) { + List> valueList = new ArrayList<>(entry.getValue().size()); + for (Expression valueExpression : entry.getValue()) { + valueList.add(Pair.of(entry.getKey(), valueExpression)); + } + tmpExpressionPairs.add(valueList); + } + List>> cartesianExpressionMap = Lists.cartesianProduct(tmpExpressionPairs); + + final List> flattenedMap = new ArrayList<>(); + for (List> listPair : cartesianExpressionMap) { + final Map expressionMap = new HashMap<>(); + listPair.forEach(pair -> expressionMap.put(pair.key(), pair.value())); + flattenedMap.add(expressionMap); + } + return flattenedMap; + } + + /**Permute the key of expression mapping. this is useful for expression rewrite, if permute key to query based + * then when expression rewrite success, we can get the mv scan expression directly. + */ + public ExpressionMapping keyPermute(SlotMapping slotMapping) { + Multimap permutedExpressionMapping = ArrayListMultimap.create(); + Map> expressionMap = + this.getExpressionMapping().asMap(); + for (Map.Entry> entry : + expressionMap.entrySet()) { + Expression replacedExpr = ExpressionUtils.replace(entry.getKey(), slotMapping.toSlotReferenceMap()); + permutedExpressionMapping.putAll(replacedExpr, entry.getValue()); + } + return new ExpressionMapping(permutedExpressionMapping); + } + + /**ExpressionMapping generate*/ + public static ExpressionMapping generate( + List sourceExpressions, + List targetExpressions) { + final Multimap expressionMultiMap = + ArrayListMultimap.create(); + for (int i = 0; i < sourceExpressions.size(); i++) { + expressionMultiMap.put(sourceExpressions.get(i), targetExpressions.get(i)); + } + return new ExpressionMapping(expressionMultiMap); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java similarity index 69% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java index 487fc92ce50a2bf..17a412dab10d39a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java @@ -15,18 +15,15 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.rules.exploration.mv; +package org.apache.doris.nereids.rules.exploration.mv.mapping; import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; - -import java.util.List; import java.util.Objects; +import javax.annotation.Nullable; /** * Mapping slot from query to view or inversely, @@ -38,6 +35,7 @@ public abstract class Mapping { * The relation for mapping */ public static final class MappedRelation { + public final RelationId relationId; public final CatalogRelation belongedRelation; @@ -46,7 +44,7 @@ public MappedRelation(RelationId relationId, CatalogRelation belongedRelation) { this.belongedRelation = belongedRelation; } - public MappedRelation of(RelationId relationId, CatalogRelation belongedRelation) { + public static MappedRelation of(RelationId relationId, CatalogRelation belongedRelation) { return new MappedRelation(relationId, belongedRelation); } @@ -82,15 +80,31 @@ public int hashCode() { public static final class MappedSlot { public final ExprId exprId; + public final Slot slot; + @Nullable public final CatalogRelation belongedRelation; - public MappedSlot(ExprId exprId, CatalogRelation belongedRelation) { + public MappedSlot(ExprId exprId, + Slot slot, + CatalogRelation belongedRelation) { this.exprId = exprId; + this.slot = slot; this.belongedRelation = belongedRelation; } - public MappedSlot of(ExprId exprId, CatalogRelation belongedRelation) { - return new MappedSlot(exprId, belongedRelation); + public static MappedSlot of(ExprId exprId, + Slot slot, + CatalogRelation belongedRelation) { + return new MappedSlot(exprId, slot, belongedRelation); + } + + public static MappedSlot of(Slot slot, + CatalogRelation belongedRelation) { + return new MappedSlot(slot.getExprId(), slot, belongedRelation); + } + + public static MappedSlot of(Slot slot) { + return new MappedSlot(slot.getExprId(), slot, null); } public ExprId getExprId() { @@ -101,6 +115,10 @@ public CatalogRelation getBelongedRelation() { return belongedRelation; } + public Slot getSlot() { + return slot; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -118,27 +136,4 @@ public int hashCode() { return Objects.hash(exprId); } } - - /** - * Expression and it's index mapping - */ - public static class ExpressionIndexMapping extends Mapping { - private final Multimap expressionIndexMapping; - - public ExpressionIndexMapping(Multimap expressionIndexMapping) { - this.expressionIndexMapping = expressionIndexMapping; - } - - public Multimap getExpressionIndexMapping() { - return expressionIndexMapping; - } - - public static ExpressionIndexMapping generate(List expressions) { - Multimap expressionIndexMapping = ArrayListMultimap.create(); - for (int i = 0; i < expressions.size(); i++) { - expressionIndexMapping.put(expressions.get(i), i); - } - return new ExpressionIndexMapping(expressionIndexMapping); - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java new file mode 100644 index 000000000000000..7fb857d3d4c77c1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv.mapping; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; + +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.Lists; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Relation mapping + * such as query pattern is a1 left join a2 left join b + * view pattern is a1 left join a2 left join b. the mapping will be + * [{a1:a1, a2:a2, b:b}, {a1:a2, a2:a1, b:b}] + */ +public class RelationMapping extends Mapping { + + private final BiMap mappedRelationMap; + + public RelationMapping(BiMap mappedRelationMap) { + this.mappedRelationMap = mappedRelationMap; + } + + public BiMap getMappedRelationMap() { + return mappedRelationMap; + } + + public static RelationMapping of(BiMap mappedRelationMap) { + return new RelationMapping(mappedRelationMap); + } + + /** + * Generate mapping according to source and target relation + */ + public static List generate(List sources, List targets) { + // Construct tmp map, key is the table qualifier, value is the corresponding catalog relations + LinkedListMultimap sourceTableRelationIdMap = LinkedListMultimap.create(); + for (CatalogRelation relation : sources) { + String tableQualifier = getTableQualifier(relation.getTable()); + if (tableQualifier == null) { + return null; + } + sourceTableRelationIdMap.put(tableQualifier, MappedRelation.of(relation.getRelationId(), relation)); + } + LinkedListMultimap targetTableRelationIdMap = LinkedListMultimap.create(); + for (CatalogRelation relation : targets) { + String tableQualifier = getTableQualifier(relation.getTable()); + if (tableQualifier == null) { + return null; + } + targetTableRelationIdMap.put(tableQualifier, MappedRelation.of(relation.getRelationId(), relation)); + } + + Set sourceTableKeySet = sourceTableRelationIdMap.keySet(); + List>> mappedRelations = new ArrayList<>(); + + for (String sourceTableQualifier : sourceTableKeySet) { + List sourceMappedRelations = sourceTableRelationIdMap.get(sourceTableQualifier); + List targetMappedRelations = targetTableRelationIdMap.get(sourceTableQualifier); + if (targetMappedRelations.isEmpty()) { + continue; + } + // if source and target relation appear once, just map them + if (targetMappedRelations.size() == 1 && sourceMappedRelations.size() == 1) { + mappedRelations.add(ImmutableList.of(Pair.of(sourceMappedRelations.get(0), + targetMappedRelations.get(0)))); + continue; + } + // relation appear more than once, should cartesian them + ImmutableList> relationMapping = Lists.cartesianProduct( + sourceTableRelationIdMap.get(sourceTableQualifier), targetMappedRelations) + .stream() + .map(listPair -> Pair.of(listPair.get(0), listPair.get(1))) + .collect(ImmutableList.toImmutableList()); + mappedRelations.add(relationMapping); + } + + int mappedRelationCount = mappedRelations.size(); + + return Lists.cartesianProduct(mappedRelations).stream() + .map(mappedRelationList -> { + BiMap relationMappedRelationBiMap = + HashBiMap.create(mappedRelationCount); + for (int relationIndex = 0; relationIndex < mappedRelationCount; relationIndex++) { + relationMappedRelationBiMap.put(mappedRelationList.get(relationIndex).key(), + mappedRelationList.get(relationIndex).value()); + } + return RelationMapping.of(relationMappedRelationBiMap); + }) + .collect(ImmutableList.toImmutableList()); + } + + private static String getTableQualifier(TableIf tableIf) { + String tableName = tableIf.getName(); + DatabaseIf database = tableIf.getDatabase(); + if (database == null) { + return null; + } + return database.getFullName() + ":" + tableName; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java new file mode 100644 index 000000000000000..3de99ea05a88d96 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv.mapping; + +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; + +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +/** + * SlotMapping, this is open generated from relationMapping + */ +public class SlotMapping extends Mapping { + + private final BiMap relationSlotMap; + private Map slotReferenceMap; + + public SlotMapping(BiMap relationSlotMap) { + this.relationSlotMap = relationSlotMap; + } + + public BiMap getRelationSlotMap() { + return relationSlotMap; + } + + public SlotMapping inverse() { + return SlotMapping.of(relationSlotMap.inverse()); + } + + public static SlotMapping of(BiMap relationSlotMap) { + return new SlotMapping(relationSlotMap); + } + + /** + * SlotMapping, this is open generated from relationMapping + */ + @Nullable + public static SlotMapping generate(RelationMapping relationMapping) { + BiMap relationSlotMap = HashBiMap.create(); + BiMap mappedRelationMap = relationMapping.getMappedRelationMap(); + for (Map.Entry mappedRelationEntry : mappedRelationMap.entrySet()) { + Map targetNameSlotMap = + mappedRelationEntry.getValue().getBelongedRelation().getOutput().stream() + .collect(Collectors.toMap(Slot::getName, slot -> slot)); + for (Slot sourceSlot : mappedRelationEntry.getKey().getBelongedRelation().getOutput()) { + Slot targetSlot = targetNameSlotMap.get(sourceSlot.getName()); + // source slot can not map from target, bail out + if (targetSlot == null) { + return null; + } + relationSlotMap.put(MappedSlot.of(sourceSlot, mappedRelationEntry.getKey().getBelongedRelation()), + MappedSlot.of(targetSlot, mappedRelationEntry.getValue().getBelongedRelation())); + } + } + return SlotMapping.of(relationSlotMap); + } + + public Map toMappedSlotMap() { + return (Map) this.getRelationSlotMap(); + } + + /** + * SlotMapping, toSlotReferenceMap + */ + public Map toSlotReferenceMap() { + if (this.slotReferenceMap != null) { + return this.slotReferenceMap; + } + Map slotReferenceSlotReferenceMap = new HashMap<>(); + for (Map.Entry entry : this.getRelationSlotMap().entrySet()) { + slotReferenceSlotReferenceMap.put((SlotReference) entry.getKey().getSlot(), + (SlotReference) entry.getValue().getSlot()); + } + this.slotReferenceMap = slotReferenceSlotReferenceMap; + return this.slotReferenceMap; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 6f937cc96d29535..694f0611567ec81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -24,6 +24,7 @@ import java.util.ArrayDeque; import java.util.Deque; import java.util.List; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -210,6 +211,19 @@ default List collectToList(Predicate> predicate) { return (List) result.build(); } + /** + * Collect the nodes that satisfied the predicate to set. + */ + default Set collectToSet(Predicate> predicate) { + ImmutableSet.Builder> result = ImmutableSet.builder(); + foreach(node -> { + if (predicate.test(node)) { + result.add(node); + } + }); + return (Set) result.build(); + } + /** * iterate top down and test predicate if contains any instance of the classes * @param types classes array diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java index 5e5a33fe497659a..77edf9835338fdc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java @@ -45,4 +45,14 @@ public ExprId getNextId() { public String toString() { return "" + id; } + + @Override + public boolean equals(Object obj) { + return super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java index 86949f63e1699e0..513da0e93d91120 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java @@ -17,16 +17,9 @@ package org.apache.doris.nereids.trees.expressions.visitor; -import org.apache.doris.nereids.rules.exploration.mv.Predicates; -import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; -import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; -import org.apache.doris.nereids.util.ExpressionUtils; - -import java.util.ArrayList; -import java.util.List; /** * This is the factory for all ExpressionVisitor instance. @@ -61,46 +54,4 @@ public Boolean visitAggregateFunction(AggregateFunction aggregateFunction, Void return true; } } - - /** - * Split the expression to equal, range and residual predicate. - * Should instance when used. - */ - public static class PredicatesSpliter extends DefaultExpressionVisitor { - - private List equalPredicates = new ArrayList<>(); - private List rangePredicates = new ArrayList<>(); - private List residualPredicates = new ArrayList<>(); - private final Expression target; - - public PredicatesSpliter(Expression target) { - this.target = target; - } - - @Override - public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Void context) { - // TODO Smallest implement, complete later - if (comparisonPredicate instanceof EqualTo) { - Expression leftArgument = comparisonPredicate.getArgument(0); - Expression rightArgument = comparisonPredicate.getArgument(1); - if (leftArgument.isSlot() && rightArgument.isSlot()) { - equalPredicates.add(comparisonPredicate); - } else { - rangePredicates.add(comparisonPredicate); - } - } - return super.visit(comparisonPredicate, context); - } - - public Expression getTarget() { - return target; - } - - public Predicates.SplitPredicate getSplitPredicate() { - return Predicates.SplitPredicate.of( - equalPredicates.isEmpty() ? null : ExpressionUtils.and(equalPredicates), - rangePredicates.isEmpty() ? null : ExpressionUtils.and(rangePredicates), - residualPredicates.isEmpty() ? null : ExpressionUtils.and(residualPredicates)); - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index 0d5fd4957a075af..0500fd5bb6f10eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -21,6 +21,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -78,6 +79,9 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { logicalPlanAdapter.setIsExplain(new ExplainOptions(level)); executor.setParsedStmt(logicalPlanAdapter); NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); + if (ctx.getSessionVariable().isEnableMaterializedViewRewrite()) { + planner.addHook(InitMaterializationContextHook.INSTANCE); + } planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); executor.setPlanner(planner); executor.checkBlockRules(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java new file mode 100644 index 000000000000000..888b538d049410c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.visitor; + +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer.ExpressionReplaceContext; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * ExpressionLineageReplacer + * Get from rewrite plan and can also get from plan struct info, if from plan struct info it depends on + * the nodes from graph. + */ +public class ExpressionLineageReplacer extends DefaultPlanVisitor { + + public static final ExpressionLineageReplacer INSTANCE = new ExpressionLineageReplacer(); + + @Override + public Expression visit(Plan plan, ExpressionReplaceContext context) { + List expressions = plan.getExpressions(); + Map targetExpressionMap = context.getExprIdExpressionMap(); + // Filter the namedExpression used by target and collect the namedExpression + expressions.stream() + .filter(expression -> expression instanceof NamedExpression + && targetExpressionMap.containsKey(((NamedExpression) expression).getExprId())) + .forEach(expression -> expression.accept(NamedExpressionCollector.INSTANCE, context)); + return super.visit(plan, context); + } + + /** + * Replace the expression with lineage according the exprIdExpressionMap + */ + public static class ExpressionReplacer extends DefaultExpressionRewriter> { + + public static final ExpressionReplacer INSTANCE = new ExpressionReplacer(); + + @Override + public Expression visitNamedExpression(NamedExpression namedExpression, + Map exprIdExpressionMap) { + if (exprIdExpressionMap.containsKey(namedExpression.getExprId())) { + return super.visit(exprIdExpressionMap.get(namedExpression.getExprId()), exprIdExpressionMap); + } + return super.visitNamedExpression(namedExpression, exprIdExpressionMap); + } + } + + /** + * The Collector for target named expressions + * TODO Collect named expression by targetTypes, tableIdentifiers + */ + public static class NamedExpressionCollector + extends DefaultExpressionVisitor { + + public static final NamedExpressionCollector INSTANCE = new NamedExpressionCollector(); + + @Override + public Void visitSlotReference(SlotReference slotReference, ExpressionReplaceContext context) { + context.getExprIdExpressionMap().put(slotReference.getExprId(), slotReference); + return super.visitSlotReference(slotReference, context); + } + + @Override + public Void visitArrayItemSlot(ArrayItemSlot arrayItemSlot, ExpressionReplaceContext context) { + context.getExprIdExpressionMap().put(arrayItemSlot.getExprId(), arrayItemSlot); + return super.visitArrayItemSlot(arrayItemSlot, context); + } + + @Override + public Void visitAlias(Alias alias, ExpressionReplaceContext context) { + // remove the alias + if (context.getExprIdExpressionMap().containsKey(alias.getExprId())) { + context.getExprIdExpressionMap().put(alias.getExprId(), alias.child()); + } + return super.visitAlias(alias, context); + } + } + + /** + * The context for replacing the expression with lineage + */ + public static class ExpressionReplaceContext { + private final List targetExpressions; + private final Set targetTypes; + private final Set tableIdentifiers; + private Map exprIdExpressionMap; + private List replacedExpressions; + + /**ExpressionReplaceContext*/ + public ExpressionReplaceContext(List targetExpressions, + Set targetTypes, + Set tableIdentifiers) { + this.targetExpressions = targetExpressions; + this.targetTypes = targetTypes; + this.tableIdentifiers = tableIdentifiers; + // collect only named expressions and replace them with linage identifier later + this.exprIdExpressionMap = targetExpressions.stream() + .map(each -> each.collectToList(NamedExpression.class::isInstance)) + .flatMap(Collection::stream) + .map(NamedExpression.class::cast) + .collect(Collectors.toMap(NamedExpression::getExprId, expr -> expr)); + } + + public List getTargetExpressions() { + return targetExpressions; + } + + public Set getTargetTypes() { + return targetTypes; + } + + public Set getTableIdentifiers() { + return tableIdentifiers; + } + + public Map getExprIdExpressionMap() { + return exprIdExpressionMap; + } + + /** + * getReplacedExpressions + */ + public List getReplacedExpressions() { + if (this.replacedExpressions == null) { + this.replacedExpressions = targetExpressions.stream() + .map(original -> original.accept(ExpressionReplacer.INSTANCE, getExprIdExpressionMap())) + .collect(Collectors.toList()); + } + return this.replacedExpressions; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java index 54ffab7596690ec..a3c874f63704444 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java @@ -39,7 +39,7 @@ public class TableCollector extends DefaultPlanVisitor type, Collection shuttleExpressionWithLineage(List expressions, + Plan plan) { + return shuttleExpressionWithLineage(expressions, plan, ImmutableSet.of(), ImmutableSet.of()); + } + /** - * Replace the slot in expression with the lineage identifier from specified - * baseTable sets or target table types. - *

- * For example as following: + * Replace the slot in expressions with the lineage identifier from specifiedbaseTable sets or target table types + * example as following: * select a + 10 as a1, d from ( * select b - 5 as a, d from table * ); - * after shuttle a1, d in select will be b - 5 + 10, d + * op expression before is: a + 10 as a1, d. after is: b - 5 + 10, d + * todo to get from plan struct info */ - public static List shuttleExpressionWithLineage(List expression, + public static List shuttleExpressionWithLineage(List expressions, Plan plan, Set targetTypes, Set tableIdentifiers) { - return ImmutableList.of(); - } - /** - * Replace the slot in expressions according to the slotMapping - * if any slot cannot be mapped then return null - */ - public static List permute(List expressions, SlotMapping slotMapping) { - return ImmutableList.of(); + ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = + new ExpressionLineageReplacer.ExpressionReplaceContext( + expressions.stream().map(Expression.class::cast).collect(Collectors.toList()), + targetTypes, + tableIdentifiers); + + plan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); + // Replace expressions by expression map + List replacedExpressions = replaceContext.getReplacedExpressions(); + if (expressions.size() != replacedExpressions.size()) { + throw new NereidsException("shuttle expression fail", + new MaterializedViewException("shuttle expression fail")); + } + return replacedExpressions; } /** @@ -292,7 +305,24 @@ public static Optional extractSlotOrCastOnSlot(Expression expr) { * */ public static Expression replace(Expression expr, Map replaceMap) { - return expr.accept(ExpressionReplacer.INSTANCE, replaceMap); + return expr.accept(ExpressionReplacer.INSTANCE, ExpressionReplacerContext.of(replaceMap, false)); + } + + /** + * Replace expression node in the expression tree by `replaceMap` in top-down manner. + * if replaced, create alias + * For example. + * 

+     * input expression: a > 1
+     * replaceMap: a -> b + c
+     *
+     * output:
+     * ((b + c) as a) > 1
+     *
+ */ + public static Expression replace(Expression expr, Map replaceMap, + boolean withAlias) { + return expr.accept(ExpressionReplacer.INSTANCE, ExpressionReplacerContext.of(replaceMap, true)); } public static List replace(List exprs, @@ -317,18 +347,54 @@ public static List rewriteDownShortCircuit( } private static class ExpressionReplacer - extends DefaultExpressionRewriter> { + extends DefaultExpressionRewriter { public static final ExpressionReplacer INSTANCE = new ExpressionReplacer(); private ExpressionReplacer() { } @Override - public Expression visit(Expression expr, Map replaceMap) { - if (replaceMap.containsKey(expr)) { + public Expression visit(Expression expr, ExpressionReplacerContext replacerContext) { + Map replaceMap = replacerContext.getReplaceMap(); + boolean isContained = replaceMap.containsKey(expr); + if (!isContained) { + return super.visit(expr, replacerContext); + } + boolean withAlias = replacerContext.isWithAlias(); + if (!withAlias) { return replaceMap.get(expr); + } else { + Expression replacedExpression = replaceMap.get(expr); + if (replacedExpression instanceof SlotReference) { + replacedExpression = ((SlotReference) (replacedExpression)).withNullable(expr.nullable()); + } + return new Alias(((NamedExpression) expr).getExprId(), replacedExpression, + ((NamedExpression) expr).getName()); } - return super.visit(expr, replaceMap); + } + } + + private static class ExpressionReplacerContext { + private final Map replaceMap; + private final boolean withAlias; + + private ExpressionReplacerContext(Map replaceMap, + boolean withAlias) { + this.replaceMap = replaceMap; + this.withAlias = withAlias; + } + + public static ExpressionReplacerContext of(Map replaceMap, + boolean withAlias) { + return new ExpressionReplacerContext(replaceMap, withAlias); + } + + public Map getReplaceMap() { + return replaceMap; + } + + public boolean isWithAlias() { + return withAlias; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 1140d326fe0c27d..e724acfd86dd12b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -42,6 +42,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.Config; import org.apache.doris.common.UserException; +import org.apache.doris.nereids.PlannerHook; import org.apache.doris.qe.CommonResultSet; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ResultSet; @@ -678,4 +679,7 @@ public Optional handleQueryInFe(StatementBase parsedStmt) { ResultSet resultSet = new CommonResultSet(metadata, Collections.singletonList(data)); return Optional.of(resultSet); } + + @Override + public void addHook(PlannerHook hook) {} } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java index b113d83797828e5..4a17574f3bdbfa7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java @@ -29,6 +29,7 @@ import org.apache.doris.common.profile.PlanTreeBuilder; import org.apache.doris.common.profile.PlanTreePrinter; import org.apache.doris.common.util.LiteralUtils; +import org.apache.doris.nereids.PlannerHook; import org.apache.doris.qe.ResultSet; import org.apache.doris.thrift.TQueryOptions; @@ -143,4 +144,6 @@ public boolean isBlockQuery() { public abstract Optional handleQueryInFe(StatementBase parsedStmt); + public abstract void addHook(PlannerHook hook); + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 8340e3b13e5fd9d..f489daa0a21dcce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -122,7 +122,7 @@ import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.minidump.MinidumpUtils; import org.apache.doris.nereids.parser.NereidsParser; -import org.apache.doris.nereids.stats.StatsErrorEstimator; +import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; import org.apache.doris.nereids.trees.plans.commands.Forward; @@ -570,6 +570,9 @@ private void executeByNereids(TUniqueId queryId) throws Exception { } // create plan planner = new NereidsPlanner(statementContext); + if (context.getSessionVariable().isEnableMaterializedViewRewrite()) { + planner.addHook(InitMaterializationContextHook.INSTANCE); + } try { planner.plan(parsedStmt, context.getSessionVariable().toThrift()); checkBlockRules(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java new file mode 100644 index 000000000000000..e03228524331b14 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java @@ -0,0 +1,309 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +/**MappingTest*/ +public class MappingTest extends TestWithFeService { + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("mapping_test"); + useDatabase("mapping_test"); + + createTable("CREATE TABLE IF NOT EXISTS lineitem (\n" + + " L_ORDERKEY INTEGER NOT NULL,\n" + + " L_PARTKEY INTEGER NOT NULL\n" + + ")\n" + + "DUPLICATE KEY(L_ORDERKEY, L_PARTKEY)\n" + + "DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"); + createTable("CREATE TABLE IF NOT EXISTS orders (\n" + + " O_ORDERKEY INTEGER NOT NULL,\n" + + " O_CUSTKEY INTEGER NOT NULL,\n" + + " O_ORDERSTATUS CHAR(1) NOT NULL\n" + + ")\n" + + "DUPLICATE KEY(O_ORDERKEY, O_CUSTKEY)\n" + + "DISTRIBUTED BY HASH(O_ORDERKEY) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"); + createTable("CREATE TABLE IF NOT EXISTS customer (\n" + + " C_CUSTKEY INTEGER NOT NULL,\n" + + " C_NAME VARCHAR(25) NOT NULL,\n" + + ")\n" + + "DUPLICATE KEY(C_CUSTKEY, C_NAME)\n" + + "DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"); + } + + // test the num of source and target table is same + @Test + public void testGenerateMapping1() { + Plan sourcePlan = PlanChecker.from(connectContext) + .analyze("SELECT * " + + "FROM\n" + + " orders,\n" + + " lineitem,\n" + + " customer\n" + + "WHERE\n" + + " c_custkey = o_custkey\n" + + " AND l_orderkey = o_orderkey") + .getPlan(); + + Plan targetPlan = PlanChecker.from(connectContext) + .analyze("SELECT * " + + "FROM\n" + + " customer,\n" + + " orders,\n" + + " lineitem\n" + + "WHERE\n" + + " c_custkey = o_custkey\n" + + " AND l_orderkey = o_orderkey") + .getPlan(); + List sourceRelations = new ArrayList<>(); + sourcePlan.accept(RelationCollector.INSTANCE, sourceRelations); + + List targetRelations = new ArrayList<>(); + targetPlan.accept(RelationCollector.INSTANCE, targetRelations); + + List generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations); + Assertions.assertNotNull(generateRelationMapping); + Assertions.assertEquals(1, generateRelationMapping.size()); + + // expected slot mapping + BiMap expectedSlotMapping = HashBiMap.create(); + expectedSlotMapping.put(new ExprId(0), new ExprId(2)); + expectedSlotMapping.put(new ExprId(1), new ExprId(3)); + expectedSlotMapping.put(new ExprId(2), new ExprId(4)); + expectedSlotMapping.put(new ExprId(3), new ExprId(5)); + expectedSlotMapping.put(new ExprId(4), new ExprId(6)); + expectedSlotMapping.put(new ExprId(5), new ExprId(0)); + expectedSlotMapping.put(new ExprId(6), new ExprId(1)); + // expected relation mapping + BiMap expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMapping.put(new RelationId(1), new RelationId(2)); + expectedRelationMapping.put(new RelationId(2), new RelationId(0)); + assertRelationMapping(generateRelationMapping.get(0), expectedRelationMapping, expectedSlotMapping); + } + + // test the num of source table is less than target table + @Test + public void testGenerateMapping2() { + Plan sourcePlan = PlanChecker.from(connectContext) + .analyze("SELECT * " + + "FROM\n" + + " orders,\n" + + " customer\n" + + "WHERE\n" + + " c_custkey = o_custkey") + .getPlan(); + + Plan targetPlan = PlanChecker.from(connectContext) + .analyze("SELECT * " + + "FROM\n" + + " customer,\n" + + " orders,\n" + + " lineitem\n" + + "WHERE\n" + + " c_custkey = o_custkey\n" + + " AND l_orderkey = o_orderkey") + .getPlan(); + List sourceRelations = new ArrayList<>(); + sourcePlan.accept(RelationCollector.INSTANCE, sourceRelations); + + List targetRelations = new ArrayList<>(); + targetPlan.accept(RelationCollector.INSTANCE, targetRelations); + + List generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations); + Assertions.assertNotNull(generateRelationMapping); + Assertions.assertEquals(1, generateRelationMapping.size()); + + // expected slot mapping + BiMap expectedSlotMapping = HashBiMap.create(); + expectedSlotMapping.put(new ExprId(0), new ExprId(2)); + expectedSlotMapping.put(new ExprId(1), new ExprId(3)); + expectedSlotMapping.put(new ExprId(2), new ExprId(4)); + expectedSlotMapping.put(new ExprId(3), new ExprId(0)); + expectedSlotMapping.put(new ExprId(4), new ExprId(1)); + // expected relation mapping + BiMap expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMapping.put(new RelationId(1), new RelationId(0)); + assertRelationMapping(generateRelationMapping.get(0), expectedRelationMapping, expectedSlotMapping); + } + + // test the num of source table is more than target table + @Test + public void testGenerateMapping3() { + Plan sourcePlan = PlanChecker.from(connectContext) + .analyze("SELECT * " + + "FROM\n" + + " orders,\n" + + " lineitem,\n" + + " customer\n" + + "WHERE\n" + + " c_custkey = o_custkey\n" + + " AND l_orderkey = o_orderkey") + .getPlan(); + + Plan targetPlan = PlanChecker.from(connectContext) + .analyze("SELECT * " + + "FROM\n" + + " customer,\n" + + " orders\n" + + "WHERE\n" + + " c_custkey = o_custkey\n") + .getPlan(); + List sourceRelations = new ArrayList<>(); + sourcePlan.accept(RelationCollector.INSTANCE, sourceRelations); + + List targetRelations = new ArrayList<>(); + targetPlan.accept(RelationCollector.INSTANCE, targetRelations); + + List generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations); + Assertions.assertNotNull(generateRelationMapping); + Assertions.assertEquals(1, generateRelationMapping.size()); + + // expected slot mapping + BiMap expectedSlotMapping = HashBiMap.create(); + expectedSlotMapping.put(new ExprId(0), new ExprId(2)); + expectedSlotMapping.put(new ExprId(1), new ExprId(3)); + expectedSlotMapping.put(new ExprId(2), new ExprId(4)); + expectedSlotMapping.put(new ExprId(5), new ExprId(0)); + expectedSlotMapping.put(new ExprId(6), new ExprId(1)); + // expected relation mapping + BiMap expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMapping.put(new RelationId(2), new RelationId(0)); + assertRelationMapping(generateRelationMapping.get(0), expectedRelationMapping, expectedSlotMapping); + } + + // test table of source query is repeated + @Test + public void testGenerateMapping4() { + Plan sourcePlan = PlanChecker.from(connectContext) + .analyze("SELECT orders.*, l1.* " + + "FROM\n" + + " orders,\n" + + " lineitem l1,\n" + + " lineitem l2\n" + + "WHERE\n" + + " l1.l_orderkey = l2.l_orderkey\n" + + " AND l1.l_orderkey = o_orderkey") + .getPlan(); + + Plan targetPlan = PlanChecker.from(connectContext) + .analyze("SELECT * " + + "FROM\n" + + " lineitem,\n" + + " orders\n" + + "WHERE\n" + + " l_orderkey = o_orderkey") + .getPlan(); + List sourceRelations = new ArrayList<>(); + sourcePlan.accept(RelationCollector.INSTANCE, sourceRelations); + + List targetRelations = new ArrayList<>(); + targetPlan.accept(RelationCollector.INSTANCE, targetRelations); + + List generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations); + Assertions.assertNotNull(generateRelationMapping); + Assertions.assertEquals(2, generateRelationMapping.size()); + + // expected slot mapping + BiMap expectedSlotMapping = HashBiMap.create(); + expectedSlotMapping.put(new ExprId(0), new ExprId(2)); + expectedSlotMapping.put(new ExprId(1), new ExprId(3)); + expectedSlotMapping.put(new ExprId(2), new ExprId(4)); + expectedSlotMapping.put(new ExprId(3), new ExprId(0)); + expectedSlotMapping.put(new ExprId(4), new ExprId(1)); + // expected relation mapping + BiMap expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMapping.put(new RelationId(1), new RelationId(0)); + assertRelationMapping(generateRelationMapping.get(0), expectedRelationMapping, expectedSlotMapping); + + // expected slot mapping + expectedSlotMapping = HashBiMap.create(); + expectedSlotMapping.put(new ExprId(0), new ExprId(2)); + expectedSlotMapping.put(new ExprId(1), new ExprId(3)); + expectedSlotMapping.put(new ExprId(2), new ExprId(4)); + expectedSlotMapping.put(new ExprId(5), new ExprId(0)); + expectedSlotMapping.put(new ExprId(6), new ExprId(1)); + // expected relation mapping + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMapping.put(new RelationId(2), new RelationId(0)); + assertRelationMapping(generateRelationMapping.get(1), expectedRelationMapping, expectedSlotMapping); + } + + private void assertRelationMapping(RelationMapping relationMapping, + BiMap expectRelationMapping, + BiMap expectSlotMapping) { + // check relation mapping + BiMap generatedRelationMapping = HashBiMap.create(); + relationMapping.getMappedRelationMap().forEach((key, value) -> + generatedRelationMapping.put(key.getRelationId(), value.getRelationId())); + Assertions.assertEquals(generatedRelationMapping, expectRelationMapping); + + // Generate slot mapping from relationMapping and check + SlotMapping slotMapping = SlotMapping.generate(relationMapping); + Assertions.assertNotNull(slotMapping); + BiMap generatedSlotMapping = HashBiMap.create(); + slotMapping.getRelationSlotMap().forEach((key, value) -> + generatedSlotMapping.put(key.getExprId(), value.getExprId()) + ); + Assertions.assertEquals(generatedSlotMapping, expectSlotMapping); + } + + protected static class RelationCollector extends DefaultPlanVisitor> { + + public static final RelationCollector INSTANCE = new RelationCollector(); + + @Override + public Void visit(Plan plan, List catalogRelations) { + if (plan instanceof CatalogRelation) { + catalogRelations.add((CatalogRelation) plan); + } + return super.visit(plan, catalogRelations); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java index e8cc4dd266a9db4..ffd3c29a18d538b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java @@ -108,7 +108,7 @@ protected Expression typeCoercion(Expression expression) { return FunctionBinder.INSTANCE.rewrite(expression, null); } - private DataType getType(char t) { + protected DataType getType(char t) { switch (t) { case 'T': return TinyIntType.INSTANCE; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java new file mode 100644 index 000000000000000..4a8f2e981ef1167 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression; + +import org.apache.doris.catalog.Column; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.rules.exploration.mv.Predicates; +import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +/** + * PredicatesSplitterTest + */ +public class PredicatesSplitterTest extends ExpressionRewriteTestHelper { + + @Test + public void testSplitPredicates() { + assetEquals("a = b and (c = d or a = 10) and a > 7 and 10 > d", + "a = b", + "a > 7 and 10 > d", + "c = d or a = 10"); + assetEquals("a = b and c + d = e and a > 7 and 10 > d", + "a = b", + "a > 7 and 10 > d", + "c + d = e"); + assetEquals("a = b and c + d = e or a > 7 and 10 > d", + "", + "", + "a = b and c + d = e or a > 7 and 10 > d"); + } + + private void assetEquals(String expression, + String expectedEqualExpr, + String expectedRangeExpr, + String expectedResidualExpr) { + + Map mem = Maps.newHashMap(); + Expression targetExpr = replaceUnboundSlot(PARSER.parseExpression(expression), mem); + SplitPredicate splitPredicate = Predicates.splitPredicates(targetExpr); + + if (!StringUtils.isEmpty(expectedEqualExpr)) { + Expression equalExpression = replaceUnboundSlot(PARSER.parseExpression(expectedEqualExpr), mem); + Assertions.assertEquals(equalExpression, splitPredicate.getEqualPredicate()); + } else { + Assertions.assertNull(splitPredicate.getEqualPredicate()); + } + + if (!StringUtils.isEmpty(expectedRangeExpr)) { + Expression rangeExpression = replaceUnboundSlot(PARSER.parseExpression(expectedRangeExpr), mem); + Assertions.assertEquals(rangeExpression, splitPredicate.getRangePredicate()); + } else { + Assertions.assertNull(splitPredicate.getRangePredicate()); + } + + if (!StringUtils.isEmpty(expectedResidualExpr)) { + Expression residualExpression = replaceUnboundSlot(PARSER.parseExpression(expectedResidualExpr), mem); + Assertions.assertEquals(residualExpression, splitPredicate.getResidualPredicate()); + } else { + Assertions.assertNull(splitPredicate.getResidualPredicate()); + } + } + + @Override + public Expression replaceUnboundSlot(Expression expression, Map mem) { + List children = Lists.newArrayList(); + boolean hasNewChildren = false; + for (Expression child : expression.children()) { + Expression newChild = replaceUnboundSlot(child, mem); + if (newChild != child) { + hasNewChildren = true; + } + children.add(newChild); + } + if (expression instanceof UnboundSlot) { + String name = ((UnboundSlot) expression).getName(); + mem.putIfAbsent(name, SlotReference.fromColumn( + new Column(name, getType(name.charAt(0)).toCatalogDataType()), + Lists.newArrayList("table"))); + return mem.get(name); + } + return hasNewChildren ? expression.withChildren(children) : expression; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java index 55f78faa20b079a..7ab3e8083d1d771 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java @@ -19,7 +19,10 @@ import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -28,10 +31,70 @@ /** * ExpressionUtils ut. */ -public class ExpressionUtilsTest { +public class ExpressionUtilsTest extends TestWithFeService { private static final NereidsParser PARSER = new NereidsParser(); + @Override + protected void runBeforeAll() throws Exception { + createDatabase("expression_test"); + useDatabase("expression_test"); + + createTable("CREATE TABLE IF NOT EXISTS lineitem (\n" + + " L_ORDERKEY INTEGER NOT NULL,\n" + + " L_PARTKEY INTEGER NOT NULL,\n" + + " L_SUPPKEY INTEGER NOT NULL,\n" + + " L_LINENUMBER INTEGER NOT NULL,\n" + + " L_QUANTITY DECIMALV3(15,2) NOT NULL,\n" + + " L_EXTENDEDPRICE DECIMALV3(15,2) NOT NULL,\n" + + " L_DISCOUNT DECIMALV3(15,2) NOT NULL,\n" + + " L_TAX DECIMALV3(15,2) NOT NULL,\n" + + " L_RETURNFLAG CHAR(1) NOT NULL,\n" + + " L_LINESTATUS CHAR(1) NOT NULL,\n" + + " L_SHIPDATE DATE NOT NULL,\n" + + " L_COMMITDATE DATE NOT NULL,\n" + + " L_RECEIPTDATE DATE NOT NULL,\n" + + " L_SHIPINSTRUCT CHAR(25) NOT NULL,\n" + + " L_SHIPMODE CHAR(10) NOT NULL,\n" + + " L_COMMENT VARCHAR(44) NOT NULL\n" + + ")\n" + + "DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)\n" + + "DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"); + createTable("CREATE TABLE IF NOT EXISTS orders (\n" + + " O_ORDERKEY INTEGER NOT NULL,\n" + + " O_CUSTKEY INTEGER NOT NULL,\n" + + " O_ORDERSTATUS CHAR(1) NOT NULL,\n" + + " O_TOTALPRICE DECIMALV3(15,2) NOT NULL,\n" + + " O_ORDERDATE DATE NOT NULL,\n" + + " O_ORDERPRIORITY CHAR(15) NOT NULL, \n" + + " O_CLERK CHAR(15) NOT NULL, \n" + + " O_SHIPPRIORITY INTEGER NOT NULL,\n" + + " O_COMMENT VARCHAR(79) NOT NULL\n" + + ")\n" + + "DUPLICATE KEY(O_ORDERKEY, O_CUSTKEY)\n" + + "DISTRIBUTED BY HASH(O_ORDERKEY) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"); + createTable("CREATE TABLE IF NOT EXISTS partsupp (\n" + + " PS_PARTKEY INTEGER NOT NULL,\n" + + " PS_SUPPKEY INTEGER NOT NULL,\n" + + " PS_AVAILQTY INTEGER NOT NULL,\n" + + " PS_SUPPLYCOST DECIMALV3(15,2) NOT NULL,\n" + + " PS_COMMENT VARCHAR(199) NOT NULL \n" + + ")\n" + + "DUPLICATE KEY(PS_PARTKEY, PS_SUPPKEY)\n" + + "DISTRIBUTED BY HASH(PS_PARTKEY) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"); + connectContext.getSessionVariable().enableNereidsTimeout = false; + + } + @Test public void extractConjunctionTest() { List expressions; @@ -93,4 +156,45 @@ public void extractDisjunctionTest() { Assertions.assertEquals(c, expressions.get(1)); Assertions.assertEquals(eAndf, expressions.get(2)); } + + @Test + public void testShuttleExpressionWithLineage1() { + PlanChecker.from(connectContext) + .checkExplain("SELECT (o.c1_abs + ps.c2_abs) as add_alias, l.L_LINENUMBER, o.O_ORDERSTATUS " + + "FROM " + + "lineitem as l " + + "LEFT JOIN " + + "(SELECT abs(O_TOTALPRICE + 10) as c1_abs, O_CUSTKEY, O_ORDERSTATUS, O_ORDERKEY " + + "FROM orders) as o " + + "ON l.L_ORDERKEY = o.O_ORDERKEY " + + "JOIN " + + "(SELECT abs(sqrt(PS_SUPPLYCOST)) as c2_abs, PS_AVAILQTY, PS_PARTKEY, PS_SUPPKEY " + + "FROM partsupp) as ps " + + "ON l.L_PARTKEY = ps.PS_PARTKEY and l.L_SUPPKEY = ps.PS_SUPPKEY", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + List originalExpressions = rewrittenPlan.getExpressions(); + List shuttledExpressions + = ExpressionUtils.shuttleExpressionWithLineage( + originalExpressions, + rewrittenPlan, + Sets.newHashSet(), + Sets.newHashSet()); + assertExpect(originalExpressions, shuttledExpressions, + "(cast(abs((cast(O_TOTALPRICE as DECIMALV3(16, 2)) + 10.00)) as " + + "DOUBLE) + abs(sqrt(cast(PS_SUPPLYCOST as DOUBLE))))", + "L_LINENUMBER", + "O_ORDERSTATUS"); + }); + } + + private void assertExpect(List originalExpressions, + List shuttledExpressions, + String... expectExpressions) { + Assertions.assertEquals(originalExpressions.size(), shuttledExpressions.size()); + Assertions.assertEquals(originalExpressions.size(), expectExpressions.length); + for (int index = 0; index < shuttledExpressions.size(); index++) { + Assertions.assertEquals(shuttledExpressions.get(index).toSql(), expectExpressions[index]); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 39aede7a7458350..058a2de2f0e3c18 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -165,7 +165,7 @@ public PlanChecker applyTopDown(RuleFactory ruleFactory) { public PlanChecker applyTopDown(List rule) { Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext, - ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteTopDownJob::new, true))) + ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteTopDownJob::new, true))) .execute(); cascadesContext.toMemo(); MemoValidator.validate(cascadesContext.getMemo()); diff --git a/regression-test/data/nereids_rules_p0/mv/inner_join.out b/regression-test/data/nereids_rules_p0/mv/inner_join.out new file mode 100644 index 000000000000000..999748537661f34 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/inner_join.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0 -- + +-- !query1_1 -- + +-- !query1_2 -- + +-- !query1_3 -- + diff --git a/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy new file mode 100644 index 000000000000000..643765ca29f0b82 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("inner_join") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql "SET enable_materialized_view_rewrite=true" + sql "SET enable_nereids_timeout = false" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + O_ORDERKEY INTEGER NOT NULL, + O_CUSTKEY INTEGER NOT NULL, + O_ORDERSTATUS CHAR(1) NOT NULL, + O_TOTALPRICE DECIMALV3(15,2) NOT NULL, + O_ORDERDATE DATE NOT NULL, + O_ORDERPRIORITY CHAR(15) NOT NULL, + O_CLERK CHAR(15) NOT NULL, + O_SHIPPRIORITY INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(O_ORDERKEY, O_CUSTKEY) + DISTRIBUTED BY HASH(O_ORDERKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + L_ORDERKEY INTEGER NOT NULL, + L_PARTKEY INTEGER NOT NULL, + L_SUPPKEY INTEGER NOT NULL, + L_LINENUMBER INTEGER NOT NULL, + L_QUANTITY DECIMALV3(15,2) NOT NULL, + L_EXTENDEDPRICE DECIMALV3(15,2) NOT NULL, + L_DISCOUNT DECIMALV3(15,2) NOT NULL, + L_TAX DECIMALV3(15,2) NOT NULL, + L_RETURNFLAG CHAR(1) NOT NULL, + L_LINESTATUS CHAR(1) NOT NULL, + L_SHIPDATE DATE NOT NULL, + L_COMMITDATE DATE NOT NULL, + L_RECEIPTDATE DATE NOT NULL, + L_SHIPINSTRUCT CHAR(25) NOT NULL, + L_SHIPMODE CHAR(10) NOT NULL, + L_COMMENT VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER) + DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + PS_PARTKEY INTEGER NOT NULL, + PS_SUPPKEY INTEGER NOT NULL, + PS_AVAILQTY INTEGER NOT NULL, + PS_SUPPLYCOST DECIMALV3(15,2) NOT NULL, + PS_COMMENT VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(PS_PARTKEY, PS_SUPPKEY) + DISTRIBUTED BY HASH(PS_PARTKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + def check_rewrite = { mv_sql, query_sql, mv_name -> + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + sleep(3000) + explain { + sql("${query_sql}") + contains "(${mv_name})" + } + } + + // select + from + inner join + def mv1_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + def query1_0 = "select lineitem.L_LINENUMBER " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + check_rewrite(mv1_0, query1_0, "mv1_0") + order_qt_query1_0 "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + + def mv1_1 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY, partsupp.PS_AVAILQTY " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + + "inner join partsupp on lineitem.L_PARTKEY = partsupp.PS_PARTKEY " + + "and lineitem.L_SUPPKEY = partsupp.PS_SUPPKEY" + def query1_1 = "select lineitem.L_LINENUMBER " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + + "inner join partsupp on lineitem.L_PARTKEY = partsupp.PS_PARTKEY " + + "and lineitem.L_SUPPKEY = partsupp.PS_SUPPKEY" + check_rewrite(mv1_1, query1_1, "mv1_1") + order_qt_query1_1 "${query1_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1""" + + + def mv1_2 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + + "from orders " + + "inner join lineitem on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + def query1_2 = "select lineitem.L_LINENUMBER " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + check_rewrite(mv1_2, query1_2, "mv1_2") + order_qt_query1_2 "${query1_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_2""" + + // because hyper graph node contains group plan, should fix it firstly + def mv1_3 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + + "from orders " + + "inner join lineitem on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + def query1_3 = "select lineitem.L_LINENUMBER " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + + "where lineitem.L_LINENUMBER > 10" +// check_rewrite(mv1_3, query1_3, "mv1_3") + order_qt_query1_3 "${query1_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3""" +}