diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index c7cbcca14ab8445..d4c6fe07c323d32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -238,9 +238,8 @@ public PhysicalPlanTranslator(PlanTranslatorContext context, StatsErrorEstimator public PlanFragment translatePlan(PhysicalPlan physicalPlan) { PlanFragment rootFragment = physicalPlan.accept(this, context); List outputExprs = Lists.newArrayList(); - physicalPlan.getOutput().stream().forEach(slotReference -> outputExprs.add( - context.findSlotRef((SlotReference) slotReference) == null ? context.findSlotRef( - slotReference.getExprId()) : context.findSlotRef((SlotReference) slotReference))); + physicalPlan.getOutput().stream().map(Slot::getExprId) + .forEach(exprId -> outputExprs.add(context.findSlotRef(exprId))); rootFragment.setOutputExprs(outputExprs); Collections.reverse(context.getPlanFragments()); // TODO: maybe we need to trans nullable directly? and then we could remove call computeMemLayout @@ -961,7 +960,7 @@ public PlanFragment visitPhysicalCTEConsumer(PhysicalCTEConsumer cteConsumer, Slot consumerSlot = cteConsumer.getProducerToConsumerSlotMap().get(producerSlot); SlotRef slotRef = context.findSlotRef(producerSlot.getExprId()); tupleDescriptor = slotRef.getDesc().getParent(); - context.addExprIdSlotRefPair((SlotReference) consumerSlot, slotRef); + context.addExprIdSlotRefPair(consumerSlot.getExprId(), slotRef); } CTEScanNode cteScanNode = new CTEScanNode(tupleDescriptor); context.getRuntimeTranslator().ifPresent(runtimeFilterTranslator -> @@ -1646,7 +1645,7 @@ public PlanFragment visitPhysicalProject(PhysicalProject project inputPlanNode.setOutputTupleDesc(projectionTuple); } else { for (int i = 0; i < slots.size(); ++i) { - context.addExprIdSlotRefPair((SlotReference) slots.get(i), + context.addExprIdSlotRefPair(slots.get(i).getExprId(), (SlotRef) projectionExprs.get(i)); slotIdsByOrder.add(((SlotRef) projectionExprs.get(i)).getSlotId()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index 855f577cfd5cf16..8f273d4ec089e31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -75,7 +75,6 @@ public class PlanTranslatorContext { * index from Nereids' slot to legacy slot. */ private final Map exprIdToSlotRef = Maps.newHashMap(); - private final Map slotReferenceToSlotRef = Maps.newHashMap(); /** * Inverted index from legacy slot to Nereids' slot. @@ -200,10 +199,9 @@ public void addPlanFragment(PlanFragment planFragment) { this.planFragments.add(planFragment); } - public void addExprIdSlotRefPair(SlotReference slotReference, SlotRef slotRef) { - exprIdToSlotRef.put(slotReference.getExprId(), slotRef); - slotIdToExprId.put(slotRef.getDesc().getId(), slotReference.getExprId()); - slotReferenceToSlotRef.put(slotReference, slotRef); + public void addExprIdSlotRefPair(ExprId exprId, SlotRef slotRef) { + exprIdToSlotRef.put(exprId, slotRef); + slotIdToExprId.put(slotRef.getDesc().getId(), exprId); } public void addExprIdColumnRefPair(ExprId exprId, ColumnRefExpr columnRefExpr) { @@ -220,10 +218,6 @@ public SlotRef findSlotRef(ExprId exprId) { return exprIdToSlotRef.get(exprId); } - public SlotRef findSlotRef(SlotReference slotReference) { - return slotReferenceToSlotRef.get(slotReference); - } - public ColumnRefExpr findColumnRef(ExprId exprId) { return exprIdToColumnRef.get(exprId); } @@ -279,7 +273,7 @@ public SlotDescriptor createSlotDesc(TupleDescriptor tupleDesc, SlotReference sl } slotRef.setTable(table); slotRef.setLabel(slotReference.getName()); - this.addExprIdSlotRefPair(slotReference, slotRef); + this.addExprIdSlotRefPair(slotReference.getExprId(), slotRef); slotDescriptor.setIsNullable(slotReference.nullable()); return slotDescriptor; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java index f8a9bafe3c7b920..6b162cf0afae870 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java @@ -28,6 +28,11 @@ /** * HyperGraph Node. + * Jc + * \ + * F + * \ + * JC */ public class Node { private final int index; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index 8009156e1340638..347a479a3ce7a23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.cost.Cost; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.rewrite.mv.StructInfo; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; @@ -74,6 +75,8 @@ public class Group { private int chosenGroupExpressionId = -1; + private Optional structInfo = Optional.empty(); + /** * Constructor for Group. * @@ -152,6 +155,7 @@ public GroupExpression logicalExpressionsAt(int index) { * @return the first logical group expression in this group */ public GroupExpression getLogicalExpression() { + // poc tmp Preconditions.checkArgument(logicalExpressions.size() == 1, "There should be only one Logical Expression in Group"); return logicalExpressions.get(0); @@ -532,4 +536,12 @@ public String treeString() { return TreeStringUtils.treeString(this, toString, getChildren, getExtraPlans, displayExtraPlan); } + + public Optional getStructInfo() { + return structInfo; + } + + public void setStructInfo(StructInfo structInfo) { + this.structInfo = Optional.ofNullable(structInfo); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index 5b75cb2d76e1934..9e0419fae3b27dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -314,6 +314,7 @@ private Group init(Plan plan) { plan = replaceChildrenToGroupPlan(plan, childrenGroups); GroupExpression newGroupExpression = new GroupExpression(plan, childrenGroups); Group group = new Group(groupIdGenerator.getNextId(), newGroupExpression, plan.getLogicalProperties()); + // PoC add struct info to group groups.put(group.getGroupId(), group); if (groupExpressions.containsKey(newGroupExpression)) { @@ -323,6 +324,22 @@ private Group init(Plan plan) { return group; } + /** initPoC */ + public Group initPoC(Plan plan) { + Preconditions.checkArgument(!(plan instanceof GroupPlan), "Cannot init memo by a GroupPlan"); + + /* initialize children recursively */ + List childrenGroups = new ArrayList<>(plan.arity()); + for (Plan child : plan.children()) { + childrenGroups.add(initPoC(child)); + } + + plan = replaceChildrenToGroupPlan(plan, childrenGroups); + GroupExpression newGroupExpression = new GroupExpression(plan, childrenGroups); + Group group = new Group(groupIdGenerator.getNextId(), newGroupExpression, plan.getLogicalProperties()); + return group; + } + /** * add or replace the plan into the target group. *

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java index 15516e0501f490b..381f66fb6395c6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java @@ -83,7 +83,7 @@ private Plan deferMaterialize(LogicalResultSink logicalResultSin LogicalTopN logicalTopN, Optional> logicalFilter, LogicalOlapScan logicalOlapScan) { Column rowId = new Column(Column.ROWID_COL, Type.STRING, false, null, false, "", "rowid column"); - SlotReference columnId = SlotReference.fromColumn(rowId, logicalOlapScan.getQualifier()); + SlotReference columnId = SlotReference.fromColumn(rowId, logicalOlapScan.getQualifier(), null); Set deferredMaterializedExprIds = Sets.newHashSet(logicalOlapScan.getOutputExprIdSet()); logicalFilter.ifPresent(filter -> filter.getConjuncts() .forEach(e -> deferredMaterializedExprIds.removeAll(e.getInputSlotExprIds()))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewJoinRule.java index b7d42d6c22cc58c..f23e874630cd6d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewJoinRule.java @@ -21,19 +21,27 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitors; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitors.SlotReferenceReplacer.ExprReplacer; import com.google.common.collect.BiMap; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; /** @@ -45,26 +53,70 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali protected Plan rewriteView(MatchMode matchMode, StructInfo queryStructInfo, StructInfo viewStructInfo, - BiMap queryToViewTableMappings, - Plan temporaryRewrite) { + BiMap queryToViewTableMappings, + Plan tempRewritedPlan) { List expressions = rewriteExpression(queryStructInfo.getTopExpressions(), queryStructInfo, viewStructInfo, queryToViewTableMappings, - temporaryRewrite + tempRewritedPlan ); if (expressions == null) { return queryStructInfo.getPlan(); } - // TODO add rewrited project correctly - Map mvOutputSet = temporaryRewrite.getLogicalProperties().getOutputMap(); - // Simplify implement - List namedExpressions = queryStructInfo.getPlan().getLogicalProperties().getOutput() - .stream() - .map(slot -> (NamedExpression) mvOutputSet.get(slot)) - .collect(Collectors.toList()); - return new LogicalProject<>(namedExpressions, temporaryRewrite); + // PoC Generate mapping from query slot reference to mv slot reference, note: clone + // if any slot can not map then bail out + // simplfy implement + Set querySlotSet = new HashSet<>(); + queryStructInfo.getPlan().accept(PlanVisitors.SLOT_REFERENCE_COLLECTOR, querySlotSet); + + Set viewSlotSet = new HashSet<>(); + viewStructInfo.getPlan().accept(PlanVisitors.SLOT_REFERENCE_COLLECTOR, viewSlotSet); + + Map queryToViewSlotMapping = new HashMap<>(); + for (SlotReference querySlot : querySlotSet) { + for (SlotReference viewSlot : viewSlotSet) { + if (Objects.equals(querySlot.getName(), viewSlot.getName()) + && Objects.equals(querySlot.getQualifier(), viewSlot.getQualifier())) { + queryToViewSlotMapping.put(querySlot, viewSlot); + } + } + } + // PoC Generate mapping from mv sql output to mv scan out put + Map mvToMvScanMapping = new HashMap<>(); + List mvScanSlotList = tempRewritedPlan.getOutput(); + List mvSlotList = viewStructInfo.getPlan().getOutput(); + for (int i = 0; i < mvSlotList.size(); i++) { + mvToMvScanMapping.put((SlotReference) mvSlotList.get(i), (SlotReference) mvScanSlotList.get(i)); + } + + // TODO check if the query expr can derive from the view + // PoC If the query expression can get from mv sql, so replace the mv scan slot reference + // PoC according to the mapping above. Simplify implement + Map mvScanToQueryMapping = new HashMap<>(); + List output = queryStructInfo.getPlan().getOutput(); + for (Slot querySlot : output) { + Slot mvSlot = queryToViewSlotMapping.get(querySlot); + if (mvSlot == null) { + return null; + } + SlotReference mvScanSlot = mvToMvScanMapping.get(mvSlot); + if (mvScanSlot == null) { + return null; + } + mvScanToQueryMapping.put(mvScanSlot, querySlot); + } + // Replace the mv scan output with query slot, lazy before add filter and other project + + // tempRewritedPlan.accept(SlotReferenceReplacer.INSTANCE, mvScanToQueryMapping); + + tempRewritedPlan.getOutput().stream() + .forEach(slot -> slot.accept(ExprReplacer.INSTANCE, mvScanToQueryMapping)); + LogicalProject planLogicalProject = new LogicalProject<>( + output.stream().map(NamedExpression.class::cast).collect(Collectors.toList()), + tempRewritedPlan); + return planLogicalProject; } protected boolean isPatternSupport(LogicalProject topProject, Plan plan) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewRule.java index 46a6ae2caf8669a..59eefa20b7e0084 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewRule.java @@ -20,6 +20,9 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -29,6 +32,8 @@ import org.apache.doris.nereids.trees.metadata.Predicates; import org.apache.doris.nereids.trees.metadata.Predicates.SplitPredicate; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.algebra.Project; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -47,6 +52,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * AbstractMaterializedViewRule @@ -63,10 +69,19 @@ protected List rewrite(LogicalProject queryTopProject, Plan queryPlan, Cas if (!isPatternSupport(queryTopProject, queryPlan)) { return rewriteResults; } - StructInfo queryStructInfo = extractStructInfo(queryTopProject, queryPlan); + StructInfo queryStructInfo = extractStructInfo(queryTopProject, queryPlan, cascadesContext); if (!checkStructInfo(queryStructInfo)) { return rewriteResults; } + + // PoC hyper graph query + // HyperGraph queryHyperGraph = new HyperGraph(); + // Plan plan = queryStructInfo.getPlan(); + // plan = plan.accept(new PlanVisitors.GroupPlanRemover(), null); + // Group poCGroup = cascadesContext.getMemo().initPoC(plan); + // JoinOrderJob joinOrderJob = new JoinOrderJob(poCGroup, cascadesContext.getCurrentJobContext()); + // joinOrderJob.buildGraph(poCGroup, queryHyperGraph); + for (MaterializationContext materializationContext : materializationContexts) { Plan mvPlan = materializationContext.getMvPlan(); LogicalProject viewTopProject; @@ -88,17 +103,26 @@ protected List rewrite(LogicalProject queryTopProject, Plan queryPlan, Cas if (!isPatternSupport(viewTopProject, viewPlan)) { continue; } - StructInfo viewStructInfo = extractStructInfo(viewTopProject, viewPlan); + StructInfo viewStructInfo = extractStructInfo(viewTopProject, viewPlan, cascadesContext); if (!checkStructInfo(viewStructInfo)) { continue; } + + // Poc Hyper graph view + HyperGraph viewHyperGraph = new HyperGraph(); + Plan view = viewStructInfo.getPlan(); + view = view.accept(new PlanVisitors.GroupPlanRemover(), null); + Group poCViewGroup = cascadesContext.getMemo().initPoC(view); + JoinOrderJob viewJoinOrderJob = new JoinOrderJob(poCViewGroup, cascadesContext.getCurrentJobContext()); + viewJoinOrderJob.buildGraph(poCViewGroup, viewHyperGraph); + MatchMode matchMode = decideMatchMode(queryStructInfo, viewStructInfo); if (MatchMode.NOT_MATCH == matchMode) { continue; } - List> queryToViewTableMappings = + List> queryToViewTableMappings = generateRelationMap(queryStructInfo, viewStructInfo); - for (BiMap queryToViewTableMapping : queryToViewTableMappings) { + for (BiMap queryToViewTableMapping : queryToViewTableMappings) { Expression compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo, queryToViewTableMapping); if (compensatePredicates == null) { @@ -109,6 +133,7 @@ protected List rewrite(LogicalProject queryTopProject, Plan queryPlan, Cas && ((BooleanLiteral) compensatePredicates).getValue()) { rewritedPlan = viewScanNode; } else { + // try to compensate predicates by using mv scan, Poc is always true List rewriteCompensatePredicates = rewriteExpression( ImmutableList.of(compensatePredicates), queryStructInfo, @@ -135,7 +160,7 @@ protected List rewrite(LogicalProject queryTopProject, Plan queryPlan, Cas protected Plan rewriteView(MatchMode matchMode, StructInfo queryStructInfo, StructInfo viewStructInfo, - BiMap queryToViewTableMappings, + BiMap queryToViewTableMappings, Plan temporaryRewrite) { return temporaryRewrite; } @@ -143,7 +168,7 @@ protected Plan rewriteView(MatchMode matchMode, protected SplitPredicate rewriteExpression(SplitPredicate splitPredicate, StructInfo sourceStructInfo, StructInfo targetStructInfo, - BiMap sourceToTargetMapping, + BiMap sourceToTargetMapping, Plan targetScanNode) { // call below rewriteExpression return null; @@ -153,7 +178,7 @@ protected SplitPredicate rewriteExpression(SplitPredicate splitPredicate, protected List rewriteExpression(List sourceExpression, StructInfo sourceStructInfo, StructInfo targetStructInfo, - BiMap sourceToTargetMapping, + BiMap sourceToTargetMapping, Plan targetScanNode) { // Firstly, rewrite the target plan output slot using query with inverse mapping // and record the position(another way, according to unify bottom mapping and up to bottom mapping separately, @@ -173,13 +198,13 @@ protected List rewriteExpression(List sourceEx protected Expression predicatesCompensate( StructInfo queryStructInfo, StructInfo viewStructInfo, - BiMap queryToViewTableMapping + BiMap queryToViewTableMapping ) { // TODO Predicate compensate should be common and move to util + return BooleanLiteral.of(true); // Equal predicate compensate // EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass(); // EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass(); - return BooleanLiteral.of(true); // if (queryEquivalenceClass.isEmpty() // && !viewEquivalenceClass.isEmpty()) { // return null; @@ -219,36 +244,41 @@ protected Expression predicatesCompensate( } // Generate table mapping between query table to view table - protected List> generateRelationMap( + protected List> generateRelationMap( StructInfo queryStructInfo, StructInfo viewStrutInfo) { - List queryTables = new ArrayList<>(); - PlanVisitors.TABLE_COLLECTOR_INSTANCE.visit(queryStructInfo.getPlan(), queryTables); + List queryRelations = new ArrayList<>(); + PlanVisitors.TABLE_COLLECTOR_INSTANCE.visit(queryStructInfo.getPlan(), queryRelations); - List viewTables = new ArrayList<>(); - PlanVisitors.TABLE_COLLECTOR_INSTANCE.visit(viewStrutInfo.getPlan(), viewTables); + List viewRelations = new ArrayList<>(); + PlanVisitors.TABLE_COLLECTOR_INSTANCE.visit(viewStrutInfo.getPlan(), viewRelations); - Multimap queryTableIdMap = ArrayListMultimap.create(); - for (TableIf tableIf : queryTables) { - queryTableIdMap.put(tableIf, tableIf.getId()); + Multimap queryTableRelationIdMap = ArrayListMultimap.create(); + for (CatalogRelation relation : queryRelations) { + queryTableRelationIdMap.put(relation.getTable(), relation.getRelationId()); } - Multimap viewTableIdMap = ArrayListMultimap.create(); - for (TableIf tableIf : viewTables) { - viewTableIdMap.put(tableIf, tableIf.getId()); + Multimap viewTableRelationIdMap = ArrayListMultimap.create(); + for (CatalogRelation relation : viewRelations) { + viewTableRelationIdMap.put(relation.getTable(), relation.getRelationId()); } // TODO Just support 1:1 - BiMap mappingMap = HashBiMap.create(); - for (Map.Entry queryEntry : queryTableIdMap.entries()) { - Collection viewTableIds = viewTableIdMap.get(queryEntry.getKey()); + BiMap mappingMap = HashBiMap.create(); + for (Map.Entry queryEntry : queryTableRelationIdMap.entries()) { + Collection viewTableIds = viewTableRelationIdMap.get(queryEntry.getKey()); mappingMap.put(queryEntry.getValue(), viewTableIds.iterator().next()); } return ImmutableList.of(mappingMap); } protected MatchMode decideMatchMode(StructInfo queryStructInfo, StructInfo viewStructInfo) { - List queryTableRefs = queryStructInfo.getReferencedTables(); - List viewTableRefs = viewStructInfo.getReferencedTables(); - + List queryTableRefs = queryStructInfo.getRelations() + .stream() + .map(CatalogRelation::getTable).collect( + Collectors.toList()); + List viewTableRefs = viewStructInfo.getRelations() + .stream() + .map(CatalogRelation::getTable).collect( + Collectors.toList()); boolean sizeSame = viewTableRefs.size() == queryTableRefs.size(); boolean queryPartial = viewTableRefs.containsAll(queryTableRefs); boolean viewPartial = queryTableRefs.containsAll(viewTableRefs); @@ -265,7 +295,7 @@ protected MatchMode decideMatchMode(StructInfo queryStructInfo, StructInfo viewS } protected boolean checkStructInfo(StructInfo info) { - if (info.getReferencedTables().isEmpty()) { + if (info.getRelations().isEmpty()) { return false; } if (!info.getPredicates().getCanNotPulledUpPredicates().isEmpty()) { @@ -274,101 +304,57 @@ protected boolean checkStructInfo(StructInfo info) { return true; } - protected StructInfo extractStructInfo(LogicalProject topProject, Plan topInput) { - // Support rewrite TableType.OLAP currently - List queryTables = - PlanMetadataQuery.getTables(topInput, Sets.newHashSet(TableType.OLAP)); - - Predicates predicates = PlanMetadataQuery.getPredicates(topInput); - - Expression composedExpressions = predicates.composedExpression(); - SplitPredicate splitPredicate = Predicates.splitPredicates(composedExpressions); + protected StructInfo extractStructInfo(LogicalProject topProject, Plan topInput, CascadesContext cascadesContext) { + + Plan query = topProject == null ? topInput : topProject; + if (query.getGroupExpression().isPresent() + && query.getGroupExpression().get().getOwnerGroup().getStructInfo().isPresent()) { + Group belongGroup = query.getGroupExpression().get().getOwnerGroup(); + return belongGroup.getStructInfo().get(); + } else { + // PoC build hyper graph and set into group + HyperGraph hyperGraph = new HyperGraph(); + query = query.accept(new PlanVisitors.GroupPlanRemover(), null); + Group poCGroup = cascadesContext.getMemo().initPoC(query); + JoinOrderJob joinOrderJob = new JoinOrderJob(poCGroup, cascadesContext.getCurrentJobContext()); + joinOrderJob.buildGraph(poCGroup, hyperGraph); + + // Support rewrite TableType.OLAP currently + // get tables + List relations = + PlanMetadataQuery.getTables(topInput, Sets.newHashSet(TableType.OLAP)); + + Predicates predicates = PlanMetadataQuery.getPredicates(topInput); + + // get predicates (include join condition and filter) + Expression composedExpressions = predicates.composedExpression(); + SplitPredicate splitPredicate = Predicates.splitPredicates(composedExpressions); + + // construct equivalenceClass according to equals predicates + final EquivalenceClass equivalenceClass = new EquivalenceClass(); + List equalPredicates = + ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicates()); + for (Expression expression : equalPredicates) { + EqualTo equalTo = (EqualTo) expression; + equivalenceClass.addEquivalenceClass( + (SlotReference) equalTo.getArguments().get(0), + (SlotReference) equalTo.getArguments().get(1)); + } - final EquivalenceClass equivalenceClass = new EquivalenceClass(); - List equalPredicates = - ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicates()); - for (Expression expression : equalPredicates) { - EqualTo equalTo = (EqualTo) expression; - equivalenceClass.addEquivalenceClass( - (SlotReference) equalTo.getArguments().get(0), - (SlotReference) equalTo.getArguments().get(1)); + // set on current group and mv scan not set + StructInfo structInfo = StructInfo.of(relations, predicates, equivalenceClass, + topProject, topInput, hyperGraph); + if (query.getGroupExpression().isPresent()) { + query.getGroupExpression().get().getOwnerGroup().setStructInfo(structInfo); + } + return structInfo; } - return StructInfo.of(queryTables, predicates, equivalenceClass, topProject, topInput); } protected boolean isPatternSupport(LogicalProject topProject, Plan plan) { return false; } - /** - * StructInfo - */ - protected static final class StructInfo { - private final List referencedTables; - private final Predicates predicates; - private final EquivalenceClass equivalenceClass; - private final Project topProject; - private final Plan topInput; - - private StructInfo(List referencedTables, - Predicates predicates, - EquivalenceClass equivalenceClass, - Project topProject, - Plan topInput) { - this.referencedTables = referencedTables; - this.predicates = predicates; - this.equivalenceClass = equivalenceClass; - this.topProject = topProject; - this.topInput = topInput; - } - - public static StructInfo of(List referencedTables, - Predicates predicates, - EquivalenceClass equivalenceClass, - Project topProject, - Plan topInput) { - return new StructInfo(referencedTables, predicates, equivalenceClass, topProject, topInput); - } - - public List getReferencedTables() { - return referencedTables; - } - - public Predicates getPredicates() { - return predicates; - } - - public EquivalenceClass getEquivalenceClass() { - return equivalenceClass; - } - - public Plan getTopInput() { - return topInput; - } - - public Project getTopProject() { - return topProject; - } - - public List getTopExpressions() { - return getTopProject() == null ? extractReferences(topInput) : - getTopProject().getProjects(); - } - - public Plan getPlan() { - return getTopProject() == null ? getTopInput() : (Plan) getTopProject(); - } - - /** - * It returns a list of references to all columns in the node. - * If the node is an Aggregate, it returns only the list of references to the grouping columns. - * The returned list is immutable. - */ - private List extractReferences(Plan plan) { - return null; - } - } - /** * MatchMode */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/StructInfo.java new file mode 100644 index 000000000000000..63362bab8d9363d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/StructInfo.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.mv; + +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.metadata.EquivalenceClass; +import org.apache.doris.nereids.trees.metadata.Predicates; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.algebra.Project; + +import java.util.List; + +/** + * StructInfo + */ +public class StructInfo { + private final List relations; + private final Predicates predicates; + private final EquivalenceClass equivalenceClass; + private final Project topProject; + private final Plan topInput; + private final HyperGraph hyperGraph; + + private StructInfo(List relations, + Predicates predicates, + EquivalenceClass equivalenceClass, + Project topProject, + Plan topInput, + HyperGraph hyperGraph) { + this.relations = relations; + this.predicates = predicates; + this.equivalenceClass = equivalenceClass; + this.topProject = topProject; + this.topInput = topInput; + this.hyperGraph = hyperGraph; + } + + public static StructInfo of(List relations, + Predicates predicates, + EquivalenceClass equivalenceClass, + Project topProject, + Plan topInput, + HyperGraph hyperGraph) { + return new StructInfo(relations, predicates, equivalenceClass, topProject, topInput, hyperGraph); + } + + public List getRelations() { + return relations; + } + + public Predicates getPredicates() { + return predicates; + } + + public EquivalenceClass getEquivalenceClass() { + return equivalenceClass; + } + + public Plan getTopInput() { + return topInput; + } + + public Project getTopProject() { + return topProject; + } + + public HyperGraph getHyperGraph() { + return hyperGraph; + } + + public List getTopExpressions() { + return getTopProject() == null ? extractReferences(topInput) : + getTopProject().getProjects(); + } + + public Plan getPlan() { + return getTopProject() == null ? getTopInput() : (Plan) getTopProject(); + } + + /** + * It returns a list of references to all columns in the node. + * If the node is an Aggregate, it returns only the list of references to the grouping columns. + * The returned list is immutable. + */ + private List extractReferences(Plan plan) { + return null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java index 877f792e5019edb..86c89efdc9a03ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java @@ -75,7 +75,8 @@ public Slot toSlot() throws UnboundException { child() instanceof SlotReference ? ((SlotReference) child()).getColumn().orElse(null) : null, - nameFromChild ? Optional.of(child().toString()) : Optional.of(name)); + nameFromChild ? Optional.of(child().toString()) : Optional.of(name), + Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java index 226e8d1f9b25380..a7f1fe171ede293 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java @@ -142,7 +142,8 @@ public static class ArrayItemSlot extends SlotReference implements SlotNotFromCh * @param nullable true if nullable */ public ArrayItemSlot(ExprId exprId, String name, DataType dataType, boolean nullable) { - super(exprId, name, dataType, nullable, ImmutableList.of(), null, Optional.empty()); + super(exprId, name, dataType, nullable, ImmutableList.of(), null, Optional.empty(), + Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 634042289db6f3b..831bec709880e7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.DataType; import com.google.common.base.Preconditions; @@ -33,7 +34,7 @@ * Reference to slot in expression. */ public class SlotReference extends Slot { - protected final ExprId exprId; + protected ExprId exprId; protected final String name; protected final DataType dataType; protected final boolean nullable; @@ -43,29 +44,31 @@ public class SlotReference extends Slot { // different SlotReference will have different internalName // TODO: remove this member variable after mv selection is refactored protected final Optional internalName; - private final Column column; + private final Optional relationId; public SlotReference(String name, DataType dataType) { - this(StatementScopeIdGenerator.newExprId(), name, dataType, true, ImmutableList.of(), null, Optional.empty()); + this(StatementScopeIdGenerator.newExprId(), name, dataType, true, ImmutableList.of(), null, Optional.empty(), + Optional.empty()); } public SlotReference(String name, DataType dataType, boolean nullable) { - this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, ImmutableList.of(), - null, Optional.empty()); + this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, ImmutableList.of(), null, + Optional.empty(), Optional.empty()); } public SlotReference(String name, DataType dataType, boolean nullable, List qualifier) { - this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, qualifier, null, Optional.empty()); + this(StatementScopeIdGenerator.newExprId(), name, dataType, nullable, qualifier, null, Optional.empty(), + Optional.empty()); } public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, List qualifier) { - this(exprId, name, dataType, nullable, qualifier, null, Optional.empty()); + this(exprId, name, dataType, nullable, qualifier, null, Optional.empty(), Optional.empty()); } - public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, - List qualifier, @Nullable Column column) { - this(exprId, name, dataType, nullable, qualifier, column, Optional.empty()); + public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, List qualifier, + @Nullable Column column) { + this(exprId, name, dataType, nullable, qualifier, column, Optional.empty(), Optional.empty()); } /** @@ -79,8 +82,8 @@ public SlotReference(ExprId exprId, String name, DataType dataType, boolean null * @param column the column which this slot come from * @param internalName the internalName of this slot */ - public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, - List qualifier, @Nullable Column column, Optional internalName) { + public SlotReference(ExprId exprId, String name, DataType dataType, boolean nullable, List qualifier, + @Nullable Column column, Optional internalName, Optional relationId) { this.exprId = exprId; this.name = name; this.dataType = dataType; @@ -88,22 +91,27 @@ public SlotReference(ExprId exprId, String name, DataType dataType, boolean null this.nullable = nullable; this.column = column; this.internalName = internalName.isPresent() ? internalName : Optional.of(name); + this.relationId = relationId; } public static SlotReference of(String name, DataType type) { return new SlotReference(name, type); } - public static SlotReference fromColumn(Column column, List qualifier) { + public static SlotReference fromColumn(Column column, List qualifier, RelationId relationId) { DataType dataType = DataType.fromCatalogType(column.getType()); return new SlotReference(StatementScopeIdGenerator.newExprId(), column.getName(), dataType, - column.isAllowNull(), qualifier, column, Optional.empty()); + column.isAllowNull(), qualifier, column, Optional.empty(), Optional.ofNullable(relationId)); } - public static SlotReference fromColumn(Column column, String name, List qualifier) { + public static SlotReference fromColumn(Column column, String name, List qualifier, RelationId relationId) { DataType dataType = DataType.fromCatalogType(column.getType()); - return new SlotReference(StatementScopeIdGenerator.newExprId(), name, dataType, - column.isAllowNull(), qualifier, column, Optional.empty()); + return new SlotReference(StatementScopeIdGenerator.newExprId(), name, dataType, column.isAllowNull(), qualifier, + column, Optional.empty(), Optional.ofNullable(relationId)); + } + + public void changeExprId(ExprId id) { + this.exprId = id; } @Override @@ -174,21 +182,24 @@ public boolean equals(Object o) { // // For aa, the qualifier of aa in the subquery is empty, but in the output column of agg, // the qualifier of aa is t2. but both actually represent the same column. - // TODO PoC - return this.internalName.orElseGet(() -> "").equals(that.getInternalName()); + return exprId.equals(that.exprId); } // The contains method needs to use hashCode, so similar to equals, it only compares exprId @Override public int hashCode() { // direct return exprId to speed up - return Objects.hashCode(this.internalName.orElseGet(() -> "")); + return exprId.asInt(); } public Optional getColumn() { return Optional.ofNullable(column); } + public Optional getRelationId() { + return relationId; + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitSlotReference(this, context); @@ -205,22 +216,22 @@ public SlotReference withNullable(boolean newNullable) { if (this.nullable == newNullable) { return this; } - return new SlotReference(exprId, name, dataType, newNullable, qualifier, column, internalName); + return new SlotReference(exprId, name, dataType, newNullable, qualifier, column, internalName, relationId); } @Override public SlotReference withQualifier(List qualifier) { - return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName); + return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, relationId); } @Override public SlotReference withName(String name) { - return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName); + return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, relationId); } @Override public SlotReference withExprId(ExprId exprId) { - return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName); + return new SlotReference(exprId, name, dataType, nullable, qualifier, column, internalName, relationId); } public boolean isVisible() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/PlanMetadataQuery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/PlanMetadataQuery.java index fce9d6a5f5bc250..2c2676c3f99ad8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/PlanMetadataQuery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/PlanMetadataQuery.java @@ -17,10 +17,10 @@ package org.apache.doris.nereids.trees.metadata; -import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitors; import com.google.common.collect.ImmutableSet; @@ -47,10 +47,10 @@ public static Expression shuttleExpressionWithLineage(Plan plan, Expression expr return null; } - public static List getTables(Plan plan, Set targetTypes) { - List usedTables = new ArrayList<>(); - PlanVisitors.TABLE_COLLECTOR_INSTANCE.visit(plan, usedTables); - return usedTables; + public static List getTables(Plan plan, Set targetTypes) { + List relations = new ArrayList<>(); + PlanVisitors.TABLE_COLLECTOR_INSTANCE.visit(plan, relations); + return relations; } public static Predicates getPredicates(Plan plan) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index 4dd5121ef83e5f8..122d298ea29e57a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -86,7 +86,7 @@ public DatabaseIf getDatabase() throws AnalysisException { public List computeOutput() { return table.getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, qualified())) + .map(col -> SlotReference.fromColumn(col, qualified(), relationId)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 69166318989ddf6..227bdc1d19bb941 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -305,7 +305,7 @@ public List computeOutput() { if (cacheSlotWithSlotName.containsKey(Pair.of(selectedIndexId, col.getName()))) { return cacheSlotWithSlotName.get(Pair.of(selectedIndexId, col.getName())); } - Slot slot = SlotReference.fromColumn(col, qualified()); + Slot slot = SlotReference.fromColumn(col, qualified(), relationId); cacheSlotWithSlotName.put(Pair.of(selectedIndexId, col.getName()), slot); return slot; }).collect(ImmutableList.toImmutableList()); @@ -343,7 +343,7 @@ private Slot generateUniqueSlot(Column column, boolean isBaseIndex, long indexId if (cacheSlotWithSlotName.containsKey(Pair.of(indexId, name))) { return cacheSlotWithSlotName.get(Pair.of(indexId, name)); } - Slot slot = SlotReference.fromColumn(column, name, qualified()); + Slot slot = SlotReference.fromColumn(column, name, qualified(), relationId); cacheSlotWithSlotName.put(Pair.of(indexId, name), slot); return slot; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java index 4527ffa31626ca1..750357fa20eabcb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java @@ -98,7 +98,7 @@ public String toString() { public List computeOutput() { return function.getTable().getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, qualifier)) + .map(col -> SlotReference.fromColumn(col, qualifier, relationId)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java index 57e3b942212726b..67e65c302c82226 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java @@ -101,7 +101,7 @@ public DatabaseIf getDatabase() throws AnalysisException { public List computeOutput() { return table.getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, qualified())) + .map(col -> SlotReference.fromColumn(col, qualified(), relationId)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java index 955ea2f45da522d..9ba4098bb2c631e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java @@ -107,7 +107,7 @@ public String toString() { public List computeOutput() { return function.getTable().getBaseSchema() .stream() - .map(col -> SlotReference.fromColumn(col, ImmutableList.of())) + .map(col -> SlotReference.fromColumn(col, ImmutableList.of(), relationId)) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitors.java index 42d3db4801217b8..adf3d91f612bc88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitors.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitors.java @@ -17,10 +17,10 @@ package org.apache.doris.nereids.trees.plans.visitor; -import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.Nondeterministic; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; @@ -30,6 +30,7 @@ import org.apache.doris.nereids.util.ExpressionUtils; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -37,58 +38,27 @@ */ public class PlanVisitors { - public static final NondeterministicCollector NON_DETERMINISTIC_FUNCTION_INSTANCE = new NondeterministicCollector(); - public static final TableCollector TABLE_COLLECTOR_INSTANCE = new TableCollector(); + public static final TableScanCollector TABLE_COLLECTOR_INSTANCE = new TableScanCollector(); public static final PredicatesCollector PREDICATES_COLLECTOR_INSTANCE = new PredicatesCollector(); - - /** - * Collect the nondeterministic expr in plan, these expressions will be put into context - */ - public static class NondeterministicCollector extends DefaultPlanVisitor> { - @Override - public Void visit(Plan plan, List context) { - List expressions = plan.getExpressions(); - if (expressions == null) { - return super.visit(plan, context); - } - expressions.forEach(expression -> NondeterministicExprCollector.INSTANCE.visit(expression, context)); - return super.visit(plan, context); - } - - private static final class NondeterministicExprCollector - extends DefaultExpressionVisitor> { - private static final NondeterministicExprCollector INSTANCE = new NondeterministicExprCollector(); - - @Override - public Void visit(Expression expr, List context) { - if (expr == null) { - return null; - } - if (expr instanceof Nondeterministic) { - context.add(expr); - } - return super.visit(expr, context); - } - } - } + public static final SlotReferenceCollector SLOT_REFERENCE_COLLECTOR = new SlotReferenceCollector(); /** * Collect the table in plan * Note: will not get table if table is eliminated by EmptyRelation in rewrite. */ - public static class TableCollector extends DefaultPlanVisitor> { + public static class TableScanCollector extends DefaultPlanVisitor> { @Override - public Void visit(Plan plan, List context) { + public Void visit(Plan plan, List collectedRelations) { if (plan instanceof CatalogRelation) { - TableIf table = ((CatalogRelation) plan).getTable(); - context.add(table); + CatalogRelation catalogRelation = (CatalogRelation) plan; + collectedRelations.add(catalogRelation); } - return super.visit(plan, context); + return super.visit(plan, collectedRelations); } @Override - public Void visitGroupPlan(GroupPlan groupPlan, List context) { + public Void visitGroupPlan(GroupPlan groupPlan, List context) { Group group = groupPlan.getGroup(); // TODO Should record the struct info on the group? return group.getLogicalExpressions().get(0).getPlan().accept(this, context); @@ -132,4 +102,87 @@ public Void visitGroupPlan(GroupPlan groupPlan, List context) { return group.getLogicalExpressions().get(0).getPlan().accept(this, context); } } + + /** + * SlotReferenceCollector + */ + public static class SlotReferenceCollector + extends DefaultPlanVisitor> { + @Override + public Void visit(Plan plan, Set collectedExpressions) { + List expressions = plan.getExpressions(); + if (expressions.isEmpty()) { + return super.visit(plan, collectedExpressions); + } + expressions.forEach(expression -> { + if (expression instanceof SlotReference && ((SlotReference) expression).getRelationId() + .isPresent()) { + collectedExpressions.add((SlotReference) expression); + } + }); + return super.visit(plan, collectedExpressions); + } + + @Override + public Void visitGroupPlan(GroupPlan groupPlan, Set context) { + Group group = groupPlan.getGroup(); + // TODO Should record the struct info on the group? + return group.getLogicalExpressions().get(0).getPlan().accept(this, context); + } + } + + /** + * GroupPlanRemover + */ + public static class GroupPlanRemover + extends DefaultPlanRewriter { + + @Override + public Plan visitGroupPlan(GroupPlan groupPlan, Void context) { + Group group = groupPlan.getGroup(); + return group.getLogicalExpressions().get(0).getPlan(); + } + } + + /** + * SlotReferenceReplacer + */ + public static class SlotReferenceReplacer + extends DefaultPlanVisitor> { + + public static final SlotReferenceReplacer INSTANCE = new SlotReferenceReplacer(); + + @Override + public Plan visit(Plan plan, Map mvScanToQueryMapping) { + List slots = plan.getOutput(); + if (slots.isEmpty()) { + return super.visit(plan, mvScanToQueryMapping); + } + slots.forEach(slot -> slot.accept(ExprReplacer.INSTANCE, mvScanToQueryMapping)); + return super.visit(plan, mvScanToQueryMapping); + } + + @Override + public Plan visitGroupPlan(GroupPlan groupPlan, Map mvScanToQueryMapping) { + Group group = groupPlan.getGroup(); + // TODO Should record the struct info on the group? + return group.getLogicalExpressions().get(0).getPlan().accept(this, mvScanToQueryMapping); + } + + /** + * ExprReplacer + */ + public static class ExprReplacer extends DefaultExpressionVisitor> { + public static final ExprReplacer INSTANCE = new ExprReplacer(); + + @Override + public Void visitSlotReference(SlotReference slot, Map context) { + Slot mappedSlot = context.get(slot); + if (mappedSlot != null) { + slot.changeExprId(mappedSlot.getExprId()); + } + return super.visit(slot, context); + } + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewTest.java index ea5be915b0df632..517810d3e1e72aa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewTest.java @@ -85,6 +85,7 @@ protected void runBeforeAll() throws Exception { public void testInnerJoin() { connectContext.getSessionVariable().enableNereidsTimeout = false; + connectContext.getSessionVariable().enableDPHypOptimizer = true; // query only l_orderkey from join(lineitem, orders) will output l_orderkey and o_orderkey // PoC just use lineitem's field PlanChecker.from(connectContext) @@ -94,25 +95,8 @@ public void testInnerJoin() { "select l_shipdate, l_linenumber from mv", (queryPlanner, mvPlanner) -> { PhysicalPlan physicalPlan = queryPlanner.getPhysicalPlan(); - Assertions.assertTrue(((PhysicalResultSink) physicalPlan).toJson().toString().contains("mv")); - } - ); - } - - @Test - public void test2() { - - connectContext.getSessionVariable().enableNereidsTimeout = false; - // query only l_orderkey from join(lineitem, orders) will output l_orderkey and o_orderkey - // PoC just use lineitem's field - PlanChecker.from(connectContext) - .checkMVRewrite( - "select l_shipdate from lineitem inner join orders on l_orderkey = o_orderkey", - "select l1.l_linenumber, l2.l_linenumber from lineitem l1 inner join lineitem l2 on l1.l_orderkey = l2.l_orderkey", - "select l_shipdate, l_linenumber from mv", - (queryPlanner, mvPlanner) -> { - PhysicalPlan physicalPlan = queryPlanner.getPhysicalPlan(); - Assertions.assertTrue(((PhysicalResultSink) physicalPlan).toJson().toString().contains("mv")); + Assertions.assertTrue( + ((PhysicalResultSink) physicalPlan).toJson().toString().contains("mv")); } ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 16c9cbc3d569bee..6d5a8b0ba843763 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -606,6 +606,7 @@ public PlanChecker checkMVRewrite(String sql, String mvSql, String mvScanSql, if (queryStmtContext.getConnectContext().getTables() != null) { queryCascadesContext.setTables(queryStmtContext.getConnectContext().getTables()); } + MaterializationContext mvContext = new MaterializationContext( mvPlanner.getRewrittenPlan(), queryCascadesContext,