From db56257b488c0a39cca08daab2e4741c5c3e192e Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 27 Dec 2023 14:29:16 +0800 Subject: [PATCH] explain --- .../apache/doris/nereids/NereidsPlanner.java | 5 +- .../hypergraph/node/StructInfoNode.java | 6 +++ ...AbstractMaterializedViewAggregateRule.java | 28 ++++++++--- .../mv/AbstractMaterializedViewRule.java | 46 +++++++++++------- .../mv/LogicalCompatibilityContext.java | 36 ++++++++++++-- .../mv/MaterializationContext.java | 47 +++++++++++++++++++ .../rules/exploration/mv/Predicates.java | 14 ++++++ .../rules/exploration/mv/StructInfo.java | 2 +- .../mv/mapping/ExpressionMapping.java | 6 +++ .../aggregate_with_roll_up.groovy | 23 +++++++-- 10 files changed, 181 insertions(+), 32 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 4ac8223cc32f41..482421cec335e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -41,6 +41,7 @@ import org.apache.doris.nereids.processor.post.PlanPostProcessors; import org.apache.doris.nereids.processor.pre.PlanPreprocessors; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.exploration.mv.MaterializationContext; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -399,7 +400,9 @@ public String getExplainString(ExplainOptions explainOptions) { case MEMO_PLAN: plan = cascadesContext.getMemo().toString() + "\n\n========== OPTIMIZED PLAN ==========\n" - + optimizedPlan.treeString(); + + optimizedPlan.treeString() + + "\n\n========== MATERIALIZATIONS ==========\n" + + MaterializationContext.toString(cascadesContext.getMaterializationContexts()); break; case ALL_PLAN: plan = "========== PARSED PLAN ==========\n" diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java index 042e22fcf88a80..424ca6a5f10a6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; @@ -67,4 +68,9 @@ public List getGraphs() { return graphs; } + @Override + public String toString() { + return Utils.toSqlString("StructInfoNode[" + this.getName() + "]", + "plan", this.plan.treeString()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 685f8a8c3a9eab..ba8526ab2ffb54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.AbstractNode; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanSplitContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; @@ -35,10 +36,13 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion; import org.apache.doris.nereids.trees.expressions.functions.agg.CouldRollUp; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.planner.PlanNodeId; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; @@ -80,12 +84,17 @@ protected Plan rewriteQueryByView(MatchMode matchMode, MaterializationContext materializationContext) { // get view and query aggregate and top plan correspondingly Pair> viewTopPlanAndAggPair = splitToTopPlanAndAggregate(viewStructInfo); + ObjectId queryObjectId = queryStructInfo.getOriginalPlan().getGroupExpression() + .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); if (viewTopPlanAndAggPair == null) { + materializationContext.recordFailReason(queryObjectId, "split to view to top plan and agg fail"); logger.warn(currentClassName + " split to view to top plan and agg fail so return null"); return null; } Pair> queryTopPlanAndAggPair = splitToTopPlanAndAggregate(queryStructInfo); if (queryTopPlanAndAggPair == null) { + materializationContext.recordFailReason(queryObjectId, + "split to query to top plan and agg fail so return null"); logger.warn(currentClassName + " split to query to top plan and agg fail so return null"); return null; } @@ -115,7 +124,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, true); if (rewrittenQueryGroupExpr.isEmpty()) { // can not rewrite, bail out. - logger.debug(currentClassName + " can not rewrite expression when not need roll up"); + materializationContext.recordFailReason(queryObjectId, + "can not rewrite expression when not need roll up"); return null; } return new LogicalProject<>( @@ -130,13 +140,16 @@ protected Plan rewriteQueryByView(MatchMode matchMode, viewExpr -> viewExpr.anyMatch(expr -> expr instanceof AggregateFunction && ((AggregateFunction) expr).isDistinct()))) { // if mv aggregate function contains distinct, can not roll up, bail out. - logger.debug(currentClassName + " view contains distinct function so can not roll up"); + materializationContext.recordFailReason(queryObjectId, + "view contains distinct function so can not roll up"); return null; } // split the query top plan expressions to group expressions and functions, if can not, bail out. Pair, Set> queryGroupAndFunctionPair = topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair); if (queryGroupAndFunctionPair == null) { + materializationContext.recordFailReason(queryObjectId, + "query top plan split to group by and function fail"); logger.warn(currentClassName + " query top plan split to group by and function fail so return null"); return null; } @@ -175,7 +188,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, queryToViewSlotMapping, false); if (rewrittenFunctionExpression == null) { - logger.debug(currentClassName + " roll up expression can not rewrite by view so return null"); + materializationContext.recordFailReason(queryObjectId, + "roll up expression can not rewrite by view"); return null; } finalAggregateExpressions.add((NamedExpression) rewrittenFunctionExpression); @@ -185,8 +199,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, ExpressionUtils.shuttleExpressionWithLineage(topExpression, queryTopPlan); if (!mvExprToMvScanExprQueryBased.containsKey(queryGroupShuttledExpr)) { // group expr can not rewrite by view - logger.debug(currentClassName - + " view group expressions can not contains the query group by expression so return null"); + materializationContext.recordFailReason(queryObjectId, + "view group expressions doesn't not contains the query group by expression"); return null; } groupRewrittenExprMap.put(queryGroupShuttledExpr, @@ -199,8 +213,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, queryToViewSlotMapping, true); if (rewrittenGroupExpression == null) { - logger.debug(currentClassName - + " query top expression can not be rewritten by view so return null"); + materializationContext.recordFailReason(queryObjectId, + "query top group expression can not be rewritten by view"); return null; } finalAggregateExpressions.add((NamedExpression) rewrittenGroupExpression); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index c63e2d85af35ab..3571b593e2f5c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -30,6 +30,7 @@ import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.executor.Rewriter; +import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping; @@ -44,12 +45,15 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.planner.PlanNodeId; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; @@ -94,13 +98,13 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { logger.debug(currentClassName + " queryStructInfo is not valid so return"); return rewriteResults; } - + ObjectId queryObjectId = queryPlan.getGroupExpression() + .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); for (MaterializationContext materializationContext : materializationContexts) { // already rewrite, bail out if (queryPlan.getGroupExpression().isPresent() && materializationContext.alreadyRewrite( queryPlan.getGroupExpression().get().getOwnerGroup().getGroupId())) { - logger.debug(currentClassName + " this group is already rewritten so skip"); continue; } List viewStructInfos = extractStructInfo(materializationContext.getMvPlan(), @@ -112,34 +116,33 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { } StructInfo viewStructInfo = viewStructInfos.get(0); if (!checkPattern(viewStructInfo)) { - logger.debug(currentClassName + " viewStructInfo is not valid so return"); + materializationContext.recordFailReason(queryObjectId, "viewStructInfo is not valid"); continue; } MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations()); if (MatchMode.COMPLETE != matchMode) { - logger.debug(currentClassName + " match mode is not complete so return"); + materializationContext.recordFailReason(queryObjectId,"match mode is not 'COMPLETE'"); continue; } List queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(), viewStructInfo.getRelations()); // if any relation in query and view can not map, bail out. if (queryToViewTableMappings == null) { - logger.warn(currentClassName + " query to view table mapping null so return"); + materializationContext.recordFailReason(queryObjectId,"query to view table mapping null"); return rewriteResults; } for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); if (queryToViewSlotMapping == null) { - logger.warn(currentClassName + " query to view slot mapping null so continue"); + materializationContext.recordFailReason(queryObjectId,"query to view slot mapping null"); continue; } LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from(queryToViewTableMapping, queryToViewSlotMapping, - queryStructInfo, viewStructInfo); + queryStructInfo, viewStructInfo, materializationContext); ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext); if (comparisonResult.isInvalid()) { - logger.debug(currentClassName + " graph logical is not equals so continue"); continue; } // TODO: Use set of list? And consider view expr @@ -152,7 +155,9 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { queryToViewSlotMapping); // Can not compensate, bail out if (compensatePredicates.isEmpty()) { - logger.debug(currentClassName + " predicate compensate fail so continue"); + materializationContext.recordFailReason(queryObjectId,"predicate compensate fail," + + "query predicates = " + queryStructInfo.getPredicates() + "," + + "view predicates = " + viewStructInfo.getPredicates()); continue; } Plan rewrittenPlan; @@ -168,7 +173,10 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { queryToViewSlotMapping, true); if (rewriteCompensatePredicates.isEmpty()) { - logger.debug(currentClassName + " compensate predicate rewrite by view fail so continue"); + materializationContext.recordFailReason(queryObjectId,"rewrite compensate predicate fail," + + "compensatePredicates = " + compensatePredicates + "," + + "mvExprToMvScanExprMapping = " + + materializationContext.getMvExprToMvScanExprMapping()); continue; } rewrittenPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan); @@ -181,15 +189,14 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { rewrittenPlan, materializationContext); if (rewrittenPlan == null) { - logger.debug(currentClassName + " rewrite query by view fail so continue"); + materializationContext.recordFailReason(queryObjectId,"rewrite query by view fail"); continue; } if (!checkPartitionIsValid(queryStructInfo, materializationContext, cascadesContext)) { - logger.debug(currentClassName + " check partition validation fail so continue"); + materializationContext.recordFailReason(queryObjectId,"check partition validation fail"); continue; } - if (!checkOutput(queryPlan, rewrittenPlan)) { - logger.debug(currentClassName + " check output validation fail so continue"); + if (!checkOutput(queryPlan, rewrittenPlan, materializationContext)) { continue; } // run rbo job on mv rewritten plan @@ -198,16 +205,23 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { cascadesContext.getCurrentJobContext().getRequiredProperties()); Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute(); rewrittenPlan = rewrittenPlanContext.getRewritePlan(); - logger.debug(currentClassName + "rewrite by materialized view success"); + materializationContext.setSuccess(true); rewriteResults.add(rewrittenPlan); } } return rewriteResults; } - protected boolean checkOutput(Plan sourcePlan, Plan rewrittenPlan) { + protected boolean checkOutput(Plan sourcePlan, Plan rewrittenPlan, MaterializationContext materializationContext) { if (sourcePlan.getGroupExpression().isPresent() && !rewrittenPlan.getLogicalProperties().equals( sourcePlan.getGroupExpression().get().getOwnerGroup().getLogicalProperties())) { + ObjectId queryObjectId = sourcePlan.getGroupExpression() + .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); + materializationContext.recordFailReason(queryObjectId, + "rewrittenPlan output logical properties is not same with target group, " + + "planOutput: " + rewrittenPlan.getLogicalProperties() + "," + + "groupOutput: " + sourcePlan.getGroupExpression().get() + .getOwnerGroup().getLogicalProperties()); logger.error("rewrittenPlan output logical properties is not same with target group"); return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index 947f117acff907..daf4c2d05e6439 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.exploration.mv.mapping.Mapping.MappedRelation; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; @@ -26,8 +27,12 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.planner.PlanNodeId; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; @@ -42,12 +47,19 @@ public class LogicalCompatibilityContext { private final BiMap queryToViewNodeMapping; private final BiMap queryToViewEdgeExpressionMapping; private final BiMap queryToViewNodeIDMapping; + private final MaterializationContext materializationContext; + private final ObjectId planNodeId; public LogicalCompatibilityContext(BiMap queryToViewNodeMapping, - BiMap queryToViewEdgeExpressionMapping) { + BiMap queryToViewEdgeExpressionMapping, + MaterializationContext materializationContext, + StructInfo queryStructInfo) { this.queryToViewNodeMapping = queryToViewNodeMapping; this.queryToViewEdgeExpressionMapping = queryToViewEdgeExpressionMapping; this.queryToViewNodeIDMapping = HashBiMap.create(); + this.materializationContext = materializationContext; + this.planNodeId = queryStructInfo.getOriginalPlan().getGroupExpression() + .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex())); } @@ -63,13 +75,23 @@ public BiMap getQueryToViewEdgeExpressionMapping() { return queryToViewEdgeExpressionMapping; } + public MaterializationContext getMaterializationContext() { + return materializationContext; + } + + + public ObjectId getPlanNodeId() { + return planNodeId; + } + /** * generate logical compatibility context */ public static LogicalCompatibilityContext from(RelationMapping relationMapping, SlotMapping slotMapping, StructInfo queryStructInfo, - StructInfo viewStructInfo) { + StructInfo viewStructInfo, + MaterializationContext materializationContext) { // init node mapping BiMap queryToViewNodeMapping = HashBiMap.create(); Map queryRelationIdStructInfoNodeMap @@ -105,7 +127,8 @@ public static LogicalCompatibilityContext from(RelationMapping relationMapping, queryToViewEdgeMapping.put(edge, viewExpr); } }); - return new LogicalCompatibilityContext(queryToViewNodeMapping, queryToViewEdgeMapping); + return new LogicalCompatibilityContext(queryToViewNodeMapping, queryToViewEdgeMapping, + materializationContext, queryStructInfo); } private static Expression orderSlotAsc(Expression expression) { @@ -130,4 +153,11 @@ public Expression visitEqualTo(EqualTo equalTo, Void context) { } } } + + @Override + public String toString() { + return Utils.toSqlString("LogicalCompatibilityContext", + "queryToViewNodeMapping", queryToViewNodeMapping.toString(), + "queryToViewEdgeExpressionMapping", queryToViewEdgeExpressionMapping.toString()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 153688ecc2d3dd..3cd78be8106e29 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -22,17 +22,23 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupId; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; +import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.planner.PlanNodeId; import com.google.common.collect.ImmutableList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -53,6 +59,11 @@ public class MaterializationContext { private boolean available = true; // the mv plan from cache at present, record it to make sure query rewrite by mv is right when cache change. private Plan mvPlan; + // mark rewrite success or not + private boolean success = false; + // if rewrite by mv fail, record the reason, if success the failReason should be empty. + // The key is the query belonged group expression objectId, the value is the fail reason + private final Map failReason = new HashMap<>(); /** * MaterializationContext, this contains necessary info for query rewriting by mv @@ -127,6 +138,42 @@ public Plan getMvPlan() { return mvPlan; } + public void setSuccess(boolean success) { + this.success = success; + this.failReason.clear(); + } + + public void recordFailReason(ObjectId objectId, String reason) { + // once success, do not record the fail reason + if (this.success) { + return; + } + this.success = false; + this.failReason.put(objectId, reason); + } + + @Override + public String toString() { + StringBuilder failReasonBuilder = new StringBuilder("[").append("\n"); + for (Map.Entry reason : this.failReason.entrySet()) { + failReasonBuilder.append(reason.getKey()).append(" : \n") + .append(reason.getValue()).append("\n"); + } + failReasonBuilder.append("\n").append("]"); + return Utils.toSqlString("MaterializationContext[" + mtmv.getName() + "]", + "rewriteSuccess", this.success, + "failReason", failReasonBuilder.toString()); + } + + public static String toString(List materializationContexts) { + StringBuilder builder = new StringBuilder(); + builder.append("materializationContexts:").append("\n"); + for (MaterializationContext ctx : materializationContexts) { + builder.append("\n\n").append(ctx).append("\n"); + } + return builder.toString(); + } + /** * MaterializationContext fromMaterializedView */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index 94799899b9ec45..0a25a603ae7d51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; @@ -71,6 +72,11 @@ public static SplitPredicate splitPredicates(Expression expression) { return predicatesSplit.getSplitPredicate(); } + @Override + public String toString() { + return Utils.toSqlString("Predicates", "pulledUpPredicates", pulledUpPredicates); + } + /** * The split different representation for predicate expression, such as equal, range and residual predicate. */ @@ -139,5 +145,13 @@ public boolean isAlwaysTrue() { && ((BooleanLiteral) rangeExpr).getValue() && ((BooleanLiteral) residualExpr).getValue(); } + + @Override + public String toString() { + return Utils.toSqlString("SplitPredicate", + "equalPredicate", equalPredicate, + "rangePredicate", rangePredicate, + "residualPredicate", residualPredicate); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 20da9ee12fdd69..ebfa2d649eea3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -101,7 +101,6 @@ private void init() { } collectStructInfoFromGraph(); initPredicates(); - predicatesDerive(); } public void addPredicates(List canPulledUpExpressions) { @@ -156,6 +155,7 @@ private void initPredicates() { Set topPlanPredicates = new HashSet<>(); topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates); topPlanPredicates.forEach(this.predicates::addPredicate); + predicatesDerive(); } // derive some useful predicate by predicates diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java index 2f1ed230145f35..5a5bfedfe170ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java @@ -20,6 +20,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableMultimap; @@ -123,4 +124,9 @@ public Mapping chainedFold(Mapping target) { } return new ExpressionMapping(foldedMappingBuilder.build()); } + + @Override + public String toString() { + return Utils.toSqlString("ExpressionMapping", "expressionMapping", expressionMapping); + } } diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index d6b5c37e59c095..caa796f23d991c 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -50,7 +50,7 @@ suite("aggregate_with_roll_up") { DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" - ) + ); """ sql """ @@ -84,7 +84,7 @@ suite("aggregate_with_roll_up") { DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" - ) + ); """ sql """ @@ -106,7 +106,16 @@ suite("aggregate_with_roll_up") { ) """ - sql """ insert into lineitem values + sql """ + insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-01', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-02', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-03', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-04', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-05', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-06', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-07', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-07', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), @@ -116,7 +125,13 @@ suite("aggregate_with_roll_up") { sql """ insert into orders values - (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 9.5, '2023-12-01', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-02', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-03', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-04', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-05', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-06', 'c','d',2, 'mm'), + (1, 1, 'o', 9.5, '2023-12-07', 'a', 'b', 1, 'yy'), (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),