From 4c301bb90fe281b382a942fcc0ecdcd498df8ee0 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 27 Dec 2023 17:32:15 +0800 Subject: [PATCH] [opt](nereids) explain add materialized view rewrite --- .../apache/doris/nereids/NereidsPlanner.java | 5 +- .../jobs/joinorder/hypergraph/edge/Edge.java | 16 +- .../hypergraph/node/StructInfoNode.java | 6 + ...AbstractMaterializedViewAggregateRule.java | 52 +++- .../mv/AbstractMaterializedViewJoinRule.java | 6 +- .../mv/AbstractMaterializedViewRule.java | 223 ++++++++---------- .../exploration/mv/ComparisonResult.java | 26 +- .../exploration/mv/EquivalenceClass.java | 5 + .../exploration/mv/HyperGraphComparator.java | 20 +- .../mv/LogicalCompatibilityContext.java | 25 +- .../mv/MaterializationContext.java | 47 ++++ .../rules/exploration/mv/Predicates.java | 14 ++ .../rules/exploration/mv/StructInfo.java | 11 +- .../mv/mapping/ExpressionMapping.java | 6 + .../rules/exploration/mv/mapping/Mapping.java | 5 + .../exploration/mv/mapping/SlotMapping.java | 5 + 16 files changed, 315 insertions(+), 157 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 4ac8223cc32f416..482421cec335e52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -41,6 +41,7 @@ import org.apache.doris.nereids.processor.post.PlanPostProcessors; import org.apache.doris.nereids.processor.pre.PlanPreprocessors; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.exploration.mv.MaterializationContext; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -399,7 +400,9 @@ public String getExplainString(ExplainOptions explainOptions) { case MEMO_PLAN: plan = cascadesContext.getMemo().toString() + "\n\n========== OPTIMIZED PLAN ==========\n" - + optimizedPlan.treeString(); + + optimizedPlan.treeString() + + "\n\n========== MATERIALIZATIONS ==========\n" + + MaterializationContext.toString(cascadesContext.getMaterializationContexts()); break; case ALL_PLAN: plan = "========== PARSED PLAN ==========\n" diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java index 9bdb9ac272fb44a..2530074931e8da7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java @@ -211,10 +211,22 @@ public Expression getExpression(int i) { return getExpressions().get(i); } + public String getTypeName() { + if (this instanceof FilterEdge) { + return "FILTER"; + } else { + return ((JoinEdge) this).getJoinType().toString(); + } + } + @Override public String toString() { - return String.format("<%s - %s>", LongBitmap.toString(leftExtendedNodes), LongBitmap.toString( - rightExtendedNodes)); + if (!leftRejectEdges.isEmpty() || !rightRejectEdges.isEmpty()) { + return String.format("<%s --%s-- %s>[%s , %s]", LongBitmap.toString(leftExtendedNodes), + this.getTypeName(), LongBitmap.toString(rightExtendedNodes), leftRejectEdges, rightRejectEdges); + } + return String.format("<%s --%s-- %s>", LongBitmap.toString(leftExtendedNodes), + this.getTypeName(), LongBitmap.toString(rightExtendedNodes)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java index 042e22fcf88a809..424ca6a5f10a6c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; @@ -67,4 +68,9 @@ public List getGraphs() { return graphs; } + @Override + public String toString() { + return Utils.toSqlString("StructInfoNode[" + this.getName() + "]", + "plan", this.plan.treeString()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 685f8a8c3a9eabc..fadb9538371073f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -81,12 +81,16 @@ protected Plan rewriteQueryByView(MatchMode matchMode, // get view and query aggregate and top plan correspondingly Pair> viewTopPlanAndAggPair = splitToTopPlanAndAggregate(viewStructInfo); if (viewTopPlanAndAggPair == null) { - logger.warn(currentClassName + " split to view to top plan and agg fail so return null"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("view split to top plan and agg fail, view plan = %s\n", + viewStructInfo.getOriginalPlan().treeString())); return null; } Pair> queryTopPlanAndAggPair = splitToTopPlanAndAggregate(queryStructInfo); if (queryTopPlanAndAggPair == null) { - logger.warn(currentClassName + " split to query to top plan and agg fail so return null"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("query split to top plan and agg fail, query plan = %s\n", + queryStructInfo.getOriginalPlan().treeString())); return null; } // Firstly, handle query group by expression rewrite @@ -115,7 +119,12 @@ protected Plan rewriteQueryByView(MatchMode matchMode, true); if (rewrittenQueryGroupExpr.isEmpty()) { // can not rewrite, bail out. - logger.debug(currentClassName + " can not rewrite expression when not need roll up"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("can not rewrite expression when need no roll up, expressionToWrite = %s,\n" + + "mvExprToMvScanExprMapping = %s,\n queryToViewSlotMapping = %s", + queryTopPlan.getExpressions(), + materializationContext.getMvExprToMvScanExprMapping(), + queryToViewSlotMapping)); return null; } return new LogicalProject<>( @@ -130,14 +139,19 @@ protected Plan rewriteQueryByView(MatchMode matchMode, viewExpr -> viewExpr.anyMatch(expr -> expr instanceof AggregateFunction && ((AggregateFunction) expr).isDistinct()))) { // if mv aggregate function contains distinct, can not roll up, bail out. - logger.debug(currentClassName + " view contains distinct function so can not roll up"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("view contains distinct function so can not roll up, view plan = %s", + viewAggregate.getOutputExpressions())); return null; } // split the query top plan expressions to group expressions and functions, if can not, bail out. Pair, Set> queryGroupAndFunctionPair = topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair); if (queryGroupAndFunctionPair == null) { - logger.warn(currentClassName + " query top plan split to group by and function fail so return null"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("query top plan split to group by and function fail," + + "queryTopPlan = %s,\n agg = %s", + queryTopPlanAndAggPair.key().treeString(), queryTopPlanAndAggPair.value().treeString())); return null; } // Secondly, try to roll up the agg functions @@ -164,18 +178,26 @@ protected Plan rewriteQueryByView(MatchMode matchMode, Function rollupAggregateFunction = rollup(queryFunction, queryFunctionShuttled, mvExprToMvScanExprQueryBased); if (rollupAggregateFunction == null) { + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("query function roll up fail, queryFunction = %s,\n" + + "mvExprToMvScanExprQueryBased = %s", + queryFunction, mvExprToMvScanExprQueryBased)); return null; } // key is query need roll up expr, value is mv scan based roll up expr needRollupExprMap.put(queryFunctionShuttled, rollupAggregateFunction); // rewrite query function expression by mv expression + ExpressionMapping needRollupExprMapping = new ExpressionMapping(needRollupExprMap); Expression rewrittenFunctionExpression = rewriteExpression(topExpression, queryTopPlan, - new ExpressionMapping(needRollupExprMap), + needRollupExprMapping, queryToViewSlotMapping, false); if (rewrittenFunctionExpression == null) { - logger.debug(currentClassName + " roll up expression can not rewrite by view so return null"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("roll up expression can not rewrite by view, topExpression = %s,\n" + + "needRollupExprMapping = %s,\n queryToViewSlotMapping = %s", + topExpression, needRollupExprMapping, queryToViewSlotMapping)); return null; } finalAggregateExpressions.add((NamedExpression) rewrittenFunctionExpression); @@ -185,22 +207,28 @@ protected Plan rewriteQueryByView(MatchMode matchMode, ExpressionUtils.shuttleExpressionWithLineage(topExpression, queryTopPlan); if (!mvExprToMvScanExprQueryBased.containsKey(queryGroupShuttledExpr)) { // group expr can not rewrite by view - logger.debug(currentClassName - + " view group expressions can not contains the query group by expression so return null"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("view group expressions doesn't not contains the query group by expression," + + "mvExprToMvScanExprQueryBased is %s,\nqueryGroupShuttledExpr is %s", + mvExprToMvScanExprQueryBased, queryGroupShuttledExpr)); return null; } groupRewrittenExprMap.put(queryGroupShuttledExpr, mvExprToMvScanExprQueryBased.get(queryGroupShuttledExpr)); // rewrite query group expression by mv expression + ExpressionMapping groupRewrittenExprMapping = new ExpressionMapping(groupRewrittenExprMap); Expression rewrittenGroupExpression = rewriteExpression( topExpression, queryTopPlan, - new ExpressionMapping(groupRewrittenExprMap), + groupRewrittenExprMapping, queryToViewSlotMapping, true); if (rewrittenGroupExpression == null) { - logger.debug(currentClassName - + " query top expression can not be rewritten by view so return null"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("query top group expression can not be rewritten by view," + + "topExpression is %s,\n groupRewrittenExprMapping is %s,\n" + + "queryToViewSlotMapping = %s", + topExpression, groupRewrittenExprMapping, queryToViewSlotMapping)); return null; } finalAggregateExpressions.add((NamedExpression) rewrittenGroupExpression); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 7825467628a2f3b..e33a533b7f0a068 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -60,7 +60,11 @@ protected Plan rewriteQueryByView(MatchMode matchMode, // Can not rewrite, bail out if (expressionsRewritten.isEmpty() || expressionsRewritten.stream().anyMatch(expr -> !(expr instanceof NamedExpression))) { - logger.warn(currentClassName + " expression to rewrite is not named expr so return null"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("join rewrite query by view fail, expressionToRewritten is %s,\n" + + "mvExprToMvScanExprMapping is %s,\n queryToViewSlotMapping = %s", + queryStructInfo.getExpressions(), materializationContext.getMvExprToMvScanExprMapping(), + queryToViewSlotMapping)); return null; } // record the group id in materializationContext, and when rewrite again in diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index c63e2d85af35ab8..2b12bbb98b733ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -30,6 +30,7 @@ import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.executor.Rewriter; +import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping; @@ -45,6 +46,7 @@ import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -70,8 +72,8 @@ * The abstract class for all materialized view rules */ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFactory { - public static final HashSet SUPPORTED_JOIN_TYPE_SET = - Sets.newHashSet(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN); + public static final HashSet SUPPORTED_JOIN_TYPE_SET = Sets.newHashSet(JoinType.INNER_JOIN, + JoinType.LEFT_OUTER_JOIN); protected final String currentClassName = this.getClass().getSimpleName(); private final Logger logger = LogManager.getLogger(this.getClass()); @@ -83,63 +85,65 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { List materializationContexts = cascadesContext.getMaterializationContexts(); List rewriteResults = new ArrayList<>(); if (materializationContexts.isEmpty()) { - logger.debug(currentClassName + " materializationContexts is empty so return"); return rewriteResults; } - List queryStructInfos = extractStructInfo(queryPlan, cascadesContext); // TODO Just Check query queryPlan firstly, support multi later. StructInfo queryStructInfo = queryStructInfos.get(0); if (!checkPattern(queryStructInfo)) { - logger.debug(currentClassName + " queryStructInfo is not valid so return"); + materializationContexts.forEach(ctx -> ctx.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("queryStructInfo is invalid, queryPlan is %s", queryPlan.treeString()))); return rewriteResults; } - for (MaterializationContext materializationContext : materializationContexts) { // already rewrite, bail out - if (queryPlan.getGroupExpression().isPresent() - && materializationContext.alreadyRewrite( + if (queryPlan.getGroupExpression().isPresent() && materializationContext.alreadyRewrite( queryPlan.getGroupExpression().get().getOwnerGroup().getGroupId())) { - logger.debug(currentClassName + " this group is already rewritten so skip"); continue; } - List viewStructInfos = extractStructInfo(materializationContext.getMvPlan(), - cascadesContext); + List viewStructInfos = extractStructInfo(materializationContext.getMvPlan(), cascadesContext); if (viewStructInfos.size() > 1) { // view struct info should only have one - logger.warn(currentClassName + " the num of view struct info is more then one so return"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("the num of view struct info is more then one, mv plan is %s", + materializationContext.getMvPlan().treeString())); return rewriteResults; } StructInfo viewStructInfo = viewStructInfos.get(0); if (!checkPattern(viewStructInfo)) { - logger.debug(currentClassName + " viewStructInfo is not valid so return"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("viewStructInfo is not valid, view plan is %s", + viewStructInfo.getOriginalPlan().treeString())); continue; } MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations()); if (MatchMode.COMPLETE != matchMode) { - logger.debug(currentClassName + " match mode is not complete so return"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + String.format("match mode is invalid, matchMode is %s", matchMode)); continue; } - List queryToViewTableMappings = - RelationMapping.generate(queryStructInfo.getRelations(), viewStructInfo.getRelations()); + List queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(), + viewStructInfo.getRelations()); // if any relation in query and view can not map, bail out. if (queryToViewTableMappings == null) { - logger.warn(currentClassName + " query to view table mapping null so return"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + "query to view table mapping null"); return rewriteResults; } for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); if (queryToViewSlotMapping == null) { - logger.warn(currentClassName + " query to view slot mapping null so continue"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + "query to view slot mapping null"); continue; } - LogicalCompatibilityContext compatibilityContext = - LogicalCompatibilityContext.from(queryToViewTableMapping, queryToViewSlotMapping, - queryStructInfo, viewStructInfo); + LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( + queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo); ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext); if (comparisonResult.isInvalid()) { - logger.debug(currentClassName + " graph logical is not equals so continue"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + comparisonResult.getErrorMessage()); continue; } // TODO: Use set of list? And consider view expr @@ -152,7 +156,13 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { queryToViewSlotMapping); // Can not compensate, bail out if (compensatePredicates.isEmpty()) { - logger.debug(currentClassName + " predicate compensate fail so continue"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), String.format( + "predicate compensate fail, query predicates = %s,\n query equivalenceClass = %s, \n" + + "view predicates = %s,\n query equivalenceClass = %s\n", + queryStructInfo.getPredicates(), + queryStructInfo.getEquivalenceClass(), + viewStructInfo.getPredicates(), + viewStructInfo.getEquivalenceClass())); continue; } Plan rewrittenPlan; @@ -161,54 +171,55 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { rewrittenPlan = mvScan; } else { // Try to rewrite compensate predicates by using mv scan - List rewriteCompensatePredicates = rewriteExpression( - compensatePredicates.toList(), - queryPlan, - materializationContext.getMvExprToMvScanExprMapping(), - queryToViewSlotMapping, + List rewriteCompensatePredicates = rewriteExpression(compensatePredicates.toList(), + queryPlan, materializationContext.getMvExprToMvScanExprMapping(), queryToViewSlotMapping, true); if (rewriteCompensatePredicates.isEmpty()) { - logger.debug(currentClassName + " compensate predicate rewrite by view fail so continue"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), String.format( + "rewrite compensate predicate fail, compensatePredicates = %s,\n" + + "mvExprToMvScanExprMapping = %s\n, queryToViewSlotMapping = %s", + compensatePredicates, materializationContext.getMvExprToMvScanExprMapping(), + queryToViewSlotMapping)); continue; } rewrittenPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan); } // Rewrite query by view - rewrittenPlan = rewriteQueryByView(matchMode, - queryStructInfo, - viewStructInfo, - queryToViewSlotMapping, - rewrittenPlan, - materializationContext); + rewrittenPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo, queryToViewSlotMapping, + rewrittenPlan, materializationContext); if (rewrittenPlan == null) { - logger.debug(currentClassName + " rewrite query by view fail so continue"); continue; } if (!checkPartitionIsValid(queryStructInfo, materializationContext, cascadesContext)) { - logger.debug(currentClassName + " check partition validation fail so continue"); + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + "check partition validation fail"); continue; } - if (!checkOutput(queryPlan, rewrittenPlan)) { - logger.debug(currentClassName + " check output validation fail so continue"); + if (!checkOutput(queryPlan, rewrittenPlan, materializationContext)) { continue; } // run rbo job on mv rewritten plan - CascadesContext rewrittenPlanContext = - CascadesContext.initContext(cascadesContext.getStatementContext(), rewrittenPlan, - cascadesContext.getCurrentJobContext().getRequiredProperties()); + CascadesContext rewrittenPlanContext = CascadesContext.initContext( + cascadesContext.getStatementContext(), rewrittenPlan, + cascadesContext.getCurrentJobContext().getRequiredProperties()); Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute(); rewrittenPlan = rewrittenPlanContext.getRewritePlan(); - logger.debug(currentClassName + "rewrite by materialized view success"); + materializationContext.setSuccess(true); rewriteResults.add(rewrittenPlan); } } return rewriteResults; } - protected boolean checkOutput(Plan sourcePlan, Plan rewrittenPlan) { - if (sourcePlan.getGroupExpression().isPresent() && !rewrittenPlan.getLogicalProperties().equals( - sourcePlan.getGroupExpression().get().getOwnerGroup().getLogicalProperties())) { - logger.error("rewrittenPlan output logical properties is not same with target group"); + protected boolean checkOutput(Plan sourcePlan, Plan rewrittenPlan, MaterializationContext materializationContext) { + if (sourcePlan.getGroupExpression().isPresent() && !rewrittenPlan.getLogicalProperties() + .equals(sourcePlan.getGroupExpression().get().getOwnerGroup().getLogicalProperties())) { + ObjectId planObjId = sourcePlan.getGroupExpression().map(GroupExpression::getId) + .orElseGet(() -> new ObjectId(-1)); + materializationContext.recordFailReason(planObjId, String.format( + "rewrittenPlan output logical properties is not same with target group," + + "planOutput = %s, groupOutput = %s", rewrittenPlan.getLogicalProperties(), + sourcePlan.getGroupExpression().get().getOwnerGroup().getLogicalProperties())); return false; } return true; @@ -220,9 +231,7 @@ protected boolean checkOutput(Plan sourcePlan, Plan rewrittenPlan) { * Maybe only just some partitions is valid in materialized view, so we should check if the mv can * offer the partitions which query used or not. */ - protected boolean checkPartitionIsValid( - StructInfo queryInfo, - MaterializationContext materializationContext, + protected boolean checkPartitionIsValid(StructInfo queryInfo, MaterializationContext materializationContext, CascadesContext cascadesContext) { // check partition is valid or not MTMV mtmv = materializationContext.getMTMV(); @@ -240,8 +249,7 @@ protected boolean checkPartitionIsValid( Optional relatedTableRelation = queryInfo.getRelations().stream() .filter(LogicalOlapScan.class::isInstance) .filter(relation -> relatedPartitionTable.equals(new BaseTableInfo(relation.getTable()))) - .map(LogicalOlapScan.class::cast) - .findFirst(); + .map(LogicalOlapScan.class::cast).findFirst(); if (!relatedTableRelation.isPresent()) { logger.warn("mv is partition update, but related table relation is null"); return false; @@ -263,37 +271,29 @@ protected boolean checkPartitionIsValid( return false; } // get mv related table valid partitions - Set relatedTalbeValidSet = mvDataValidPartitions.stream() - .map(partition -> { - Set relatedBaseTablePartitions = mvToBasePartitionMap.get(partition.getId()); - if (relatedBaseTablePartitions == null || relatedBaseTablePartitions.isEmpty()) { - return ImmutableList.of(); - } else { - return relatedBaseTablePartitions; - } - }) - .flatMap(Collection::stream) - .map(Long.class::cast) - .collect(Collectors.toSet()); + Set relatedTalbeValidSet = mvDataValidPartitions.stream().map(partition -> { + Set relatedBaseTablePartitions = mvToBasePartitionMap.get(partition.getId()); + if (relatedBaseTablePartitions == null || relatedBaseTablePartitions.isEmpty()) { + return ImmutableList.of(); + } else { + return relatedBaseTablePartitions; + } + }).flatMap(Collection::stream).map(Long.class::cast).collect(Collectors.toSet()); // get query selected partitions to make the partitions is valid or not - Set relatedTableSelectedPartitionToCheck = - new HashSet<>(relatedTableRelation.get().getSelectedPartitionIds()); + Set relatedTableSelectedPartitionToCheck = new HashSet<>( + relatedTableRelation.get().getSelectedPartitionIds()); if (relatedTableSelectedPartitionToCheck.isEmpty()) { relatedTableSelectedPartitionToCheck.addAll(relatedTable.getPartitionIds()); } - return !relatedTalbeValidSet.isEmpty() - && relatedTalbeValidSet.containsAll(relatedTableSelectedPartitionToCheck); + return !relatedTalbeValidSet.isEmpty() && relatedTalbeValidSet.containsAll( + relatedTableSelectedPartitionToCheck); } /** * Rewrite query by view, for aggregate or join rewriting should be different inherit class implementation */ - protected Plan rewriteQueryByView(MatchMode matchMode, - StructInfo queryStructInfo, - StructInfo viewStructInfo, - SlotMapping queryToViewSlotMapping, - Plan tempRewritedPlan, - MaterializationContext materializationContext) { + protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInfo, StructInfo viewStructInfo, + SlotMapping queryToViewSlotMapping, Plan tempRewritedPlan, MaterializationContext materializationContext) { return tempRewritedPlan; } @@ -306,11 +306,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, * the key expression in targetExpressionMapping should be shuttled. with the method * ExpressionUtils.shuttleExpressionWithLineage. */ - protected List rewriteExpression( - List sourceExpressionsToWrite, - Plan sourcePlan, - ExpressionMapping targetExpressionMapping, - SlotMapping sourceToTargetMapping, + protected List rewriteExpression(List sourceExpressionsToWrite, Plan sourcePlan, + ExpressionMapping targetExpressionMapping, SlotMapping sourceToTargetMapping, boolean targetExpressionNeedSourceBased) { // Firstly, rewrite the target expression using source with inverse mapping // then try to use the target expression to represent the query. if any of source expressions @@ -325,13 +322,12 @@ protected List rewriteExpression( // project(slot 2, 1) // target // generate target to target replacement expression mapping, and change target expression to source based - List sourceShuttledExpressions = - ExpressionUtils.shuttleExpressionWithLineage(sourceExpressionsToWrite, sourcePlan); + List sourceShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( + sourceExpressionsToWrite, sourcePlan); ExpressionMapping expressionMappingKeySourceBased = targetExpressionNeedSourceBased ? targetExpressionMapping.keyPermute(sourceToTargetMapping.inverse()) : targetExpressionMapping; // target to target replacement expression mapping, because mv is 1:1 so get first element - List> flattenExpressionMap = - expressionMappingKeySourceBased.flattenMap(); + List> flattenExpressionMap = expressionMappingKeySourceBased.flattenMap(); Map targetToTargetReplacementMapping = flattenExpressionMap.get(0); List rewrittenExpressions = new ArrayList<>(); @@ -341,8 +337,8 @@ protected List rewriteExpression( rewrittenExpressions.add(expressionToRewrite); continue; } - final Set slotsToRewrite = - expressionToRewrite.collectToSet(expression -> expression instanceof Slot); + final Set slotsToRewrite = expressionToRewrite.collectToSet( + expression -> expression instanceof Slot); Expression replacedExpression = ExpressionUtils.replace(expressionToRewrite, targetToTargetReplacementMapping); if (replacedExpression.anyMatch(slotsToRewrite::contains)) { @@ -360,11 +356,8 @@ protected List rewriteExpression( return rewrittenExpressions; } - protected Expression rewriteExpression( - Expression sourceExpressionsToWrite, - Plan sourcePlan, - ExpressionMapping targetExpressionMapping, - SlotMapping sourceToTargetMapping, + protected Expression rewriteExpression(Expression sourceExpressionsToWrite, Plan sourcePlan, + ExpressionMapping targetExpressionMapping, SlotMapping sourceToTargetMapping, boolean targetExpressionNeedSourceBased) { List expressionToRewrite = new ArrayList<>(); expressionToRewrite.add(sourceExpressionsToWrite); @@ -382,11 +375,8 @@ protected Expression rewriteExpression( * For another example as following: * predicate a = b in mv, and a = b and c = d in query, the compensatory predicate is c = d */ - protected SplitPredicate predicatesCompensate( - StructInfo queryStructInfo, - StructInfo viewStructInfo, - SlotMapping queryToViewSlotMapping - ) { + protected SplitPredicate predicatesCompensate(StructInfo queryStructInfo, StructInfo viewStructInfo, + SlotMapping queryToViewSlotMapping) { EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass(); EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass(); // viewEquivalenceClass to query based @@ -394,24 +384,20 @@ protected SplitPredicate predicatesCompensate( .toSlotReferenceMap(); EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMapping); if (viewEquivalenceClassQueryBased == null) { - logger.info(currentClassName + " permute view equivalence class by query fail so return empty"); return SplitPredicate.empty(); } final List equalCompensateConjunctions = new ArrayList<>(); if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) { equalCompensateConjunctions.add(BooleanLiteral.of(true)); } - if (queryEquivalenceClass.isEmpty() - && !viewEquivalenceClass.isEmpty()) { - logger.info(currentClassName + " view has equivalence class but query not so return empty"); + if (queryEquivalenceClass.isEmpty() && !viewEquivalenceClass.isEmpty()) { return SplitPredicate.empty(); } - EquivalenceClassSetMapping queryToViewEquivalenceMapping = - EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased); + EquivalenceClassSetMapping queryToViewEquivalenceMapping = EquivalenceClassSetMapping.generate( + queryEquivalenceClass, viewEquivalenceClassQueryBased); // can not map all target equivalence class, can not compensate if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size() < viewEquivalenceClass.getEquivalenceSetList().size()) { - logger.info(currentClassName + " view has more equivalence than query so return empty"); return SplitPredicate.empty(); } // do equal compensate @@ -449,17 +435,14 @@ protected SplitPredicate predicatesCompensate( List rangeCompensate = new ArrayList<>(); Expression queryRangePredicate = querySplitPredicate.getRangePredicate(); Expression viewRangePredicate = viewSplitPredicate.getRangePredicate(); - Expression viewRangePredicateQueryBased = - ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping); + Expression viewRangePredicateQueryBased = ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping); - Set queryRangeSet = - Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate)); - Set viewRangeQueryBasedSet = - Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased)); + Set queryRangeSet = Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate)); + Set viewRangeQueryBasedSet = Sets.newHashSet( + ExpressionUtils.extractConjunction(viewRangePredicateQueryBased)); // query range predicate can not contain all view range predicate when view have range predicate, bail out - if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE) - && !queryRangeSet.containsAll(viewRangeQueryBasedSet)) { - logger.info(currentClassName + " query range predicate set can not contains all view range predicate"); + if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE) && !queryRangeSet.containsAll( + viewRangeQueryBasedSet)) { return SplitPredicate.empty(); } queryRangeSet.removeAll(viewRangeQueryBasedSet); @@ -477,10 +460,8 @@ protected SplitPredicate predicatesCompensate( Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased)); // query residual predicate can not contain all view residual predicate when view have residual predicate, // bail out - if (!viewResidualPredicateQueryBased.equals(BooleanLiteral.TRUE) - && !queryResidualSet.containsAll(viewResidualQueryBasedSet)) { - logger.info( - currentClassName + " query residual predicate set can not contains all view residual predicate"); + if (!viewResidualPredicateQueryBased.equals(BooleanLiteral.TRUE) && !queryResidualSet.containsAll( + viewResidualQueryBasedSet)) { return SplitPredicate.empty(); } queryResidualSet.removeAll(viewResidualQueryBasedSet); @@ -497,13 +478,9 @@ protected SplitPredicate predicatesCompensate( * @see MatchMode */ private MatchMode decideMatchMode(List queryRelations, List viewRelations) { - List queryTableRefs = queryRelations - .stream() - .map(CatalogRelation::getTable) + List queryTableRefs = queryRelations.stream().map(CatalogRelation::getTable) .collect(Collectors.toList()); - List viewTableRefs = viewRelations - .stream() - .map(CatalogRelation::getTable) + List viewTableRefs = viewRelations.stream().map(CatalogRelation::getTable) .collect(Collectors.toList()); boolean sizeSame = viewTableRefs.size() == queryTableRefs.size(); boolean queryPartial = viewTableRefs.containsAll(queryTableRefs); @@ -524,8 +501,8 @@ private MatchMode decideMatchMode(List queryRelations, List extractStructInfo(Plan plan, CascadesContext cascadesContext) { - if (plan.getGroupExpression().isPresent() - && !plan.getGroupExpression().get().getOwnerGroup().getStructInfos().isEmpty()) { + if (plan.getGroupExpression().isPresent() && !plan.getGroupExpression().get().getOwnerGroup().getStructInfos() + .isEmpty()) { return plan.getGroupExpression().get().getOwnerGroup().getStructInfos(); } else { // build struct info and add them to current group diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java index 8836745465ec022..90c3491fe6a258c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -31,24 +32,23 @@ * comparison result of view and query */ public class ComparisonResult { - public static final ComparisonResult INVALID = - new ComparisonResult(ImmutableList.of(), ImmutableList.of(), ImmutableSet.of(), false); private final boolean valid; private final List viewExpressions; private final List queryExpressions; private final Set> viewNoNullableSlot; - - public ComparisonResult(List queryExpressions, List viewExpressions, - Set> viewNoNullableSlot) { - this(queryExpressions, viewExpressions, viewNoNullableSlot, true); - } + private final String errorMessage; ComparisonResult(List queryExpressions, List viewExpressions, - Set> viewNoNullableSlot, boolean valid) { + Set> viewNoNullableSlot, boolean valid, String message) { this.viewExpressions = ImmutableList.copyOf(viewExpressions); this.queryExpressions = ImmutableList.copyOf(queryExpressions); this.viewNoNullableSlot = ImmutableSet.copyOf(viewNoNullableSlot); this.valid = valid; + this.errorMessage = message; + } + + public static ComparisonResult newInvalidResWithErrorMessage(String errorMessage) { + return new ComparisonResult(ImmutableList.of(), ImmutableList.of(), ImmutableSet.of(), false, errorMessage); } public List getViewExpressions() { @@ -67,6 +67,10 @@ public boolean isInvalid() { return !valid; } + public String getErrorMessage() { + return errorMessage; + } + /** * Builder */ @@ -109,11 +113,9 @@ public boolean isInvalid() { } public ComparisonResult build() { - if (isInvalid()) { - return ComparisonResult.INVALID; - } + Preconditions.checkArgument(valid, "Comparison result must be valid"); return new ComparisonResult(queryBuilder.build(), viewBuilder.build(), - viewNoNullableSlotBuilder.build(), valid); + viewNoNullableSlotBuilder.build(), valid, ""); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java index 23249cf5ae7817c..d6b59b6866c307a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java @@ -141,4 +141,9 @@ public List> getEquivalenceSetList() { this.equivalenceSlotList = equivalenceSets; return this.equivalenceSlotList; } + + @Override + public String toString() { + return "EquivalenceClass{" + "equivalenceSlotMap=" + equivalenceSlotMap + '}'; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index a869fe729a94f18..a0490eb29320070 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -67,7 +67,7 @@ public class HyperGraphComparator { private final Map> pullUpViewExprWithEdge = new HashMap<>(); private final LogicalCompatibilityContext logicalCompatibilityContext; - HyperGraphComparator(HyperGraph queryHyperGraph, HyperGraph viewHyperGraph, + public HyperGraphComparator(HyperGraph queryHyperGraph, HyperGraph viewHyperGraph, LogicalCompatibilityContext logicalCompatibilityContext) { this.queryHyperGraph = queryHyperGraph; this.viewHyperGraph = viewHyperGraph; @@ -109,13 +109,13 @@ private ComparisonResult buildComparisonRes() { ComparisonResult.Builder builder = new ComparisonResult.Builder(); for (Entry> e : pullUpQueryExprWithEdge.entrySet()) { if (!e.getValue().isEmpty() && !canPullUp(e.getKey())) { - return ComparisonResult.INVALID; + return ComparisonResult.newInvalidResWithErrorMessage(getErrorMessage() + "\nwith error edge " + e); } builder.addQueryExpressions(e.getValue()); } for (Entry> e : pullUpViewExprWithEdge.entrySet()) { if (!e.getValue().isEmpty() && !canPullUp(e.getKey())) { - return ComparisonResult.INVALID; + return ComparisonResult.newInvalidResWithErrorMessage(getErrorMessage() + "\nwith error edge " + e); } builder.addViewExpressions(e.getValue()); } @@ -125,6 +125,20 @@ private ComparisonResult buildComparisonRes() { return builder.build(); } + /** + * get error message + */ + public String getErrorMessage() { + return String.format( + "graph logical is not equal, query join edges is %s,\n" + "query filter edges is %s,\n" + + "view join edges is %s,\n" + "view filter edges is %s\n" + "inferred edge with conds %s", + getQueryJoinEdges(), + getQueryFilterEdges(), + getViewJoinEdges(), + getViewFilterEdges(), + inferredViewEdgeMap); + } + private boolean canPullUp(Edge edge) { // Only inner join and filter with none rejectNodes can be pull up if (edge instanceof JoinEdge && !((JoinEdge) edge).getJoinType().isInnerJoin()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index 947f117acff9079..9c494ae80b2392f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.exploration.mv.mapping.Mapping.MappedRelation; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; @@ -26,8 +27,10 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; @@ -42,12 +45,19 @@ public class LogicalCompatibilityContext { private final BiMap queryToViewNodeMapping; private final BiMap queryToViewEdgeExpressionMapping; private final BiMap queryToViewNodeIDMapping; + private final ObjectId planNodeId; + /** + * LogicalCompatibilityContext + */ public LogicalCompatibilityContext(BiMap queryToViewNodeMapping, - BiMap queryToViewEdgeExpressionMapping) { + BiMap queryToViewEdgeExpressionMapping, + StructInfo queryStructInfo) { this.queryToViewNodeMapping = queryToViewNodeMapping; this.queryToViewEdgeExpressionMapping = queryToViewEdgeExpressionMapping; this.queryToViewNodeIDMapping = HashBiMap.create(); + this.planNodeId = queryStructInfo.getOriginalPlan().getGroupExpression() + .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex())); } @@ -63,6 +73,10 @@ public BiMap getQueryToViewEdgeExpressionMapping() { return queryToViewEdgeExpressionMapping; } + public ObjectId getPlanNodeId() { + return planNodeId; + } + /** * generate logical compatibility context */ @@ -105,7 +119,7 @@ public static LogicalCompatibilityContext from(RelationMapping relationMapping, queryToViewEdgeMapping.put(edge, viewExpr); } }); - return new LogicalCompatibilityContext(queryToViewNodeMapping, queryToViewEdgeMapping); + return new LogicalCompatibilityContext(queryToViewNodeMapping, queryToViewEdgeMapping, queryStructInfo); } private static Expression orderSlotAsc(Expression expression) { @@ -130,4 +144,11 @@ public Expression visitEqualTo(EqualTo equalTo, Void context) { } } } + + @Override + public String toString() { + return Utils.toSqlString("LogicalCompatibilityContext", + "queryToViewNodeMapping", queryToViewNodeMapping.toString(), + "queryToViewEdgeExpressionMapping", queryToViewEdgeExpressionMapping.toString()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 153688ecc2d3ddd..8a1c98e9ddade7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -24,15 +24,19 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupId; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; +import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -53,6 +57,11 @@ public class MaterializationContext { private boolean available = true; // the mv plan from cache at present, record it to make sure query rewrite by mv is right when cache change. private Plan mvPlan; + // mark rewrite success or not + private boolean success = false; + // if rewrite by mv fail, record the reason, if success the failReason should be empty. + // The key is the query belonged group expression objectId, the value is the fail reason + private final Map failReason = new HashMap<>(); /** * MaterializationContext, this contains necessary info for query rewriting by mv @@ -127,6 +136,44 @@ public Plan getMvPlan() { return mvPlan; } + public void setSuccess(boolean success) { + this.success = success; + this.failReason.clear(); + } + + /**recordFailReason*/ + public void recordFailReason(ObjectId objectId, String reason) { + // once success, do not record the fail reason + if (this.success) { + return; + } + this.success = false; + this.failReason.put(objectId, reason); + } + + @Override + public String toString() { + StringBuilder failReasonBuilder = new StringBuilder("[").append("\n"); + for (Map.Entry reason : this.failReason.entrySet()) { + failReasonBuilder.append(reason.getKey()).append(" : \n") + .append(reason.getValue()).append("\n"); + } + failReasonBuilder.append("\n").append("]"); + return Utils.toSqlString("MaterializationContext[" + mtmv.getName() + "]", + "rewriteSuccess", this.success, + "failReason", failReasonBuilder.toString()); + } + + /**toString*/ + public static String toString(List materializationContexts) { + StringBuilder builder = new StringBuilder(); + builder.append("materializationContexts:").append("\n"); + for (MaterializationContext ctx : materializationContexts) { + builder.append("\n\n").append(ctx).append("\n"); + } + return builder.toString(); + } + /** * MaterializationContext fromMaterializedView */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index 94799899b9ec45b..0a25a603ae7d515 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; @@ -71,6 +72,11 @@ public static SplitPredicate splitPredicates(Expression expression) { return predicatesSplit.getSplitPredicate(); } + @Override + public String toString() { + return Utils.toSqlString("Predicates", "pulledUpPredicates", pulledUpPredicates); + } + /** * The split different representation for predicate expression, such as equal, range and residual predicate. */ @@ -139,5 +145,13 @@ public boolean isAlwaysTrue() { && ((BooleanLiteral) rangeExpr).getValue() && ((BooleanLiteral) residualExpr).getValue(); } + + @Override + public String toString() { + return Utils.toSqlString("SplitPredicate", + "equalPredicate", equalPredicate, + "rangePredicate", rangePredicate, + "residualPredicate", residualPredicate); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 2688882be97f3e4..0ac3085faebba45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -20,12 +20,14 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; @@ -65,6 +67,7 @@ public class StructInfo { private static final PredicateCollector PREDICATE_COLLECTOR = new PredicateCollector(); // source data private final Plan originalPlan; + private ObjectId originalPlanId; private final HyperGraph hyperGraph; private boolean valid = true; // derived data following @@ -85,6 +88,8 @@ public class StructInfo { private StructInfo(Plan originalPlan, @Nullable Plan topPlan, @Nullable Plan bottomPlan, HyperGraph hyperGraph) { this.originalPlan = originalPlan; + this.originalPlanId = originalPlan.getGroupExpression() + .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); this.hyperGraph = hyperGraph; this.topPlan = topPlan; this.bottomPlan = bottomPlan; @@ -101,7 +106,6 @@ private void init() { } collectStructInfoFromGraph(); initPredicates(); - predicatesDerive(); } public void addPredicates(List canPulledUpExpressions) { @@ -156,6 +160,7 @@ private void initPredicates() { Set topPlanPredicates = new HashSet<>(); topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates); topPlanPredicates.forEach(this.predicates::addPredicate); + predicatesDerive(); } // derive some useful predicate by predicates @@ -258,6 +263,10 @@ public List getExpressions() { ? ((LogicalProject) originalPlan).getProjects() : originalPlan.getOutput(); } + public ObjectId getOriginalPlanId() { + return originalPlanId; + } + /** * Judge the source graph logical is whether the same as target * For inner join should judge only the join tables, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java index 2f1ed230145f350..5a5bfedfe170ce1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java @@ -20,6 +20,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableMultimap; @@ -123,4 +124,9 @@ public Mapping chainedFold(Mapping target) { } return new ExpressionMapping(foldedMappingBuilder.build()); } + + @Override + public String toString() { + return Utils.toSqlString("ExpressionMapping", "expressionMapping", expressionMapping); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java index 3d9f95a5049dc00..18fa282267c2cc8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java @@ -135,6 +135,11 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(exprId); } + + @Override + public String toString() { + return "MappedSlot{" + "slot=" + slot + '}'; + } } /** Chain fold tow mapping, such as this mapping is {[a -> b]}, the target mapping is diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java index 3de99ea05a88d96..f95bcedd2fab5e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java @@ -95,4 +95,9 @@ public Map toSlotReferenceMap() { this.slotReferenceMap = slotReferenceSlotReferenceMap; return this.slotReferenceMap; } + + @Override + public String toString() { + return "SlotMapping{" + "relationSlotMap=" + relationSlotMap + '}'; + } }