diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index 07fda55e49413d6..8a4f8d92723e239 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -64,6 +64,8 @@ public MTMVCache(Plan logicalPlan, List mvOutputExpressions) { public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) { LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql()); // TODO: connect context set current db when create mv by use database + // this will be removed in the future when support join derivation + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES, ELIMINATE_OUTER_JOIN"); StatementContext mvSqlStatementContext = new StatementContext(connectContext, new OriginStatement(mtmv.getQuerySql(), 0)); NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java index 7199fef100277e0..5e2eb5757aac817 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java @@ -93,6 +93,9 @@ public PatternMatcher thenAp return new PatternMatcher<>(pattern, defaultPromise, matchedAction); } + /** + * Apply rule to return multi result, catch exception to make sure no influence on other rule + */ public PatternMatcher thenApplyMultiNoThrow( MatchedMultiAction matchedMultiAction) { MatchedMultiAction adaptMatchedMultiAction = ctx -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 6ef144c286c05b6..e530b9b7fce8f35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -159,10 +159,10 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { logger.info(currentClassName + " predicate compensate fail so continue"); continue; } - Plan rewritedPlan; + Plan rewrittenPlan; Plan mvScan = materializationContext.getMvScanPlan(); if (compensatePredicates.isAlwaysTrue()) { - rewritedPlan = mvScan; + rewrittenPlan = mvScan; } else { // Try to rewrite compensate predicates by using mv scan List rewriteCompensatePredicates = rewriteExpression( @@ -175,16 +175,16 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { logger.info(currentClassName + " compensate predicate rewrite by view fail so continue"); continue; } - rewritedPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan); + rewrittenPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan); } // Rewrite query by view - rewritedPlan = rewriteQueryByView(matchMode, + rewrittenPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo, queryToViewSlotMapping, - rewritedPlan, + rewrittenPlan, materializationContext); - if (rewritedPlan == null) { + if (rewrittenPlan == null) { logger.info(currentClassName + " rewrite query by view fail so continue"); continue; } @@ -192,19 +192,31 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { logger.info(currentClassName + " check partition validation fail so continue"); continue; } + if (!checkOutput(queryPlan, rewrittenPlan)) { + continue; + } // run rbo job on mv rewritten plan CascadesContext rewrittenPlanContext = - CascadesContext.initContext(cascadesContext.getStatementContext(), rewritedPlan, + CascadesContext.initContext(cascadesContext.getStatementContext(), rewrittenPlan, cascadesContext.getCurrentJobContext().getRequiredProperties()); Rewriter.getWholeTreeRewriter(cascadesContext).execute(); - rewritedPlan = rewrittenPlanContext.getRewritePlan(); + rewrittenPlan = rewrittenPlanContext.getRewritePlan(); logger.info(currentClassName + "rewrite by materialized view success"); - rewriteResults.add(rewritedPlan); + rewriteResults.add(rewrittenPlan); } } return rewriteResults; } + protected boolean checkOutput(Plan sourcePlan, Plan rewrittenPlan) { + if (sourcePlan.getGroupExpression().isPresent() && !rewrittenPlan.getLogicalProperties().equals( + sourcePlan.getGroupExpression().get().getOwnerGroup().getLogicalProperties())) { + logger.error("rewrittenPlan output logical properties is not same with target group"); + return false; + } + return true; + } + /** * Partition will be pruned in query then add the record the partitions to select partitions on * catalog relation. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index 96f89a513cf1a0e..94799899b9ec45b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -23,9 +23,10 @@ import com.google.common.collect.ImmutableList; -import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; /** @@ -34,7 +35,7 @@ public class Predicates { // Predicates that can be pulled up - private final List pulledUpPredicates = new ArrayList<>(); + private final Set pulledUpPredicates = new HashSet<>(); private Predicates() { } @@ -49,7 +50,7 @@ public static Predicates of(List pulledUpPredicates) { return predicates; } - public List getPulledUpPredicates() { + public Set getPulledUpPredicates() { return pulledUpPredicates; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java index 5f9a5e242de62e9..df987ebaa281660 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java @@ -20,17 +20,16 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; -import org.apache.doris.nereids.trees.expressions.CompoundPredicate; import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; import org.apache.doris.nereids.util.ExpressionUtils; -import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * Split the expression to equal, range and residual predicate. @@ -39,27 +38,26 @@ */ public class PredicatesSplitter { - private final List equalPredicates = new ArrayList<>(); - private final List rangePredicates = new ArrayList<>(); - private final List residualPredicates = new ArrayList<>(); + private final Set equalPredicates = new HashSet<>(); + private final Set rangePredicates = new HashSet<>(); + private final Set residualPredicates = new HashSet<>(); private final List conjunctExpressions; - private final PredicateExtract instance = new PredicateExtract(); - public PredicatesSplitter(Expression target) { this.conjunctExpressions = ExpressionUtils.extractConjunction(target); + PredicateExtract instance = new PredicateExtract(); for (Expression expression : conjunctExpressions) { - expression.accept(instance, expression); + expression.accept(instance, null); } } /** - * PredicateExtract + * extract to equal, range, residual predicate set */ - public class PredicateExtract extends DefaultExpressionVisitor { + public class PredicateExtract extends DefaultExpressionVisitor { @Override - public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Expression sourceExpression) { + public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Void context) { Expression leftArg = comparisonPredicate.getArgument(0); Expression rightArg = comparisonPredicate.getArgument(1); boolean leftArgOnlyContainsColumnRef = containOnlyColumnRef(leftArg, true); @@ -69,7 +67,7 @@ public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Ex equalPredicates.add(comparisonPredicate); return null; } else { - residualPredicates.add(comparisonPredicate); + rangePredicates.add(comparisonPredicate); } } else if ((leftArgOnlyContainsColumnRef && rightArg instanceof Literal) || (rightArgOnlyContainsColumnRef && leftArg instanceof Literal)) { @@ -81,12 +79,9 @@ public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Ex } @Override - public Void visitCompoundPredicate(CompoundPredicate compoundPredicate, Expression context) { - if (compoundPredicate instanceof Or) { - residualPredicates.add(compoundPredicate); - return null; - } - return super.visitCompoundPredicate(compoundPredicate, context); + public Void visit(Expression expr, Void context) { + residualPredicates.add(expr); + return null; } } @@ -98,7 +93,7 @@ public Predicates.SplitPredicate getSplitPredicate() { } private static boolean containOnlyColumnRef(Expression expression, boolean allowCast) { - if (expression instanceof SlotReference && ((SlotReference) expression).isColumnFromTable()) { + if (expression instanceof SlotReference && expression.isColumnFromTable()) { return true; } if (allowCast && expression instanceof Cast) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 1d38e893dba481c..3c9814cdce31c5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -162,7 +162,7 @@ private void initPredicates() { private void predicatesDerive() { // construct equivalenceClass according to equals predicates List shuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( - this.predicates.getPulledUpPredicates(), originalPlan).stream() + new ArrayList<>(this.predicates.getPulledUpPredicates()), originalPlan).stream() .map(Expression.class::cast) .collect(Collectors.toList()); SplitPredicate splitPredicate = Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression)); diff --git a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out index 346814bc083bf72..cb924d69bbbfb7a 100644 --- a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out +++ b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out @@ -2,9 +2,20 @@ -- !query1_0_before -- -- !query1_0_after -- -1 yy 0 0 77.50 33.50 9.50 5 -2 mi 0 0 57.40 56.20 1.20 2 -2 mm 0 0 43.20 43.20 43.20 1 + +-- !query1_1_before -- +2023-12-08 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +2023-12-09 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +2023-12-10 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +2023-12-11 2 mm 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +2023-12-12 2 mi 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + +-- !query1_1_after -- +2023-12-08 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +2023-12-09 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +2023-12-10 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +2023-12-11 2 mm 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +2023-12-12 2 mi 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 -- !query1_2_before -- 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 @@ -67,7 +78,6 @@ 2 3 2023-12-12 57.40 56.20 1.20 2 0 2 4 2023-12-10 46.00 33.50 12.50 2 0 3 3 2023-12-11 43.20 43.20 43.20 1 0 -4 3 2023-12-09 11.50 11.50 11.50 1 0 -- !query16_0_before -- 2 3 2023-12-08 20.00 10.50 9.50 2 0 @@ -89,6 +99,20 @@ -- !query17_0_after -- 3 3 2023-12-11 43.20 43.20 43.20 1 0 +-- !query17_1_before -- +1 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 +2 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 +3 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 +4 2 mm 0 0 0 0 0 0 0 0 0 0 0 0 0 +5 2 mi 0 0 0 0 0 0 0 0 0 0 0 0 0 + +-- !query17_1_after -- +1 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 +2 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 +3 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 +4 2 mm 0 0 0 0 0 0 0 0 0 0 0 0 0 +5 2 mi 0 0 0 0 0 0 0 0 0 0 0 0 0 + -- !query18_0_before -- -- !query18_0_after -- @@ -125,3 +149,9 @@ 4 2 43.20 6 2 57.40 +-- !query20_0_before -- +0 0 0 0 0 0 0 0 0 0 0 0 + +-- !query20_0_after -- +0 0 0 0 0 0 0 0 0 0 0 0 + diff --git a/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out b/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out index 7ae2e05e5233278..df3e20dc64ce0be 100644 --- a/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out +++ b/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out @@ -238,8 +238,6 @@ 4 4 4 -6 -6 -- !query5_0_before -- 4 @@ -261,6 +259,18 @@ 6 6 +-- !query6_0_before -- +2 3 2023-12-08 +2 3 2023-12-08 + +-- !query6_0_after -- +2 3 2023-12-08 +2 3 2023-12-08 + +-- !query7_0_before -- + +-- !query7_0_after -- + -- !query10_0_before -- -- !query10_0_after -- diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index 055f8d01804a658..0e068e03b8d7509 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -194,7 +194,7 @@ suite("aggregate_without_roll_up") { sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" - def mv1_1 = "select O_SHIPPRIORITY, O_COMMENT, " + + def mv1_1 = "select O_SHIPPRIORITY, O_COMMENT, O_ORDERDATE, " + "count(distinct case when O_SHIPPRIORITY > 1 and O_ORDERKEY IN (1, 3) then O_ORDERSTATUS else null end) as filter_cnt_1, " + "count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN (2) then O_ORDERSTATUS else null end) as filter_cnt_2, " + "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN (3, 4) then O_ORDERSTATUS else null end) as filter_cnt_3, " + @@ -218,9 +218,10 @@ suite("aggregate_without_roll_up") { "from orders " + "where O_ORDERDATE < '2023-12-30'" + "group by " + + "O_ORDERDATE, " + "O_SHIPPRIORITY, " + "O_COMMENT " - def query1_1 = "select O_SHIPPRIORITY, O_COMMENT, " + + def query1_1 = "select O_ORDERDATE, O_SHIPPRIORITY, O_COMMENT, " + "count(distinct case when O_SHIPPRIORITY > 1 and O_ORDERKEY IN (1, 3) then O_ORDERSTATUS else null end) as filter_cnt_1, " + "count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN (2) then O_ORDERSTATUS else null end) as filter_cnt_2, " + "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN (3, 4) then O_ORDERSTATUS else null end) as filter_cnt_3, " + @@ -239,13 +240,13 @@ suite("aggregate_without_roll_up") { "from orders " + "where O_ORDERDATE < '2023-12-30' and O_ORDERDATE > '2023-12-01'" + "group by " + + "O_ORDERDATE, " + "O_SHIPPRIORITY, " + "O_COMMENT " - // should support but not, tmp -// order_qt_query1_1_before "${query1_1}" -// check_rewrite(mv1_1, query1_1, "mv1_1") -// order_qt_query1_1_after "${query1_1}" -// sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1""" + order_qt_query1_1_before "${query1_1}" + check_rewrite(mv1_1, query1_1, "mv1_1") + order_qt_query1_1_after "${query1_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1""" def mv1_2 = "select O_SHIPPRIORITY, O_COMMENT, " + @@ -609,11 +610,10 @@ suite("aggregate_without_roll_up") { "lineitem.L_ORDERKEY, " + "orders.O_SHIPPRIORITY, " + "orders.O_COMMENT " - // rewrite success but cbo not chose, tmp -// order_qt_query17_1_before "${query17_1}" -// check_rewrite(mv17_1, query17_1, "mv17_1") -// order_qt_query17_1_after "${query17_1}" -// sql """ DROP MATERIALIZED VIEW IF EXISTS mv17_1""" + order_qt_query17_1_before "${query17_1}" + check_rewrite(mv17_1, query17_1, "mv17_1") + order_qt_query17_1_after "${query17_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv17_1""" // filter outside + left + right def mv18_0 = "select l_shipdate, l_suppkey, " + @@ -760,9 +760,8 @@ suite("aggregate_without_roll_up") { "orders " + "on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + "where orders.O_ORDERDATE < '2023-12-30' and orders.O_ORDERDATE > '2023-12-01' " - // rewrite success but cbo not chose, tmp -// order_qt_query20_0_before "${query20_0}" -// check_rewrite(mv20_0, query20_0, "mv20_0") -// order_qt_query20_0_after "${query20_0}" -// sql """ DROP MATERIALIZED VIEW IF EXISTS mv20_0""" + order_qt_query20_0_before "${query20_0}" + check_rewrite(mv20_0, query20_0, "mv20_0") + order_qt_query20_0_after "${query20_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv20_0""" } diff --git a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy index e0c0452415dae72..d8025e0bc60b14e 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy @@ -387,11 +387,10 @@ suite("inner_join") { "from lineitem t1 " + "inner join (select * from orders where o_orderdate = '2023-12-08') t2 " + "on t1.l_orderkey = o_orderkey and t1.l_shipdate = o_orderdate " - // should passed but not as isGraphLogicalEquals is false -// order_qt_query6_0_before "${query6_0}" -// check_rewrite(mv6_0, query6_0, "mv6_0") -// order_qt_query6_0_after "${query6_0}" -// sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_0""" + order_qt_query6_0_before "${query6_0}" + check_rewrite(mv6_0, query6_0, "mv6_0") + order_qt_query6_0_after "${query6_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_0""" // filter inside + inner + right @@ -405,11 +404,10 @@ suite("inner_join") { "inner join (select * from orders where o_orderdate = '2023-12-08') t2 " + "on t1.l_orderkey = o_orderkey and t1.l_shipdate = o_orderdate " + "where l_partkey = 3" - // should passed but not, because isGraphLogicalEquals is false -// order_qt_query7_0_before "${query7_0}" -// check_rewrite(mv7_0, query7_0, "mv7_0") -// order_qt_query7_0_after "${query7_0}" -// sql """ DROP MATERIALIZED VIEW IF EXISTS mv7_0""" + order_qt_query7_0_before "${query7_0}" + check_rewrite(mv7_0, query7_0, "mv7_0") + order_qt_query7_0_after "${query7_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv7_0""" // check not match, because use a filed orders.O_SHIPPRIORITY which not in mv