From 9c9ba36a1ae89419350d6495797eacc366114d46 Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 5 Dec 2023 21:08:24 +0800 Subject: [PATCH] fix filter regression test --- .../mv/AbstractMaterializedViewJoinRule.java | 13 ++++--- .../mv/AbstractMaterializedViewRule.java | 34 ++++++++++++------- .../nereids_rules_p0/mv/inner_join.groovy | 4 +-- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 4bd21474cb79da4..4574b9c8c28e1e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -33,6 +33,7 @@ import java.util.HashSet; import java.util.List; +import java.util.stream.Collectors; /** * AbstractMaterializedViewJoinRule @@ -52,21 +53,23 @@ protected Plan rewriteQueryByView(MatchMode matchMode, queryStructInfo.getExpressions(), queryStructInfo.getOriginalPlan()); // Rewrite top projects, represent the query projects by view - List expressions = rewriteExpression( + List expressionsRewritten = rewriteExpression( queryShuttleExpression, materializationContext.getViewExpressionIndexMapping(), - queryToViewSlotMappings, - tempRewritedPlan + queryToViewSlotMappings ); // Can not rewrite, bail out - if (expressions == null) { + if (expressionsRewritten == null + || expressionsRewritten.stream().anyMatch(expr -> !(expr instanceof NamedExpression))) { return null; } if (queryStructInfo.getOriginalPlan().getGroupExpression().isPresent()) { materializationContext.addMatchedGroup( queryStructInfo.getOriginalPlan().getGroupExpression().get().getOwnerGroup().getGroupId()); } - return new LogicalProject<>(expressions, tempRewritedPlan); + return new LogicalProject<>( + expressionsRewritten.stream().map(NamedExpression.class::cast).collect(Collectors.toList()), + tempRewritedPlan); } // Check join is whether valid or not. Support join's input can not contain aggregate diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index ddcbf219e565ba1..d5c6e9053387751 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -118,11 +119,10 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { rewritedPlan = mvScan; } else { // Try to rewrite compensate predicates by using mv scan - List rewriteCompensatePredicates = rewriteExpression( + List rewriteCompensatePredicates = rewriteExpression( compensatePredicates.toList(), materializationContext.getViewExpressionIndexMapping(), - queryToViewSlotMapping, - mvScan); + queryToViewSlotMapping); if (rewriteCompensatePredicates.isEmpty()) { continue; } @@ -159,11 +159,10 @@ protected Plan rewriteQueryByView(MatchMode matchMode, /** * Use target output expression to represent the source expression */ - protected List rewriteExpression( + protected List rewriteExpression( List sourceExpressions, ExpressionMapping expressionMapping, - SlotMapping sourceToTargetMapping, - Plan targetScanNode) { + SlotMapping sourceToTargetMapping) { // Firstly, rewrite the target plan output expression using query with inverse mapping // then try to use the mv expression to represent the query. if any of source expressions // can not be represented by mv, return null @@ -184,22 +183,23 @@ protected List rewriteExpression( // view to view scan expression is 1:1 so get first element Map mvSqlToMvScanMappingQueryBased = flattenExpressionMap.get(0); - List rewrittenExpressions = new ArrayList<>(); + List rewrittenExpressions = new ArrayList<>(); for (Expression expressionToRewrite : sourceExpressions) { - if (expressionToRewrite instanceof BooleanLiteral - && ((BooleanLiteral) expressionToRewrite).getValue()) { + if (expressionToRewrite instanceof Literal) { + rewrittenExpressions.add(expressionToRewrite); continue; } final Set slotsToRewrite = expressionToRewrite.collectToSet(expression -> expression instanceof Slot); + boolean wiAlias = expressionToRewrite instanceof NamedExpression; Expression replacedExpression = ExpressionUtils.replace(expressionToRewrite, mvSqlToMvScanMappingQueryBased, - true); + wiAlias); if (replacedExpression.anyMatch(slotsToRewrite::contains)) { // if contains any slot to rewrite, which means can not be rewritten by target, bail out return null; } - rewrittenExpressions.add((NamedExpression) replacedExpression); + rewrittenExpressions.add(replacedExpression); } return rewrittenExpressions; } @@ -221,6 +221,9 @@ protected SplitPredicate predicatesCompensate( Map viewToQuerySlotMapping = queryToViewSlotMapping.inverse() .toSlotReferenceMap(); EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMapping); + if (viewEquivalenceClassQueryBased == null) { + return SplitPredicate.empty(); + } final List equalCompensateConjunctions = new ArrayList<>(); if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) { equalCompensateConjunctions.add(BooleanLiteral.of(true)); @@ -278,7 +281,9 @@ protected SplitPredicate predicatesCompensate( Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate)); Set viewRangeQueryBasedSet = Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased)); - if (!queryRangeSet.containsAll(viewRangeQueryBasedSet)) { + // query range predicate can not contain all view range predicate when view have range predicate, bail out + if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE) + && !queryRangeSet.containsAll(viewRangeQueryBasedSet)) { return SplitPredicate.empty(); } queryRangeSet.removeAll(viewRangeQueryBasedSet); @@ -294,7 +299,10 @@ protected SplitPredicate predicatesCompensate( Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate)); Set viewResidualQueryBasedSet = Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased)); - if (!queryResidualSet.containsAll(viewResidualQueryBasedSet)) { + // query residual predicate can not contain all view residual predicate when view have residual predicate, + // bail out + if (!viewResidualPredicateQueryBased.equals(BooleanLiteral.TRUE) + && !queryResidualSet.containsAll(viewResidualQueryBasedSet)) { return SplitPredicate.empty(); } queryResidualSet.removeAll(viewResidualQueryBasedSet); diff --git a/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy index 27249fdbccc8e6c..9d0bc62f05c22e2 100644 --- a/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy @@ -151,7 +151,7 @@ suite("inner_join") { order_qt_query1_2 "${query1_2}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_2""" - // because hyper graph node contains group plan, should fix it firstly + // select + from + inner join + filter def mv1_3 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + "from orders " + "inner join lineitem on lineitem.L_ORDERKEY = orders.O_ORDERKEY " @@ -159,7 +159,7 @@ suite("inner_join") { "from lineitem " + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + "where lineitem.L_LINENUMBER > 10" -// check_rewrite(mv1_3, query1_3, "mv1_3") + check_rewrite(mv1_3, query1_3, "mv1_3") order_qt_query1_3 "${query1_3}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3""" }