diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java index 192adf89d413b82..f75ed83250119a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java @@ -43,7 +43,9 @@ public abstract class Edge implements HyperElement { // added by the graph simplifier. private final long leftRequiredNodes; private final long rightRequiredNodes; + // The nodes needed which to prevent wrong association or l-association private long leftExtendedNodes; + // The nodes needed which to prevent wrong association or r-association private long rightExtendedNodes; // record the left child edges and right child edges in origin plan tree @@ -54,8 +56,11 @@ public abstract class Edge implements HyperElement { private final BitSet curOperatorEdges = new BitSet(); // record all sub nodes behind in this operator. It's T function in paper private final long subTreeNodes; - + // The edges which prevents association or l-association when join edge + // and prevents push down or pull up when filter edge in the left of edge private final Set leftRejectEdges; + // The edges which prevents association or r-association + // and prevents push down or pull up when filter edge in the right of edge private final Set rightRejectEdges; /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index 3cd0ddd83e3b411..22282a2351627b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -37,9 +37,12 @@ import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.JoinUtils; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import java.util.ArrayList; @@ -426,28 +429,76 @@ private Map constructQueryToViewJoinMapWithExpr() { return edgeMap; } + // Such as the filter as following, their expression is same, but should be different filter edge + // Only construct edge that can mapping, the edges which can not mapping would be handled by buildComparisonRes + // LogicalJoin[569] + // |--LogicalProject[567] + // | +--LogicalFilter[566] ( predicates=(l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1))) ) + // | +--LogicalJoin[565] + // | |--LogicalProject[562] + // | | +--LogicalOlapScan + // | +--LogicalProject[564] + // | +--LogicalFilter[563] ( predicates=(l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1)))) + // | +--LogicalOlapScan + // +--LogicalProject[568] + // +--LogicalOlapScan private Map constructQueryToViewFilterMapWithExpr() { - Map viewExprToEdge = getViewFilterEdges().stream() + Multimap viewExprToEdge = HashMultimap.create(); + getViewFilterEdges().stream() .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) - .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); - Map queryExprToEdge = getQueryFilterEdges().stream() - .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) - .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); + .forEach(pair -> viewExprToEdge.put(pair.key(), pair.value())); - HashMap edgeMap = new HashMap<>(); - for (Entry entry : queryExprToEdge.entrySet()) { - if (edgeMap.containsKey(entry.getValue())) { - continue; + Multimap queryExprToEdge = HashMultimap.create(); + getQueryFilterEdges().stream() + .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) + .forEach(pair -> queryExprToEdge.put(pair.key(), pair.value())); + + HashMap queryToViewEdgeMap = new HashMap<>(); + for (Entry> entry : queryExprToEdge.asMap().entrySet()) { + Expression queryExprViewBased = null; + for (Edge queryEdge : entry.getValue()) { + queryExprViewBased = getMappingViewExprByQueryExpr(entry.getKey(), + queryEdge, + logicalCompatibilityContext, + ExpressionPosition.FILTER_EDGE).orElse(null); + if (queryExprViewBased == null) { + continue; + } + Collection viewEdges = viewExprToEdge.get(queryExprViewBased); + if (viewEdges.isEmpty()) { + continue; + } + for (Edge viewEdge : viewEdges) { + if (!isSubTreeNodesEquals(queryEdge, viewEdge, logicalCompatibilityContext)) { + // Such as query filter edge is <{1} --FILTER-- {}> but view filter edge is + // <{0, 1} --FILTER-- {}>, though they are all + // l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1)) but they are different actually + continue; + } + queryToViewEdgeMap.put(queryEdge, viewEdge); + } } - Expression viewExpr = getMappingViewExprByQueryExpr(entry.getKey(), - entry.getValue(), - logicalCompatibilityContext, - ExpressionPosition.FILTER_EDGE).orElse(null); - if (viewExprToEdge.containsKey(viewExpr)) { - edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr))); + } + return queryToViewEdgeMap; + } + + private static boolean isSubTreeNodesEquals(Edge queryEdge, Edge viewEdge, + LogicalCompatibilityContext logicalCompatibilityContext) { + if (!(queryEdge instanceof FilterEdge) || !(viewEdge instanceof FilterEdge)) { + return false; + } + // subTreeNodes should be equal + BiMap queryToViewNodeIdMapping = + logicalCompatibilityContext.getQueryToViewNodeIDMapping(); + List queryNodeIndexViewBasedList = new ArrayList<>(); + for (int queryNodeIndex : LongBitmap.getIterator(queryEdge.getSubTreeNodes())) { + Integer queryNodeIndexViewBased = queryToViewNodeIdMapping.get(queryNodeIndex); + if (queryNodeIndexViewBased == null) { + return false; } + queryNodeIndexViewBasedList.add(queryNodeIndexViewBased); } - return edgeMap; + return LongBitmap.newBitmap(queryNodeIndexViewBasedList) == viewEdge.getSubTreeNodes(); } private void refreshViewEdges() { diff --git a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out index e192959a03d4c24..c400e078daff1a3 100644 --- a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out +++ b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out @@ -315,6 +315,14 @@ a 3 3 a,a,a 4.0 yy 3 1 a 4 2 a,a 4.0 yy 2 1 c 3 6 c,c,c 5.333333333333333 mi 3 2 +-- !query28_0_before -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + +-- !query28_0_after -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + -- !query29_0_before -- 1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index 183ad259686ad88..f082b3bdefd20c0 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -120,6 +120,12 @@ suite("aggregate_without_roll_up") { (2, 3, 10, 11.01, 'supply2', null); """ + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // single table // with filter def mv1_0 = """ @@ -1357,6 +1363,158 @@ suite("aggregate_without_roll_up") { sql """ DROP MATERIALIZED VIEW IF EXISTS mv27_0""" + // query and mv has the same filter but position is different, should rewrite successfully + def mv28_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as col1, + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey + l_orderkey + ps_partkey * 2, + sum( + o_orderkey + l_orderkey + ps_partkey * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + l_orderkey is null + or l_orderkey <> 8 + ) lineitem on l_orderkey = o_orderkey + inner join ( + select + ps_partkey, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on ps_partkey = o_orderkey + where + l_orderkey is null + or l_orderkey <> 8 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + def query28_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as col1, + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey + l_orderkey + ps_partkey * 2, + sum( + o_orderkey + l_orderkey + ps_partkey * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + l_orderkey is null + or l_orderkey <> 8 + ) lineitem on l_orderkey = o_orderkey + inner join ( + select + ps_partkey, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on ps_partkey = o_orderkey + where + l_orderkey is null + or l_orderkey <> 8 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + order_qt_query28_0_before "${query28_0}" + async_mv_rewrite_success(db, mv28_0, query28_0, "mv28_0") + order_qt_query28_0_after "${query28_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv28_0""" + + + // query and mv has the same filter but position is different, should rewrite successfully def mv29_0 = """ select