(fix)[mtmv] merge pr

fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java

@@ -43,7 +43,9 @@ public abstract class Edge implements HyperElement {
     // added by the graph simplifier.
     private final long leftRequiredNodes;
     private final long rightRequiredNodes;
+    // The nodes needed which to prevent wrong association or l-association
     private long leftExtendedNodes;
+    // The nodes needed which to prevent wrong association or r-association
     private long rightExtendedNodes;
 
     // record the left child edges and right child edges in origin plan tree
@@ -54,8 +56,11 @@ public abstract class Edge implements HyperElement {
     private final BitSet curOperatorEdges = new BitSet();
     // record all sub nodes behind in this operator. It's T function in paper
     private final long subTreeNodes;
-
+    // The edges which prevents association or l-association when join edge
+    // and prevents push down or pull up when filter edge in the left of edge
     private final Set<JoinEdge> leftRejectEdges;
+    // The edges which prevents association or r-association
+    // and prevents push down or pull up when filter edge in the right of edge
     private final Set<JoinEdge> rightRejectEdges;
 
     /**

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java

@@ -37,9 +37,12 @@
 import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.JoinUtils;
 
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 
 import java.util.ArrayList;
@@ -426,28 +429,76 @@ private Map<Edge, Edge> constructQueryToViewJoinMapWithExpr() {
         return edgeMap;
     }
 
+    // Such as the filter as following, their expression is same, but should be different filter edge
+    // Only construct edge that can mapping, the edges which can not mapping would be handled by buildComparisonRes
+    //     LogicalJoin[569]
+    //       |--LogicalProject[567]
+    //       |  +--LogicalFilter[566] ( predicates=(l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1))) )
+    //       |     +--LogicalJoin[565]
+    //       |        |--LogicalProject[562]
+    //       |        |  +--LogicalOlapScan
+    //       |        +--LogicalProject[564]
+    //       |           +--LogicalFilter[563] ( predicates=(l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1))))
+    //       |              +--LogicalOlapScan
+    //       +--LogicalProject[568]
+    //         +--LogicalOlapScan
     private Map<Edge, Edge> constructQueryToViewFilterMapWithExpr() {
-        Map<Expression, Edge> viewExprToEdge = getViewFilterEdges().stream()
+        Multimap<Expression, Edge> viewExprToEdge = HashMultimap.create();
+        getViewFilterEdges().stream()
                 .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
-                .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
-        Map<Expression, Edge> queryExprToEdge = getQueryFilterEdges().stream()
-                .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
-                .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
+                .forEach(pair -> viewExprToEdge.put(pair.key(), pair.value()));
 
-        HashMap<Edge, Edge> edgeMap = new HashMap<>();
-        for (Entry<Expression, Edge> entry : queryExprToEdge.entrySet()) {
-            if (edgeMap.containsKey(entry.getValue())) {
-                continue;
+        Multimap<Expression, Edge> queryExprToEdge = HashMultimap.create();
+        getQueryFilterEdges().stream()
+                .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
+                .forEach(pair -> queryExprToEdge.put(pair.key(), pair.value()));
+
+        HashMap<Edge, Edge> queryToViewEdgeMap = new HashMap<>();
+        for (Entry<Expression, Collection<Edge>> entry : queryExprToEdge.asMap().entrySet()) {
+            Expression queryExprViewBased = null;
+            for (Edge queryEdge : entry.getValue()) {
+                queryExprViewBased = getMappingViewExprByQueryExpr(entry.getKey(),
+                        queryEdge,
+                        logicalCompatibilityContext,
+                        ExpressionPosition.FILTER_EDGE).orElse(null);
+                if (queryExprViewBased == null) {
+                    continue;
+                }
+                Collection<Edge> viewEdges = viewExprToEdge.get(queryExprViewBased);
+                if (viewEdges.isEmpty()) {
+                    continue;
+                }
+                for (Edge viewEdge : viewEdges) {
+                    if (!isSubTreeNodesEquals(queryEdge, viewEdge, logicalCompatibilityContext)) {
+                        // Such as query filter edge is <{1} --FILTER-- {}> but view filter edge is
+                        // <{0, 1} --FILTER-- {}>, though they are all
+                        // l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1)) but they are different actually
+                        continue;
+                    }
+                    queryToViewEdgeMap.put(queryEdge, viewEdge);
+                }
             }
-            Expression viewExpr = getMappingViewExprByQueryExpr(entry.getKey(),
-                    entry.getValue(),
-                    logicalCompatibilityContext,
-                    ExpressionPosition.FILTER_EDGE).orElse(null);
-            if (viewExprToEdge.containsKey(viewExpr)) {
-                edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr)));
+        }
+        return queryToViewEdgeMap;
+    }
+
+    private static boolean isSubTreeNodesEquals(Edge queryEdge, Edge viewEdge,
+            LogicalCompatibilityContext logicalCompatibilityContext) {
+        if (!(queryEdge instanceof FilterEdge) || !(viewEdge instanceof FilterEdge)) {
+            return false;
+        }
+        // subTreeNodes should be equal
+        BiMap<Integer, Integer> queryToViewNodeIdMapping =
+                logicalCompatibilityContext.getQueryToViewNodeIDMapping();
+        List<Integer> queryNodeIndexViewBasedList = new ArrayList<>();
+        for (int queryNodeIndex : LongBitmap.getIterator(queryEdge.getSubTreeNodes())) {
+            Integer queryNodeIndexViewBased = queryToViewNodeIdMapping.get(queryNodeIndex);
+            if (queryNodeIndexViewBased == null) {
+                return false;
             }
+            queryNodeIndexViewBasedList.add(queryNodeIndexViewBased);
         }
-        return edgeMap;
+        return LongBitmap.newBitmap(queryNodeIndexViewBasedList) == viewEdge.getSubTreeNodes();
     }
 
     private void refreshViewEdges() {

regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out

@@ -315,6 +315,14 @@ a	3	3	a,a,a	4.0	yy	3	1
 a	4	2	a,a	4.0	yy	2	1
 c	3	6	c,c,c	5.333333333333333	mi	3	2
 
+-- !query28_0_before --
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1
+
+-- !query28_0_after --
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1
+
 -- !query29_0_before --
 1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
 1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1

regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy

@@ -120,6 +120,12 @@ suite("aggregate_without_roll_up") {
     (2, 3, 10, 11.01, 'supply2', null);
     """
 
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+
+    sql """alter table orders modify column o_comment set stats ('row_count'='8');"""
+
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     // single table
     // with filter
     def mv1_0 = """
@@ -1357,6 +1363,158 @@ suite("aggregate_without_roll_up") {
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv27_0"""
 
 
+   // query and mv has the same filter but position is different, should rewrite successfully
+    def mv28_0 = """
+    select 
+      o_custkey, 
+      o_orderdate, 
+      o_shippriority, 
+      o_comment, 
+      o_orderkey, 
+      orders.public_col as col1, 
+      l_orderkey, 
+      l_partkey, 
+      l_suppkey, 
+      lineitem.public_col as col2, 
+      ps_partkey, 
+      ps_suppkey, 
+      partsupp.public_col as col3, 
+      partsupp.public_col * 2 as col4, 
+      o_orderkey + l_orderkey + ps_partkey * 2, 
+      sum(
+        o_orderkey + l_orderkey + ps_partkey * 2
+      ), 
+      count() as count_all 
+    from 
+      (
+        select 
+          o_custkey, 
+          o_orderdate, 
+          o_shippriority, 
+          o_comment, 
+          o_orderkey, 
+          orders.public_col as public_col 
+        from 
+          orders
+      ) orders 
+      left join (
+        select 
+          l_orderkey, 
+          l_partkey, 
+          l_suppkey, 
+          lineitem.public_col as public_col 
+        from 
+          lineitem 
+        where 
+          l_orderkey is null 
+          or l_orderkey <> 8
+      ) lineitem on l_orderkey = o_orderkey 
+      inner join (
+        select 
+          ps_partkey, 
+          ps_suppkey, 
+          partsupp.public_col as public_col 
+        from 
+          partsupp
+      ) partsupp on ps_partkey = o_orderkey 
+    where 
+      l_orderkey is null 
+      or l_orderkey <> 8
+    group by 
+      1, 
+      2, 
+      3, 
+      4, 
+      5, 
+      6, 
+      7, 
+      8, 
+      9, 
+      10, 
+      11, 
+      12, 
+      13, 
+      14;
+    """
+    def query28_0 = """
+    select 
+      o_custkey, 
+      o_orderdate, 
+      o_shippriority, 
+      o_comment, 
+      o_orderkey, 
+      orders.public_col as col1, 
+      l_orderkey, 
+      l_partkey, 
+      l_suppkey, 
+      lineitem.public_col as col2, 
+      ps_partkey, 
+      ps_suppkey, 
+      partsupp.public_col as col3, 
+      partsupp.public_col * 2 as col4, 
+      o_orderkey + l_orderkey + ps_partkey * 2, 
+      sum(
+        o_orderkey + l_orderkey + ps_partkey * 2
+      ), 
+      count() as count_all 
+    from 
+      (
+        select 
+          o_custkey, 
+          o_orderdate, 
+          o_shippriority, 
+          o_comment, 
+          o_orderkey, 
+          orders.public_col as public_col 
+        from 
+          orders
+      ) orders 
+      left join (
+        select 
+          l_orderkey, 
+          l_partkey, 
+          l_suppkey, 
+          lineitem.public_col as public_col 
+        from 
+          lineitem 
+        where 
+          l_orderkey is null 
+          or l_orderkey <> 8
+      ) lineitem on l_orderkey = o_orderkey 
+      inner join (
+        select 
+          ps_partkey, 
+          ps_suppkey, 
+          partsupp.public_col as public_col 
+        from 
+          partsupp
+      ) partsupp on ps_partkey = o_orderkey 
+    where 
+      l_orderkey is null 
+      or l_orderkey <> 8 
+    group by 
+      1, 
+      2, 
+      3, 
+      4, 
+      5, 
+      6, 
+      7, 
+      8, 
+      9, 
+      10, 
+      11, 
+      12, 
+      13, 
+      14;
+    """
+    order_qt_query28_0_before "${query28_0}"
+    async_mv_rewrite_success(db, mv28_0, query28_0, "mv28_0")
+    order_qt_query28_0_after "${query28_0}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv28_0"""
+
+
+
     // query and mv has the same filter but position is different, should rewrite successfully
     def mv29_0 = """
       select