Pick some pr from master to 3.0 (apache#40871) apache#41145 (apache#4…

…2303) ## Proposed changes pr: apache#40871 commitId: da6ac0c pr: apache#41145 commitId: 5e6e4bf
seawinde · Oct 23, 2024 · fd4e9a8 · fd4e9a8
1 parent ae5535e
commit fd4e9a8
Show file tree

Hide file tree

Showing 192 changed files with 1,881 additions and 5,914 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -48,6 +48,7 @@
 import org.apache.commons.lang3.tuple.Pair;
 import org.json.JSONObject;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -502,7 +503,8 @@ AGGREGATE KEY (siteid,citycode,username)
         if (getTable() instanceof MTMV) {
             MTMV mtmv = (MTMV) getTable();
             MTMVCache cache = mtmv.getCache();
-            if (cache == null) {
+            // Maybe query specified index, should not calc, such as select count(*) from t1 index col_index
+            if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
                 return;
             }
             Plan originalPlan = cache.getOriginalPlan();
@@ -528,7 +530,8 @@ public void computeUniform(DataTrait.Builder builder) {
         if (getTable() instanceof MTMV) {
             MTMV mtmv = (MTMV) getTable();
             MTMVCache cache = mtmv.getCache();
-            if (cache == null) {
+            // Maybe query specified index, should not calc, such as select count(*) from t1 index col_index
+            if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
                 return;
             }
             Plan originalPlan = cache.getOriginalPlan();
@@ -542,7 +545,8 @@ public void computeEqualSet(DataTrait.Builder builder) {
         if (getTable() instanceof MTMV) {
             MTMV mtmv = (MTMV) getTable();
             MTMVCache cache = mtmv.getCache();
-            if (cache == null) {
+            // Maybe query specified index, should not calc, such as select count(*) from t1 index col_index
+            if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
                 return;
             }
             Plan originalPlan = cache.getOriginalPlan();
@@ -556,7 +560,8 @@ public void computeFd(DataTrait.Builder builder) {
         if (getTable() instanceof MTMV) {
             MTMV mtmv = (MTMV) getTable();
             MTMVCache cache = mtmv.getCache();
-            if (cache == null) {
+            // Maybe query specified index, should not calc, such as select count(*) from t1 index col_index
+            if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
                 return;
             }
             Plan originalPlan = cache.getOriginalPlan();
@@ -567,9 +572,23 @@ public void computeFd(DataTrait.Builder builder) {
 
     Map<Slot, Slot> constructReplaceMap(MTMV mtmv) {
         Map<Slot, Slot> replaceMap = new HashMap<>();
-        List<Slot> originOutputs = mtmv.getCache().getOriginalPlan().getOutput();
-        for (int i = 0; i < getOutput().size(); i++) {
-            replaceMap.put(originOutputs.get(i), getOutput().get(i));
+        // Need remove invisible column, and then mapping them
+        List<Slot> originOutputs = new ArrayList<>();
+        for (Slot originSlot : mtmv.getCache().getOriginalPlan().getOutput()) {
+            if (!(originSlot instanceof SlotReference) || (((SlotReference) originSlot).isVisible())) {
+                originOutputs.add(originSlot);
+            }
+        }
+        List<Slot> targetOutputs = new ArrayList<>();
+        for (Slot targeSlot : getOutput()) {
+            if (!(targeSlot instanceof SlotReference) || (((SlotReference) targeSlot).isVisible())) {
+                targetOutputs.add(targeSlot);
+            }
+        }
+        Preconditions.checkArgument(originOutputs.size() == targetOutputs.size(),
+                "constructReplaceMap, the size of originOutputs and targetOutputs should be same");
+        for (int i = 0; i < targetOutputs.size(); i++) {
+            replaceMap.put(originOutputs.get(i), targetOutputs.get(i));
         }
         return replaceMap;
     }

diff --git a/...ion-test/data/nereids_rules_p0/mv/agg_optimize_when_uniform/agg_optimize_when_uniform.out b/...ion-test/data/nereids_rules_p0/mv/agg_optimize_when_uniform/agg_optimize_when_uniform.out
@@ -102,10 +102,10 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.l_orderkey = orders.o_orderkey) and (t1.l_shipdate = orders.o_orderdate)) otherCondition=()
---------filter((t1.l_shipdate = '2023-12-09'))
-----------PhysicalOlapScan[lineitem]
 --------filter((orders.o_orderdate = '2023-12-09') and (orders.o_shippriority = 1) and (orders.o_totalprice = 11.50))
 ----------PhysicalOlapScan[orders]
+--------filter((t1.l_shipdate = '2023-12-09'))
+----------PhysicalOlapScan[lineitem]
 
 -- !query7_1_after --
 yy	4	11.50	11.50	11.50	1

diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/MVWithAs.out b/regression-test/data/nereids_syntax_p0/mv/ut/MVWithAs.out
@@ -2,8 +2,12 @@
 -- !select_star --
 2020-01-01	1	a	1
 2020-01-01	1	a	1
+2020-01-01	1	a	1
+2020-01-01	1	a	1
+2020-01-02	2	b	2
+2020-01-02	2	b	2
 2020-01-02	2	b	2
 
 -- !select_mv --
-3
+7