tmp

fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java

@@ -30,6 +30,7 @@
 import org.apache.doris.mtmv.MTMVRefreshInfo;
 import org.apache.doris.mtmv.MTMVRelation;
 import org.apache.doris.mtmv.MTMVStatus;
+import org.apache.doris.mtmv.MVCache;
 import org.apache.doris.persist.gson.GsonUtils;
 
 import com.google.gson.annotations.SerializedName;
@@ -61,6 +62,8 @@ public class MTMV extends OlapTable {
     private Map<String, String> mvProperties;
     @SerializedName("r")
     private MTMVRelation relation;
+    // Should update after every fresh
+    private MVCache mvCache;
 
     // For deserialization
     public MTMV() {
@@ -116,6 +119,14 @@ public MTMVRelation getRelation() {
         return relation;
     }
 
+    public MVCache getMvCache() {
+        return mvCache;
+    }
+
+    public void setMvCache(MVCache mvCache) {
+        this.mvCache = mvCache;
+    }
+
     public MTMVRefreshInfo alterRefreshInfo(MTMVRefreshInfo newRefreshInfo) {
         return refreshInfo.updateNotNull(newRefreshInfo);
     }

fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCacheManager.java

@@ -48,6 +48,7 @@
 import org.apache.doris.persist.AlterMTMV;
 import org.apache.doris.qe.ConnectContext;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import org.apache.commons.collections.CollectionUtils;
@@ -70,6 +71,11 @@ public Set<BaseTableInfo> getMtmvsByBaseTable(BaseTableInfo table) {
         return tableMTMVs.get(table);
     }
 
+    // TODO Implement the method which getting materialized view by tables
+    public List<MTMV> getAvailableMaterializedView(List<BaseTableInfo> tables){
+        return ImmutableList.of();
+    }
+
     public boolean isAvailableMTMV(MTMV mtmv, ConnectContext ctx) throws AnalysisException, DdlException {
         // check session variable if enable rewrite
         if (!ctx.getSessionVariable().isEnableMvRewrite()) {

fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java

@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.mtmv;
+
+import org.apache.doris.catalog.MTMV;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+
+import java.util.List;
+
+/**The cache for materialized view cache */
+public class MVCache {
+
+    // the materialized view plan which should be optimized by the same rules to query
+    private final Plan logicalPlan;
+    // this should be shuttle expression with lineage
+    private final List<Expression> mvOutputExpressions;
+    // the context when parse, analyze, optimize the mv logical plan
+    private final CascadesContext context;
+
+    public MVCache(MTMV materializedView, Plan logicalPlan, List<Expression> mvOutputExpressions,
+            CascadesContext context) {
+        this.logicalPlan = logicalPlan;
+        this.mvOutputExpressions = mvOutputExpressions;
+        this.context = context;
+    }
+
+    public Plan getLogicalPlan() {
+        return logicalPlan;
+    }
+
+    public List<Expression> getMvOutputExpressions() {
+        return mvOutputExpressions;
+    }
+
+    public CascadesContext getContext() {
+        return context;
+    }
+
+    public static MVCache from(MTMV mtmv) {
+        // TODO Init the MVCache
+        return new MVCache(mtmv, null, null, null);
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java

@@ -18,9 +18,12 @@
 package org.apache.doris.nereids.rules.exploration.mv;
 
 import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
+import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.util.ExpressionUtils;
 
 import java.util.List;
 
@@ -34,15 +37,18 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali
     protected Plan rewriteQueryByView(MatchMode matchMode,
             StructInfo queryStructInfo,
             StructInfo viewStructInfo,
-            RelationMapping queryToViewTableMappings,
-            Plan tempRewritedPlan) {
+            SlotMapping queryToViewSlotMappings,
+            Plan tempRewritedPlan,
+            MaterializationContext materializationContext) {
 
+        List<? extends Expression> queryShuttleExpression = ExpressionUtils.shuttleExpressionWithLineage(
+                queryStructInfo.getExpressions(),
+                queryStructInfo.getOriginalPlan());
         // Rewrite top projects, represent the query projects by view
         List<NamedExpression> expressions = rewriteExpression(
-                queryStructInfo.getExpressions(),
-                queryStructInfo,
-                viewStructInfo,
-                queryToViewTableMappings,
+                queryShuttleExpression,
+                materializationContext.getViewExpressionIndexMapping(),
+                queryToViewSlotMappings,
                 tempRewritedPlan
         );
         // Can not rewrite, bail out
@@ -58,6 +64,6 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
     @Override
     protected boolean checkPattern(StructInfo structInfo) {
         // TODO Should get struct info from hyper graph and check
-        return false;
+        return true;
     }
 }

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java

@@ -22,24 +22,27 @@
 import org.apache.doris.nereids.memo.Group;
 import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
 import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionIndexMapping;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
 import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
 import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
 import org.apache.doris.nereids.util.ExpressionUtils;
 
-import com.clearspring.analytics.util.Lists;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -69,7 +72,7 @@ protected List<Plan> rewrite(Plan queryPlan, CascadesContext cascadesContext) {
         }
 
         for (MaterializationContext materializationContext : materializationContexts) {
-            Plan mvPlan = materializationContext.getMvPlan();
+            Plan mvPlan = materializationContext.getMtmv().getMvCache().getLogicalPlan();
             StructInfo viewStructInfo = extractStructInfo(mvPlan, cascadesContext);
             if (!checkPattern(viewStructInfo)) {
                 continue;
@@ -92,25 +95,28 @@ protected List<Plan> rewrite(Plan queryPlan, CascadesContext cascadesContext) {
                     continue;
                 }
                 Plan rewritedPlan;
-                Plan mvScan = materializationContext.getScanPlan();
+                Plan mvScan = materializationContext.getMvScanPlan();
                 if (compensatePredicates.isAlwaysTrue()) {
                     rewritedPlan = mvScan;
                 } else {
                     // Try to rewrite compensate predicates by using mv scan
                     List<NamedExpression> rewriteCompensatePredicates = rewriteExpression(
                             compensatePredicates.toList(),
-                            queryStructInfo,
-                            viewStructInfo,
-                            queryToViewTableMapping,
+                            materializationContext.getViewExpressionIndexMapping(),
+                            queryToViewSlotMapping,
                             mvScan);
                     if (rewriteCompensatePredicates.isEmpty()) {
                         continue;
                     }
                     rewritedPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan);
                 }
                 // Rewrite query by view
-                rewritedPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo,
-                        queryToViewTableMapping, rewritedPlan);
+                rewritedPlan = rewriteQueryByView(matchMode,
+                        queryStructInfo,
+                        viewStructInfo,
+                        queryToViewSlotMapping,
+                        rewritedPlan,
+                        materializationContext);
                 if (rewritedPlan == null) {
                     continue;
                 }
@@ -124,18 +130,19 @@ protected List<Plan> rewrite(Plan queryPlan, CascadesContext cascadesContext) {
     protected Plan rewriteQueryByView(MatchMode matchMode,
             StructInfo queryStructInfo,
             StructInfo viewStructInfo,
-            RelationMapping queryToViewTableMappings,
-            Plan tempRewritedPlan) {
+            SlotMapping queryToViewSlotMappings,
+            Plan tempRewritedPlan,
+            MaterializationContext materializationContext) {
         return tempRewritedPlan;
     }
 
-    /**Use target output expression to represent the source expression*/
-    protected List<NamedExpression> rewriteExpression(List<? extends Expression> sourceExpressions,
-            StructInfo sourceStructInfo,
-            StructInfo targetStructInfo,
-            RelationMapping sourceToTargetMapping,
+    /**Use target output expression to represent the source expression
+     * */
+    protected List<NamedExpression> rewriteExpression(
+            List<? extends Expression> sourceExpressions,
+            ExpressionMapping expressionMapping,
+            SlotMapping sourceToTargetMapping,
             Plan targetScanNode) {
-        // TODO represent the sourceExpressions by using target scan node
         // Firstly, rewrite the target plan output expression using query with inverse mapping
         // then try to use the mv expression to represent the query. if any of source expressions
         // can not be represented by mv, return null
@@ -148,18 +155,21 @@ protected List<NamedExpression> rewriteExpression(List<? extends Expression> sou
         //     transform source to:
         //        project(slot 2, 1)
         //            target
-        List<? extends Expression> targetTopExpressions = targetStructInfo.getExpressions();
-        List<? extends Expression> shuttledTargetExpressions = ExpressionUtils.shuttleExpressionWithLineage(
-                targetTopExpressions, targetStructInfo.getOriginalPlan(), Sets.newHashSet(), Sets.newHashSet());
-        SlotMapping sourceToTargetSlotMapping = SlotMapping.generate(sourceToTargetMapping);
-        // mv sql plan expressions transform to query based
-        List<? extends Expression> queryBasedExpressions = ExpressionUtils.replace(
-                shuttledTargetExpressions.stream().map(Expression.class::cast).collect(Collectors.toList()),
-                (Map)sourceToTargetSlotMapping.inverse().toMappedSlotMap());
-        // mv sql query based expression and index mapping
-        ExpressionIndexMapping.generate(queryBasedExpressions);
-        // TODO visit source expression and replace the expression with expressionIndexMapping
-        return ImmutableList.of();
+        List<Map<? extends Expression, ? extends Expression>> maps = expressionMapping.flattenMap();
+        // view to view scan expression is 1:1 so get first element
+        Map<? extends Expression, ? extends Expression> expressionMap = maps.get(0);
+
+        List<NamedExpression> result = new ArrayList<>();
+        for (Expression expressionToRewrite : sourceExpressions) {
+            final Set<Object> slotSet = expressionToRewrite.collectToSet(expression -> expression instanceof Slot);
+            Expression replacedExpression = ExpressionUtils.replace(expressionToRewrite, expressionMap, true);
+            if (replacedExpression.anyMatch(slotSet::contains)) {
+                // can not rewrite
+                return null;
+            }
+            result.add((NamedExpression) replacedExpression);
+        }
+        return result;
     }
 
     /**

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java

@@ -17,12 +17,15 @@
 
 package org.apache.doris.nereids.rules.exploration.mv;
 
+import org.apache.doris.catalog.MTMV;
 import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.View;
-import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.mtmv.MVCache;
 import org.apache.doris.nereids.memo.GroupId;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
 import org.apache.doris.nereids.trees.plans.Plan;
 
+import com.google.common.collect.ImmutableList;
+
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -32,22 +35,32 @@
  */
 public class MaterializationContext {
 
-    // TODO add MaterializedView class
-    private final Plan mvPlan;
-    private final CascadesContext context;
+    private MTMV mtmv;
+    // Should use stmt id generator in query context
+    private final Plan mvScanPlan;
     private final List<Table> baseTables;
-    private final List<View> baseViews;
+    private final List<Table> baseViews;
     // Group ids that are rewritten by this mv to reduce rewrite times
     private final Set<GroupId> matchedGroups = new HashSet<>();
-    private final Plan scanPlan;
+    // generate form mv scan plan
+    private ExpressionMapping viewExpressionMapping;
 
-    public MaterializationContext(Plan mvPlan, CascadesContext context,
-            List<Table> baseTables, List<View> baseViews, Plan scanPlan) {
-        this.mvPlan = mvPlan;
-        this.context = context;
+    public MaterializationContext(MTMV mtmv, Plan mvScanPlan,
+            List<Table> baseTables,
+            List<Table> baseViews) {
+        this.mtmv = mtmv;
+        this.mvScanPlan = mvScanPlan;
         this.baseTables = baseTables;
         this.baseViews = baseViews;
-        this.scanPlan = scanPlan;
+        MVCache mvCache = mtmv.getMvCache();
+        if (mvCache == null) {
+            // Laze init
+            mvCache = MVCache.from(mtmv);
+            mtmv.setMvCache(mvCache);
+        }
+        this.viewExpressionMapping = ExpressionMapping.generate(
+                mvCache.getMvOutputExpressions(),
+                mvScanPlan.getExpressions());
     }
 
     public Set<GroupId> getMatchedGroups() {
@@ -58,11 +71,32 @@ public void addMatchedGroup(GroupId groupId) {
         matchedGroups.add(groupId);
     }
 
-    public Plan getMvPlan() {
-        return mvPlan;
+    public MTMV getMtmv() {
+        return mtmv;
+    }
+
+    public Plan getMvScanPlan() {
+        return mvScanPlan;
+    }
+
+    public List<Table> getBaseTables() {
+        return baseTables;
+    }
+
+    public List<Table> getBaseViews() {
+        return baseViews;
+    }
+
+    public ExpressionMapping getViewExpressionIndexMapping() {
+        return viewExpressionMapping;
     }
 
-    public Plan getScanPlan() {
-        return scanPlan;
+    public static MaterializationContext fromMaterializedView(MTMV materializedView,
+            Plan mvScanPlan){
+        return new MaterializationContext(
+                materializedView,
+                mvScanPlan,
+                ImmutableList.of(),
+                ImmutableList.of());
     }
 }