multi mv add some comments

seawinde · Nov 15, 2023 · f426e1b · f426e1b
1 parent 5051496
commit f426e1b
Show file tree

Hide file tree

Showing 18 changed files with 90 additions and 235 deletions.
diff --git a/.../org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/.../org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
@@ -19,6 +19,7 @@
 
 /**
  * AbstractMaterializedViewAggregateRule
+ * This is responsible for common aggregate rewriting
  * */
 public abstract class AbstractMaterializedViewAggregateRule extends AbstractMaterializedViewRule {
 }
diff --git a/.../java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/.../java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
@@ -25,7 +25,7 @@
 
 /**
  * AbstractMaterializedViewJoinRule
- * This is responsible for join rewriting and join derivation
+ * This is responsible for common join rewriting
  */
 public abstract class AbstractMaterializedViewJoinRule extends AbstractMaterializedViewRule {
 

diff --git a/...main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/...main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -18,27 +18,22 @@
 package org.apache.doris.nereids.rules.exploration.mv;
 
 import org.apache.doris.catalog.TableIf;
-import org.apache.doris.catalog.TableIf.TableType;
 import org.apache.doris.nereids.CascadesContext;
-import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
 import org.apache.doris.nereids.memo.Group;
 import org.apache.doris.nereids.rules.exploration.mv.Mapping.ExpressionIndexMapping;
+import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.metadata.EquivalenceClass;
-import org.apache.doris.nereids.trees.metadata.Predicates;
-import org.apache.doris.nereids.trees.metadata.Predicates.SplitPredicate;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.util.ExpressionUtils;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Sets;
 
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
@@ -64,7 +59,10 @@ protected List<Plan> rewrite(Plan queryPlan, CascadesContext cascadesContext) {
             if (!isPatternSupport(viewStructInfo)) {
                 continue;
             }
-            MatchMode matchMode = decideMatchMode(queryStructInfo, viewStructInfo);
+            if (!StructInfo.isGraphLogicalEquals(queryStructInfo.getHyperGraph(), viewStructInfo.getHyperGraph())) {
+                continue;
+            }
+            MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations());
             if (MatchMode.NOT_MATCH == matchMode) {
                 continue;
             }
@@ -135,11 +133,11 @@ protected List<NamedExpression> rewriteExpression(List<? extends Expression> sou
         //        project(slot 2, 1)
         //            target
         List<? extends Expression> targetTopExpressions = targetStructInfo.getExpressions();
-        List<? extends Expression> shuttledTargetExpressions = shuttleExpressionWithLineage(
+        List<? extends Expression> shuttledTargetExpressions = ExpressionUtils.shuttleExpressionWithLineage(
                 targetTopExpressions, targetStructInfo.getOriginalPlan(), Sets.newHashSet(), Sets.newHashSet());
         SlotMapping sourceToTargetSlotMapping = SlotMapping.generate(sourceToTargetMapping);
         // mv sql plan expressions transform to query based
-        List<? extends Expression> queryBasedExpressions = permute(shuttledTargetExpressions,
+        List<? extends Expression> queryBasedExpressions = ExpressionUtils.permute(shuttledTargetExpressions,
                 sourceToTargetSlotMapping.inverse());
         // mv sql query based expression and index mapping
         ExpressionIndexMapping.generate(queryBasedExpressions);
@@ -168,12 +166,12 @@ protected SplitPredicate predicatesCompensate(
         return SplitPredicate.empty();
     }
 
-    private MatchMode decideMatchMode(StructInfo queryStructInfo, StructInfo viewStructInfo) {
-        List<TableIf> queryTableRefs = queryStructInfo.getRelations()
+    private MatchMode decideMatchMode(List<CatalogRelation> queryRelations, List<CatalogRelation> viewRelations) {
+        List<TableIf> queryTableRefs = queryRelations
                 .stream()
                 .map(CatalogRelation::getTable)
                 .collect(Collectors.toList());
-        List<TableIf> viewTableRefs = viewStructInfo.getRelations()
+        List<TableIf> viewTableRefs = viewRelations
                 .stream()
                 .map(CatalogRelation::getTable)
                 .collect(Collectors.toList());
@@ -200,39 +198,15 @@ protected StructInfo extractStructInfo(Plan plan, CascadesContext cascadesContex
             return belongGroup.getStructInfo().get();
         } else {
             // TODO build graph from plan and extract struct from graph and set to group if exist
-            // Should get from hyper graph
-            HyperGraph graph = new HyperGraph();
-            List<CatalogRelation> relations = new ArrayList<>();
-            Predicates predicates = Predicates.of(new HashSet<>());
-            // Set on current group and mv scan not set
-            StructInfo structInfo = StructInfo.of(relations, predicates, plan, graph);
+            // Should get structInfo from hyper graph and add into current group
+            StructInfo structInfo = StructInfo.of(plan);
             if (plan.getGroupExpression().isPresent()) {
                 plan.getGroupExpression().get().getOwnerGroup().setStructInfo(structInfo);
             }
             return structInfo;
         }
     }
 
-    // Replace the slot in expression with the lineage identifier from specified
-    // baseTable sets or target table types
-    // example as following:
-    // select a + 10 as a1, d from (
-    //     select b - 5 as a, d from table
-    // );
-    // after shuttle a1 and d is [b - 5 + 10, d]
-    public static List<? extends Expression> shuttleExpressionWithLineage(List<? extends Expression> expression,
-            Plan plan,
-            Set<TableType> targetTypes,
-            Set<String> tableIdentifiers) {
-        return ImmutableList.of();
-    }
-
-    // Replace the slot in expressions according to the slotMapping
-    // if any slot cannot be mapped then return null
-    public static List<? extends Expression> permute(List<? extends Expression> expressions, SlotMapping slotMapping) {
-        return ImmutableList.of();
-    }
-
     protected boolean isPatternSupport(StructInfo structInfo) {
         if (structInfo.getRelations().isEmpty()) {
             return false;

diff --git a/...eids/trees/metadata/EquivalenceClass.java → ...ules/exploration/mv/EquivalenceClass.java b/...eids/trees/metadata/EquivalenceClass.java → ...ules/exploration/mv/EquivalenceClass.java
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package org.apache.doris.nereids.trees.metadata;
+package org.apache.doris.nereids.rules.exploration.mv;
 
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 
@@ -27,7 +27,7 @@
 import java.util.Set;
 
 /**
- * EquivalenceClass
+ * EquivalenceClass, this is used for equality propagation when predicate compensation
  */
 public class EquivalenceClass {
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java
@@ -30,7 +30,7 @@
 
 /**
  * Mapping slot from query to view or inversely,
- * or mapping slot and it's index
+ * it can also represent the mapping from slot to it's index
  */
 public abstract class Mapping {
 

diff --git a/...e/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/...e/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
@@ -28,11 +28,11 @@
 import java.util.Set;
 
 /**
- * MaterializationContext
+ * Maintain the context for query rewrite by materialized view
  */
 public class MaterializationContext {
 
-    // TODO: 2023/11/1 add MaterializedView class
+    // TODO add MaterializedView class
     private final Plan mvPlan;
     private final CascadesContext context;
     private final List<Table> baseTables;

diff --git a/...ain/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java b/...ain/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
@@ -23,7 +23,7 @@
 import java.util.List;
 
 /**
- * MaterializedViewAggregateRule
+ * This is responsible for aggregate rewriting according to different pattern
  * */
 public class MaterializedViewAggregateRule extends AbstractMaterializedViewAggregateRule implements RewriteRuleFactory {
     @Override

diff --git a/...n/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java b/...n/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
@@ -30,7 +30,7 @@
 import java.util.List;
 
 /**
- * MaterializedViewJoinRule
+ * This is responsible for join rewriting according to different pattern
  * */
 public class MaterializedViewProjectJoinRule extends AbstractMaterializedViewJoinRule implements RewriteRuleFactory {
 

diff --git a/...ion/mv/MaterializedProjectFilterRule.java → ...loration/mv/MaterializedViewScanRule.java b/...ion/mv/MaterializedProjectFilterRule.java → ...loration/mv/MaterializedViewScanRule.java
@@ -23,9 +23,9 @@
 import java.util.List;
 
 /**
- * MaterializedProjectFilterRule
+ * This is responsible for single table rewriting according to different pattern
  * */
-public class MaterializedProjectFilterRule extends AbstractMaterializedViewRule implements RewriteRuleFactory {
+public class MaterializedViewScanRule extends AbstractMaterializedViewRule implements RewriteRuleFactory {
 
     @Override
     public List<Rule> buildRules() {

diff --git a/...is/nereids/trees/metadata/Predicates.java → ...eids/rules/exploration/mv/Predicates.java b/...is/nereids/trees/metadata/Predicates.java → ...eids/rules/exploration/mv/Predicates.java
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package org.apache.doris.nereids.trees.metadata;
+package org.apache.doris.nereids.rules.exploration.mv;
 
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
@@ -28,7 +28,7 @@
 import java.util.Set;
 
 /**
- * Predicates
+ * This record the predicates which can be pulled up or some other type predicates
  * */
 public class Predicates {
 
@@ -52,16 +52,16 @@ public Expression composedExpression() {
     }
 
     /**
-     * SplitPredicate
+     * Split the expression to equal, range and residual predicate.
      * */
     public static SplitPredicate splitPredicates(Expression expression) {
-        PredicatesSpliter predicatesSplit = new PredicatesSpliter();
+        PredicatesSpliter predicatesSplit = new PredicatesSpliter(expression);
         expression.accept(predicatesSplit, null);
         return predicatesSplit.getSplitPredicate();
     }
 
     /**
-     * SplitPredicate
+     * The split different representation for predicate expression, such as equal, range and residual predicate.
      * */
     public static final class SplitPredicate {
         private final Expression equalPredicates;
@@ -91,7 +91,7 @@ public static SplitPredicate empty() {
         }
 
         /**
-         * SplitPredicate
+         * SplitPredicate construct
          * */
         public static SplitPredicate of(Expression equalPredicates,
                 Expression rangePredicates,

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java
@@ -28,7 +28,10 @@
 import java.util.List;
 
 /**
- * RelationMapping
+ * Relation mapping
+ * such as query pattern is a1 left join a2 left join b
+ * view pattern is a1 left join a2 left join b. the mapping will be
+ * [{a1:a1, a2:a2, b:b}, {a1:a2, a2:a1, b:b}]
  */
 public class RelationMapping extends Mapping {
 
@@ -44,9 +47,6 @@ public BiMap<MappedRelation, MappedRelation> getMappedRelationMap() {
 
     /**
      * Generate mapping according to source and target relation
-     * such as query pattern is a1 left join a2 left join b
-     * view pattern is a1 left join a2 left join b. the mapping will be
-     * [{a1:a1, a2:a2, b:b}, {a1:a2, a2:a1, b:b}]
      */
     public static List<RelationMapping> generate(List<CatalogRelation> source, List<CatalogRelation> target) {
         Multimap<TableIf, CatalogRelation> queryTableRelationIdMap = ArrayListMultimap.create();

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java
@@ -20,7 +20,7 @@
 import com.google.common.collect.BiMap;
 
 /**
- * SlotMapping
+ * SlotMapping, this is open generated from relationMapping
  */
 public class SlotMapping extends Mapping {
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -18,12 +18,11 @@
 package org.apache.doris.nereids.rules.exploration.mv;
 
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
+import org.apache.doris.nereids.memo.Group;
+import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
-import org.apache.doris.nereids.trees.metadata.EquivalenceClass;
-import org.apache.doris.nereids.trees.metadata.Predicates;
-import org.apache.doris.nereids.trees.metadata.Predicates.SplitPredicate;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@@ -61,11 +60,14 @@ private StructInfo(List<CatalogRelation> relations,
         }
     }
 
-    public static StructInfo of(List<CatalogRelation> relations,
-            Predicates predicates,
-            Plan originalPlan,
-            HyperGraph hyperGraph) {
-        return new StructInfo(relations, predicates, originalPlan, hyperGraph);
+    public static StructInfo of(Plan originalPlan) {
+        // TODO build graph from original plan and get relations and predicates from graph
+        return new StructInfo(null, null, originalPlan, null);
+    }
+
+    public static StructInfo of(Group group) {
+        // TODO build graph from original plan and get relations and predicates from graph
+        return new StructInfo(null, null, group.getLogicalExpression().getPlan(), null);
     }
 
     public List<CatalogRelation> getRelations() {
@@ -92,4 +94,13 @@ public List<? extends Expression> getExpressions() {
         return originalPlan instanceof LogicalProject
                 ? ((LogicalProject<Plan>) originalPlan).getProjects() : originalPlan.getOutput();
     }
+
+    /**
+     * Judge the source graph logical is whether the same as target
+     * For inner join should judge only the join tables,
+     * for other join type should also judge the join direction, it's input filter that can not be pulled up etc.
+     * */
+    public static boolean isGraphLogicalEquals(HyperGraph source, HyperGraph target) {
+        return false;
+    }
 }
diff --git a/.../src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java b/.../src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java
@@ -17,12 +17,12 @@
 
 package org.apache.doris.nereids.trees.expressions.visitor;
 
+import org.apache.doris.nereids.rules.exploration.mv.Predicates;
 import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.WindowExpression;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
-import org.apache.doris.nereids.trees.metadata.Predicates;
 import org.apache.doris.nereids.util.ExpressionUtils;
 
 import java.util.ArrayList;
@@ -63,14 +63,19 @@ public Boolean visitAggregateFunction(AggregateFunction aggregateFunction, Void
     }
 
     /**
-     * Split the expression to
-     * Should new instance when used.
+     * Split the expression to equal, range and residual predicate.
+     * Should instance when used.
      */
     public static class PredicatesSpliter extends DefaultExpressionVisitor<Void, Void> {
 
         private List<Expression> equalPredicates = new ArrayList<>();
         private List<Expression> rangePredicates = new ArrayList<>();
         private List<Expression> residualPredicates = new ArrayList<>();
+        private final Expression target;
+
+        public PredicatesSpliter(Expression target) {
+            this.target = target;
+        }
 
         @Override
         public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Void context) {
@@ -87,6 +92,10 @@ public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Vo
             return super.visit(comparisonPredicate, context);
         }
 
+        public Expression getTarget() {
+            return target;
+        }
+
         public Predicates.SplitPredicate getSplitPredicate() {
             return Predicates.SplitPredicate.of(
                     equalPredicates.isEmpty() ? null : ExpressionUtils.and(equalPredicates),