diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index f14110eae23fb9d..a9a8b754d33648b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -19,6 +19,7 @@ /** * AbstractMaterializedViewAggregateRule + * This is responsible for common aggregate rewriting * */ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMaterializedViewRule { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 978e8d665ac06ac..3080377e32018ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -25,7 +25,7 @@ /** * AbstractMaterializedViewJoinRule - * This is responsible for join rewriting and join derivation + * This is responsible for common join rewriting */ public abstract class AbstractMaterializedViewJoinRule extends AbstractMaterializedViewRule { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 2a1c60a6537089e..8275df732637b5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -18,27 +18,22 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.catalog.TableIf; -import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.rules.exploration.mv.Mapping.ExpressionIndexMapping; +import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.metadata.EquivalenceClass; -import org.apache.doris.nereids.trees.metadata.Predicates; -import org.apache.doris.nereids.trees.metadata.Predicates.SplitPredicate; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; /** @@ -64,7 +59,10 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { if (!isPatternSupport(viewStructInfo)) { continue; } - MatchMode matchMode = decideMatchMode(queryStructInfo, viewStructInfo); + if (!StructInfo.isGraphLogicalEquals(queryStructInfo.getHyperGraph(), viewStructInfo.getHyperGraph())) { + continue; + } + MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations()); if (MatchMode.NOT_MATCH == matchMode) { continue; } @@ -135,11 +133,11 @@ protected List rewriteExpression(List sou // project(slot 2, 1) // target List targetTopExpressions = targetStructInfo.getExpressions(); - List shuttledTargetExpressions = shuttleExpressionWithLineage( + List shuttledTargetExpressions = ExpressionUtils.shuttleExpressionWithLineage( targetTopExpressions, targetStructInfo.getOriginalPlan(), Sets.newHashSet(), Sets.newHashSet()); SlotMapping sourceToTargetSlotMapping = SlotMapping.generate(sourceToTargetMapping); // mv sql plan expressions transform to query based - List queryBasedExpressions = permute(shuttledTargetExpressions, + List queryBasedExpressions = ExpressionUtils.permute(shuttledTargetExpressions, sourceToTargetSlotMapping.inverse()); // mv sql query based expression and index mapping ExpressionIndexMapping.generate(queryBasedExpressions); @@ -168,12 +166,12 @@ protected SplitPredicate predicatesCompensate( return SplitPredicate.empty(); } - private MatchMode decideMatchMode(StructInfo queryStructInfo, StructInfo viewStructInfo) { - List queryTableRefs = queryStructInfo.getRelations() + private MatchMode decideMatchMode(List queryRelations, List viewRelations) { + List queryTableRefs = queryRelations .stream() .map(CatalogRelation::getTable) .collect(Collectors.toList()); - List viewTableRefs = viewStructInfo.getRelations() + List viewTableRefs = viewRelations .stream() .map(CatalogRelation::getTable) .collect(Collectors.toList()); @@ -200,12 +198,8 @@ protected StructInfo extractStructInfo(Plan plan, CascadesContext cascadesContex return belongGroup.getStructInfo().get(); } else { // TODO build graph from plan and extract struct from graph and set to group if exist - // Should get from hyper graph - HyperGraph graph = new HyperGraph(); - List relations = new ArrayList<>(); - Predicates predicates = Predicates.of(new HashSet<>()); - // Set on current group and mv scan not set - StructInfo structInfo = StructInfo.of(relations, predicates, plan, graph); + // Should get structInfo from hyper graph and add into current group + StructInfo structInfo = StructInfo.of(plan); if (plan.getGroupExpression().isPresent()) { plan.getGroupExpression().get().getOwnerGroup().setStructInfo(structInfo); } @@ -213,26 +207,6 @@ protected StructInfo extractStructInfo(Plan plan, CascadesContext cascadesContex } } - // Replace the slot in expression with the lineage identifier from specified - // baseTable sets or target table types - // example as following: - // select a + 10 as a1, d from ( - // select b - 5 as a, d from table - // ); - // after shuttle a1 and d is [b - 5 + 10, d] - public static List shuttleExpressionWithLineage(List expression, - Plan plan, - Set targetTypes, - Set tableIdentifiers) { - return ImmutableList.of(); - } - - // Replace the slot in expressions according to the slotMapping - // if any slot cannot be mapped then return null - public static List permute(List expressions, SlotMapping slotMapping) { - return ImmutableList.of(); - } - protected boolean isPatternSupport(StructInfo structInfo) { if (structInfo.getRelations().isEmpty()) { return false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/EquivalenceClass.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java similarity index 95% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/EquivalenceClass.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java index ce6a7609c522438..2d9880c23bcaa45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/EquivalenceClass.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.trees.metadata; +package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -27,7 +27,7 @@ import java.util.Set; /** - * EquivalenceClass + * EquivalenceClass, this is used for equality propagation when predicate compensation */ public class EquivalenceClass { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java index b2e630306f6e93a..487fc92ce50a2bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java @@ -30,7 +30,7 @@ /** * Mapping slot from query to view or inversely, - * or mapping slot and it's index + * it can also represent the mapping from slot to it's index */ public abstract class Mapping { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 885cac2796dc94d..b0de1ccfa469bb7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -28,11 +28,11 @@ import java.util.Set; /** - * MaterializationContext + * Maintain the context for query rewrite by materialized view */ public class MaterializationContext { - // TODO: 2023/11/1 add MaterializedView class + // TODO add MaterializedView class private final Plan mvPlan; private final CascadesContext context; private final List baseTables; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java index 4db963317eb048a..ce9c208e5f5d160 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java @@ -23,7 +23,7 @@ import java.util.List; /** - * MaterializedViewAggregateRule + * This is responsible for aggregate rewriting according to different pattern * */ public class MaterializedViewAggregateRule extends AbstractMaterializedViewAggregateRule implements RewriteRuleFactory { @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java index 286ab4e2e9029f5..92f102dc1dece39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java @@ -30,7 +30,7 @@ import java.util.List; /** - * MaterializedViewJoinRule + * This is responsible for join rewriting according to different pattern * */ public class MaterializedViewProjectJoinRule extends AbstractMaterializedViewJoinRule implements RewriteRuleFactory { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedProjectFilterRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java similarity index 85% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedProjectFilterRule.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java index a3aebd3ef6d3299..c5909822adbb185 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedProjectFilterRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java @@ -23,9 +23,9 @@ import java.util.List; /** - * MaterializedProjectFilterRule + * This is responsible for single table rewriting according to different pattern * */ -public class MaterializedProjectFilterRule extends AbstractMaterializedViewRule implements RewriteRuleFactory { +public class MaterializedViewScanRule extends AbstractMaterializedViewRule implements RewriteRuleFactory { @Override public List buildRules() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java similarity index 92% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/Predicates.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index ecc5078465cc336..40b91994be76e9e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/metadata/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.trees.metadata; +package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; @@ -28,7 +28,7 @@ import java.util.Set; /** - * Predicates + * This record the predicates which can be pulled up or some other type predicates * */ public class Predicates { @@ -52,16 +52,16 @@ public Expression composedExpression() { } /** - * SplitPredicate + * Split the expression to equal, range and residual predicate. * */ public static SplitPredicate splitPredicates(Expression expression) { - PredicatesSpliter predicatesSplit = new PredicatesSpliter(); + PredicatesSpliter predicatesSplit = new PredicatesSpliter(expression); expression.accept(predicatesSplit, null); return predicatesSplit.getSplitPredicate(); } /** - * SplitPredicate + * The split different representation for predicate expression, such as equal, range and residual predicate. * */ public static final class SplitPredicate { private final Expression equalPredicates; @@ -91,7 +91,7 @@ public static SplitPredicate empty() { } /** - * SplitPredicate + * SplitPredicate construct * */ public static SplitPredicate of(Expression equalPredicates, Expression rangePredicates, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java index 5fabee6b293c8fc..ea91104a246e096 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java @@ -28,7 +28,10 @@ import java.util.List; /** - * RelationMapping + * Relation mapping + * such as query pattern is a1 left join a2 left join b + * view pattern is a1 left join a2 left join b. the mapping will be + * [{a1:a1, a2:a2, b:b}, {a1:a2, a2:a1, b:b}] */ public class RelationMapping extends Mapping { @@ -44,9 +47,6 @@ public BiMap getMappedRelationMap() { /** * Generate mapping according to source and target relation - * such as query pattern is a1 left join a2 left join b - * view pattern is a1 left join a2 left join b. the mapping will be - * [{a1:a1, a2:a2, b:b}, {a1:a2, a2:a1, b:b}] */ public static List generate(List source, List target) { Multimap queryTableRelationIdMap = ArrayListMultimap.create(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java index c9fcd3a4bf8c6e2..7c50d79c6a23c27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java @@ -20,7 +20,7 @@ import com.google.common.collect.BiMap; /** - * SlotMapping + * SlotMapping, this is open generated from relationMapping */ public class SlotMapping extends Mapping { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index a822ce95477bc3b..141b8a98bccfdbf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -18,12 +18,11 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.metadata.EquivalenceClass; -import org.apache.doris.nereids.trees.metadata.Predicates; -import org.apache.doris.nereids.trees.metadata.Predicates.SplitPredicate; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -61,11 +60,14 @@ private StructInfo(List relations, } } - public static StructInfo of(List relations, - Predicates predicates, - Plan originalPlan, - HyperGraph hyperGraph) { - return new StructInfo(relations, predicates, originalPlan, hyperGraph); + public static StructInfo of(Plan originalPlan) { + // TODO build graph from original plan and get relations and predicates from graph + return new StructInfo(null, null, originalPlan, null); + } + + public static StructInfo of(Group group) { + // TODO build graph from original plan and get relations and predicates from graph + return new StructInfo(null, null, group.getLogicalExpression().getPlan(), null); } public List getRelations() { @@ -92,4 +94,13 @@ public List getExpressions() { return originalPlan instanceof LogicalProject ? ((LogicalProject) originalPlan).getProjects() : originalPlan.getOutput(); } + + /** + * Judge the source graph logical is whether the same as target + * For inner join should judge only the join tables, + * for other join type should also judge the join direction, it's input filter that can not be pulled up etc. + * */ + public static boolean isGraphLogicalEquals(HyperGraph source, HyperGraph target) { + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java index 715e7ac6e3f4b95..1d6a82c84a39479 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java @@ -17,12 +17,12 @@ package org.apache.doris.nereids.trees.expressions.visitor; +import org.apache.doris.nereids.rules.exploration.mv.Predicates; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; -import org.apache.doris.nereids.trees.metadata.Predicates; import org.apache.doris.nereids.util.ExpressionUtils; import java.util.ArrayList; @@ -63,14 +63,19 @@ public Boolean visitAggregateFunction(AggregateFunction aggregateFunction, Void } /** - * Split the expression to - * Should new instance when used. + * Split the expression to equal, range and residual predicate. + * Should instance when used. */ public static class PredicatesSpliter extends DefaultExpressionVisitor { private List equalPredicates = new ArrayList<>(); private List rangePredicates = new ArrayList<>(); private List residualPredicates = new ArrayList<>(); + private final Expression target; + + public PredicatesSpliter(Expression target) { + this.target = target; + } @Override public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Void context) { @@ -87,6 +92,10 @@ public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Vo return super.visit(comparisonPredicate, context); } + public Expression getTarget() { + return target; + } + public Predicates.SplitPredicate getSplitPredicate() { return Predicates.SplitPredicate.of( equalPredicates.isEmpty() ? null : ExpressionUtils.and(equalPredicates), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index f3116ebdcd69c75..00512cb476e50b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -17,7 +17,9 @@ package org.apache.doris.nereids.util; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.exploration.mv.SlotMapping; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule; import org.apache.doris.nereids.trees.TreeNode; @@ -39,10 +41,10 @@ import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.Plan; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; @@ -197,22 +199,24 @@ public static Expression combine(Class type, Collection tableRelationMapping) { - return null; - } - - /** - * Given an expression, it will permute the column contained in its - * using the contents in the map. - */ - public static Expression permuteColumn(Expression expression, - BiMap columnMapping) { - return null; + // Replace the slot in expression with the lineage identifier from specified + // baseTable sets or target table types + // example as following: + // select a + 10 as a1, d from ( + // select b - 5 as a, d from table + // ); + // after shuttle a1 and d is [b - 5 + 10, d] + public static List shuttleExpressionWithLineage(List expression, + Plan plan, + Set targetTypes, + Set tableIdentifiers) { + return ImmutableList.of(); + } + + // Replace the slot in expressions according to the slotMapping + // if any slot cannot be mapped then return null + public static List permute(List expressions, SlotMapping slotMapping) { + return ImmutableList.of(); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 6cc51f68d9fc1ad..a706c6ba3a6bb83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1366,7 +1366,8 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { public int tableStatsHealthThreshold = 60; @VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_REWRITE, needForward = true, - description = {"是否开启基于结构信息的透明改写", "Whether to enable materialized rewriting based on struct info"}) + description = {"是否开启基于结构信息的透明改写", + "Whether to enable materialized rewriting based on struct info"}) public boolean enableMaterializedViewRewrite = false; @VariableMgr.VarAttr(name = MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_FOREIGN_TABLE, needForward = true, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewTest.java deleted file mode 100644 index 517810d3e1e72aa..000000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/MaterializedViewTest.java +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite.mv; - -import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; -import org.apache.doris.nereids.util.PlanChecker; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class MaterializedViewTest extends TestWithFeService { - - @Override - protected void runBeforeAll() throws Exception { - createDatabase("mv_poc"); - useDatabase("mv_poc"); - - createTable("CREATE TABLE lineitem (\n" - + " l_shipdate DATE NOT NULL,\n" - + " l_orderkey bigint NOT NULL,\n" - + " l_linenumber int not null,\n" - + " l_partkey int NOT NULL,\n" - + " l_suppkey int not null,\n" - + " l_quantity decimal(15, 2) NOT NULL,\n" - + " l_extendedprice decimal(15, 2) NOT NULL,\n" - + " l_discount decimal(15, 2) NOT NULL,\n" - + " l_tax decimal(15, 2) NOT NULL,\n" - + " l_returnflag VARCHAR(1) NOT NULL,\n" - + " l_linestatus VARCHAR(1) NOT NULL,\n" - + " l_commitdate DATE NOT NULL,\n" - + " l_receiptdate DATE NOT NULL,\n" - + " l_shipinstruct VARCHAR(25) NOT NULL,\n" - + " l_shipmode VARCHAR(10) NOT NULL,\n" - + " l_comment VARCHAR(44) NOT NULL\n" - + ")ENGINE=OLAP\n" - + "DUPLICATE KEY(`l_shipdate`, `l_orderkey`)\n" - + "COMMENT \"OLAP\"\n" - + "DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 32\n" - + "PROPERTIES (\n" - + " \"replication_num\" = \"1\",\n" - + " \"colocate_with\" = \"lineitem_orders\"\n" - + ");"); - - createTable("CREATE TABLE orders (\n" - + " o_orderkey bigint NOT NULL,\n" - + " o_orderdate DATE NOT NULL,\n" - + " o_custkey int NOT NULL,\n" - + " o_orderstatus VARCHAR(1) NOT NULL,\n" - + " o_totalprice decimal(15, 2) NOT NULL,\n" - + " o_orderpriority VARCHAR(15) NOT NULL,\n" - + " o_clerk VARCHAR(15) NOT NULL,\n" - + " o_shippriority int NOT NULL,\n" - + " o_comment VARCHAR(79) NOT NULL\n" - + ")ENGINE=OLAP\n" - + "DUPLICATE KEY(`o_orderkey`, `o_orderdate`)\n" - + "COMMENT \"OLAP\"\n" - + "DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 32\n" - + "PROPERTIES (\n" - + " \"replication_num\" = \"1\",\n" - + " \"colocate_with\" = \"lineitem_orders\"\n" - + ");"); - - createTableAsSelect("CREATE TABLE mv PROPERTIES(\"replication_num\" = \"1\") " - + "as select l_shipdate, l_linenumber from lineitem inner join orders on l_orderkey = o_orderkey;"); - } - - @Test - public void testInnerJoin() { - - connectContext.getSessionVariable().enableNereidsTimeout = false; - connectContext.getSessionVariable().enableDPHypOptimizer = true; - // query only l_orderkey from join(lineitem, orders) will output l_orderkey and o_orderkey - // PoC just use lineitem's field - PlanChecker.from(connectContext) - .checkMVRewrite( - "select l_shipdate from lineitem inner join orders on l_orderkey = o_orderkey", - "select l_shipdate, l_linenumber from lineitem inner join orders on l_orderkey = o_orderkey", - "select l_shipdate, l_linenumber from mv", - (queryPlanner, mvPlanner) -> { - PhysicalPlan physicalPlan = queryPlanner.getPhysicalPlan(); - Assertions.assertTrue( - ((PhysicalResultSink) physicalPlan).toJson().toString().contains("mv")); - } - ); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 3a0974aa6c3f9fd..9274fdd0abd4dca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -49,7 +49,6 @@ import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; -import org.apache.doris.nereids.rules.exploration.mv.MaterializationContext; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.GroupPlan; @@ -74,7 +73,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Supplier; @@ -169,7 +167,7 @@ public PlanChecker applyTopDown(RuleFactory ruleFactory) { public PlanChecker applyTopDown(List rule) { Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext, - ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteTopDownJob::new, true))) + ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteTopDownJob::new, true))) .execute(); cascadesContext.toMemo(); MemoValidator.validate(cascadesContext.getMemo()); @@ -581,46 +579,6 @@ public PlanChecker checkPlannerResult(String sql) { }); } - public PlanChecker checkMVRewrite(String sql, String mvSql, String mvScanSql, - BiConsumer consumer) { - - PhysicalProperties physicalProperties = NereidsPlanner.buildInitRequireProperties(); - // Mock materialized view define sql - LogicalPlan mvUnboundPlan = new NereidsParser().parseSingle(mvSql); - NereidsPlanner mvPlanner = new NereidsPlanner( - new StatementContext(connectContext, new OriginStatement(mvSql, 0))); - mvPlanner.plan(mvUnboundPlan, physicalProperties, ExplainLevel.ALL_PLAN); - - // mock the mv scan, this should be from materializedView - LogicalPlan mvScanUnboundPlan = new NereidsParser().parseSingle(mvScanSql); - NereidsPlanner mvScanPlanner = new NereidsPlanner( - new StatementContext(connectContext, new OriginStatement(mvScanSql, 0))); - mvScanPlanner.plan(mvScanUnboundPlan, physicalProperties, ExplainLevel.ALL_PLAN); - - // mock the mv context and query rewrite, should call actual materialized view instead - LogicalPlan queryUnboundPlan = new NereidsParser().parseSingle(sql); - StatementContext queryStmtContext = new StatementContext(connectContext, new OriginStatement(sql, 0)); - NereidsPlanner queryPlanner = new NereidsPlanner(queryStmtContext); - CascadesContext queryCascadesContext = - CascadesContext.initContext(queryStmtContext, queryUnboundPlan, physicalProperties); - if (queryStmtContext.getConnectContext().getTables() != null) { - queryCascadesContext.setTables(queryStmtContext.getConnectContext().getTables()); - } - - MaterializationContext mvContext = new MaterializationContext( - mvPlanner.getRewrittenPlan(), - queryCascadesContext, - ImmutableList.of(), - ImmutableList.of(), - mvScanPlanner.getRewrittenPlan() - ); - queryCascadesContext.addMaterializationContext(mvContext); - - queryPlanner.plan(LogicalPlanAdapter.of(queryUnboundPlan)); - consumer.accept(queryPlanner, mvPlanner); - return this; - } - public CascadesContext getCascadesContext() { return cascadesContext; }