diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java index d45fa70cf33d543..231ccbf0f67f538 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java @@ -33,7 +33,9 @@ import java.util.List; import java.util.stream.Collectors; -/**The cache for materialized view cache */ +/** + * The cache for materialized view cache + */ public class MVCache { // the materialized view plan which should be optimized by the same rules to query @@ -67,12 +69,13 @@ public static MVCache from(MTMV mtmv, ConnectContext connectContext) { new OriginStatement(mtmv.getQuerySql(), 0)); NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); - planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); - Plan mvAnalyzedPlan = planner.getAnalyzedPlan(); - Plan mvRewrittenPlan = planner.getRewrittenPlan(); + Plan mvRewrittenPlan = + planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); Plan mvPlan = mvRewrittenPlan instanceof LogicalResultSink ? (Plan) ((LogicalResultSink) mvRewrittenPlan).child() : mvRewrittenPlan; - List mvOutputExpressions = mvAnalyzedPlan.getExpressions().stream() + // use rewritten plan output expression currently, if expression rewrite fail, + // consider to use the analyzed plan for output expressions only + List mvOutputExpressions = mvRewrittenPlan.getExpressions().stream() .map(NamedExpression.class::cast) .collect(Collectors.toList()); return new MVCache(mvPlan, mvOutputExpressions); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 4574b9c8c28e1e5..acc6c271ca29d2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -40,6 +40,8 @@ * This is responsible for common join rewriting */ public abstract class AbstractMaterializedViewJoinRule extends AbstractMaterializedViewRule { + private static final HashSet SUPPORTED_JOIN_TYPE_SET = + Sets.newHashSet(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN); @Override protected Plan rewriteQueryByView(MatchMode matchMode, @@ -72,21 +74,22 @@ protected Plan rewriteQueryByView(MatchMode matchMode, tempRewritedPlan); } - // Check join is whether valid or not. Support join's input can not contain aggregate - // Only support project, filter, join, logical relation node and - // join condition should be slot reference equals currently + /** + * Check join is whether valid or not. Support join's input can not contain aggregate + * Only support project, filter, join, logical relation node and + * join condition should be slot reference equals currently + */ @Override protected boolean checkPattern(StructInfo structInfo) { HyperGraph hyperGraph = structInfo.getHyperGraph(); - HashSet requiredJoinType = Sets.newHashSet(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN); for (AbstractNode node : hyperGraph.getNodes()) { StructInfoNode structInfoNode = (StructInfoNode) node; if (!structInfoNode.getPlan().accept(StructInfo.JOIN_PATTERN_CHECKER, - requiredJoinType)) { + SUPPORTED_JOIN_TYPE_SET)) { return false; } for (Edge edge : hyperGraph.getEdges()) { - if (!edge.getJoin().accept(StructInfo.JOIN_PATTERN_CHECKER, requiredJoinType)) { + if (!edge.getJoin().accept(StructInfo.JOIN_PATTERN_CHECKER, SUPPORTED_JOIN_TYPE_SET)) { return false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index d5c6e9053387751..5eff40bd4bd8967 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -160,8 +160,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, * Use target output expression to represent the source expression */ protected List rewriteExpression( - List sourceExpressions, - ExpressionMapping expressionMapping, + List sourceExpressionsToWrite, + ExpressionMapping mvExpressionToMvScanExpressionMapping, SlotMapping sourceToTargetMapping) { // Firstly, rewrite the target plan output expression using query with inverse mapping // then try to use the mv expression to represent the query. if any of source expressions @@ -175,16 +175,16 @@ protected List rewriteExpression( // transform source to: // project(slot 2, 1) // target - // generate mvSql to mvScan expressionMapping, and change mv sql expression to query based + // generate mvSql to mvScan mvExpressionToMvScanExpressionMapping, and change mv sql expression to query based ExpressionMapping expressionMappingKeySourceBased = - expressionMapping.keyPermute(sourceToTargetMapping.inverse()); + mvExpressionToMvScanExpressionMapping.keyPermute(sourceToTargetMapping.inverse()); List> flattenExpressionMap = expressionMappingKeySourceBased.flattenMap(); // view to view scan expression is 1:1 so get first element Map mvSqlToMvScanMappingQueryBased = flattenExpressionMap.get(0); List rewrittenExpressions = new ArrayList<>(); - for (Expression expressionToRewrite : sourceExpressions) { + for (Expression expressionToRewrite : sourceExpressionsToWrite) { if (expressionToRewrite instanceof Literal) { rewrittenExpressions.add(expressionToRewrite); continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index 4f116b7aed0a60f..96f89a513cf1a0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -74,26 +74,26 @@ public static SplitPredicate splitPredicates(Expression expression) { * The split different representation for predicate expression, such as equal, range and residual predicate. */ public static final class SplitPredicate { - private Optional equalPredicates; - private Optional rangePredicates; - private Optional residualPredicates; - - public SplitPredicate(Expression equalPredicates, Expression rangePredicates, Expression residualPredicates) { - this.equalPredicates = Optional.ofNullable(equalPredicates); - this.rangePredicates = Optional.ofNullable(rangePredicates); - this.residualPredicates = Optional.ofNullable(residualPredicates); + private Optional equalPredicate; + private Optional rangePredicate; + private Optional residualPredicate; + + public SplitPredicate(Expression equalPredicate, Expression rangePredicate, Expression residualPredicate) { + this.equalPredicate = Optional.ofNullable(equalPredicate); + this.rangePredicate = Optional.ofNullable(rangePredicate); + this.residualPredicate = Optional.ofNullable(residualPredicate); } public Expression getEqualPredicate() { - return equalPredicates.orElse(BooleanLiteral.TRUE); + return equalPredicate.orElse(BooleanLiteral.TRUE); } public Expression getRangePredicate() { - return rangePredicates.orElse(BooleanLiteral.TRUE); + return rangePredicate.orElse(BooleanLiteral.TRUE); } public Expression getResidualPredicate() { - return residualPredicates.orElse(BooleanLiteral.TRUE); + return residualPredicate.orElse(BooleanLiteral.TRUE); } public static SplitPredicate empty() { @@ -113,24 +113,24 @@ public static SplitPredicate of(Expression equalPredicates, * isEmpty */ public boolean isEmpty() { - return !equalPredicates.isPresent() - && !rangePredicates.isPresent() - && !residualPredicates.isPresent(); + return !equalPredicate.isPresent() + && !rangePredicate.isPresent() + && !residualPredicate.isPresent(); } public List toList() { - return ImmutableList.of(equalPredicates.orElse(BooleanLiteral.TRUE), - rangePredicates.orElse(BooleanLiteral.TRUE), - residualPredicates.orElse(BooleanLiteral.TRUE)); + return ImmutableList.of(equalPredicate.orElse(BooleanLiteral.TRUE), + rangePredicate.orElse(BooleanLiteral.TRUE), + residualPredicate.orElse(BooleanLiteral.TRUE)); } /** * Check the predicates in SplitPredicate is whether all true or not */ public boolean isAlwaysTrue() { - Expression equalExpr = equalPredicates.orElse(BooleanLiteral.TRUE); - Expression rangeExpr = rangePredicates.orElse(BooleanLiteral.TRUE); - Expression residualExpr = residualPredicates.orElse(BooleanLiteral.TRUE); + Expression equalExpr = equalPredicate.orElse(BooleanLiteral.TRUE); + Expression rangeExpr = rangePredicate.orElse(BooleanLiteral.TRUE); + Expression residualExpr = residualPredicate.orElse(BooleanLiteral.TRUE); return equalExpr instanceof BooleanLiteral && rangeExpr instanceof BooleanLiteral && residualExpr instanceof BooleanLiteral diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java index 2975b0c61aaf47f..5f9a5e242de62e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java @@ -21,9 +21,8 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.CompoundPredicate; -import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -40,10 +39,10 @@ */ public class PredicatesSplitter { - private List equalPredicates = new ArrayList<>(); - private List rangePredicates = new ArrayList<>(); - private List residualPredicates = new ArrayList<>(); - private List conjunctExpressions; + private final List equalPredicates = new ArrayList<>(); + private final List rangePredicates = new ArrayList<>(); + private final List residualPredicates = new ArrayList<>(); + private final List conjunctExpressions; private final PredicateExtract instance = new PredicateExtract(); @@ -54,7 +53,9 @@ public PredicatesSplitter(Expression target) { } } - /**PredicateExtract*/ + /** + * PredicateExtract + */ public class PredicateExtract extends DefaultExpressionVisitor { @Override @@ -63,7 +64,7 @@ public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Ex Expression rightArg = comparisonPredicate.getArgument(1); boolean leftArgOnlyContainsColumnRef = containOnlyColumnRef(leftArg, true); boolean rightArgOnlyContainsColumnRef = containOnlyColumnRef(rightArg, true); - if (comparisonPredicate instanceof EqualTo || comparisonPredicate instanceof NullSafeEqual) { + if (comparisonPredicate instanceof EqualPredicate) { if (leftArgOnlyContainsColumnRef && rightArgOnlyContainsColumnRef) { equalPredicates.add(comparisonPredicate); return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java index 7fb857d3d4c77c1..fdf47d13433f00a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java @@ -17,13 +17,13 @@ package org.apache.doris.nereids.rules.exploration.mv.mapping; -import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableBiMap.Builder; import com.google.common.collect.ImmutableList; import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.Lists; @@ -40,9 +40,9 @@ */ public class RelationMapping extends Mapping { - private final BiMap mappedRelationMap; + private final ImmutableBiMap mappedRelationMap; - public RelationMapping(BiMap mappedRelationMap) { + public RelationMapping(ImmutableBiMap mappedRelationMap) { this.mappedRelationMap = mappedRelationMap; } @@ -50,7 +50,7 @@ public BiMap getMappedRelationMap() { return mappedRelationMap; } - public static RelationMapping of(BiMap mappedRelationMap) { + public static RelationMapping of(ImmutableBiMap mappedRelationMap) { return new RelationMapping(mappedRelationMap); } @@ -59,27 +59,20 @@ public static RelationMapping of(BiMap mappedRel */ public static List generate(List sources, List targets) { // Construct tmp map, key is the table qualifier, value is the corresponding catalog relations - LinkedListMultimap sourceTableRelationIdMap = LinkedListMultimap.create(); + LinkedListMultimap sourceTableRelationIdMap = LinkedListMultimap.create(); for (CatalogRelation relation : sources) { - String tableQualifier = getTableQualifier(relation.getTable()); - if (tableQualifier == null) { - return null; - } - sourceTableRelationIdMap.put(tableQualifier, MappedRelation.of(relation.getRelationId(), relation)); + sourceTableRelationIdMap.put(getTableQualifier(relation.getTable()), + MappedRelation.of(relation.getRelationId(), relation)); } - LinkedListMultimap targetTableRelationIdMap = LinkedListMultimap.create(); + LinkedListMultimap targetTableRelationIdMap = LinkedListMultimap.create(); for (CatalogRelation relation : targets) { - String tableQualifier = getTableQualifier(relation.getTable()); - if (tableQualifier == null) { - return null; - } - targetTableRelationIdMap.put(tableQualifier, MappedRelation.of(relation.getRelationId(), relation)); + targetTableRelationIdMap.put(getTableQualifier(relation.getTable()), + MappedRelation.of(relation.getRelationId(), relation)); } - - Set sourceTableKeySet = sourceTableRelationIdMap.keySet(); + Set sourceTableKeySet = sourceTableRelationIdMap.keySet(); List>> mappedRelations = new ArrayList<>(); - for (String sourceTableQualifier : sourceTableKeySet) { + for (Long sourceTableQualifier : sourceTableKeySet) { List sourceMappedRelations = sourceTableRelationIdMap.get(sourceTableQualifier); List targetMappedRelations = targetTableRelationIdMap.get(sourceTableQualifier); if (targetMappedRelations.isEmpty()) { @@ -104,23 +97,17 @@ public static List generate(List sources, List return Lists.cartesianProduct(mappedRelations).stream() .map(mappedRelationList -> { - BiMap relationMappedRelationBiMap = - HashBiMap.create(mappedRelationCount); + Builder mapBuilder = ImmutableBiMap.builder(); for (int relationIndex = 0; relationIndex < mappedRelationCount; relationIndex++) { - relationMappedRelationBiMap.put(mappedRelationList.get(relationIndex).key(), + mapBuilder.put(mappedRelationList.get(relationIndex).key(), mappedRelationList.get(relationIndex).value()); } - return RelationMapping.of(relationMappedRelationBiMap); + return RelationMapping.of(mapBuilder.build()); }) .collect(ImmutableList.toImmutableList()); } - private static String getTableQualifier(TableIf tableIf) { - String tableName = tableIf.getName(); - DatabaseIf database = tableIf.getDatabase(); - if (database == null) { - return null; - } - return database.getFullName() + ":" + tableName; + private static Long getTableQualifier(TableIf tableIf) { + return tableIf.getId(); } }