diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index 4fd0919774b3cc..07fda55e49413d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -64,7 +64,6 @@ public MTMVCache(Plan logicalPlan, List mvOutputExpressions) { public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) { LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql()); // TODO: connect context set current db when create mv by use database - // view should also disable the predicate infer and join eliminate. StatementContext mvSqlStatementContext = new StatementContext(connectContext, new OriginStatement(mtmv.getQuerySql(), 0)); NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index b418d29e8d3104..11da5104255fc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -50,8 +50,7 @@ public class MTMVRelationManager implements MTMVHookService { private Map> tableMTMVs = Maps.newConcurrentMap(); public Set getMtmvsByBaseTable(BaseTableInfo table) { - Set baseTableInfos = tableMTMVs.get(table); - return baseTableInfos == null ? ImmutableSet.of() : baseTableInfos; + return tableMTMVs.getOrDefault(table, ImmutableSet.of()); } public Set getAvailableMTMVs(List tableInfos) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java index 54d9b97bb085a8..b2acdddd64fec5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java @@ -33,7 +33,6 @@ * @see PlaceholderCollector */ public class PlaceholderExpression extends Expression implements AlwaysNotNullable { - protected boolean distinct; private final Class delegateClazz; /** * 1 based @@ -46,23 +45,10 @@ public PlaceholderExpression(List children, Class children, Class delegateClazz, int position, - boolean distinct) { - super(children); - this.delegateClazz = Objects.requireNonNull(delegateClazz, "delegateClazz should not be null"); - this.position = position; - this.distinct = distinct; - } - public static PlaceholderExpression of(Class delegateClazz, int position) { return new PlaceholderExpression(ImmutableList.of(), delegateClazz, position); } - public static PlaceholderExpression of(Class delegateClazz, int position, - boolean distinct) { - return new PlaceholderExpression(ImmutableList.of(), delegateClazz, position, distinct); - } - @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visit(this, context); @@ -76,10 +62,6 @@ public int getPosition() { return position; } - public boolean isDistinct() { - return distinct; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -92,13 +74,11 @@ public boolean equals(Object o) { return false; } PlaceholderExpression that = (PlaceholderExpression) o; - return position == that.position - && Objects.equals(delegateClazz, that.delegateClazz) - && distinct == that.distinct; + return position == that.position && Objects.equals(delegateClazz, that.delegateClazz); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), delegateClazz, position, distinct); + return Objects.hash(super.hashCode(), delegateClazz, position); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 5c0ff200a67da5..a9ce2cce17412d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -86,7 +86,6 @@ import org.apache.doris.nereids.rules.rewrite.MergeGenerates; import org.apache.doris.nereids.rules.rewrite.MergeLimits; import org.apache.doris.nereids.rules.rewrite.MergeProjects; -import org.apache.doris.nereids.rules.rewrite.OrExpansion; import org.apache.doris.nereids.rules.rewrite.PushDownAliasThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownExpressionsInHashCondition; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughAggregation; @@ -126,7 +125,6 @@ public class RuleSet { .add(PushDownProjectThroughSemiJoin.INSTANCE) .add(TransposeAggSemiJoin.INSTANCE) .add(TransposeAggSemiJoinProject.INSTANCE) - .add(OrExpansion.INSTANCE) .build(); public static final List PUSH_DOWN_FILTERS = ImmutableList.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 5b56c0afe6fdd4..e3784c2969b324 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.common.Pair; -import org.apache.doris.nereids.analyzer.PlaceholderExpression; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.AbstractNode; @@ -30,21 +29,18 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.Any; +import org.apache.doris.nereids.trees.expressions.functions.CouldRollUp; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion; -import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; -import org.apache.doris.nereids.trees.expressions.functions.agg.Max; -import org.apache.doris.nereids.trees.expressions.functions.agg.Min; -import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; @@ -64,15 +60,15 @@ */ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMaterializedViewRule { - protected static final Map + protected static final Map AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP = new HashMap<>(); protected final String currentClassName = this.getClass().getSimpleName(); + private final Logger logger = LogManager.getLogger(this.getClass()); static { - AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put( - PlaceholderExpression.of(Count.class, 0, true), - new PlaceholderExpression(ImmutableList.of(), BitmapUnion.class, 0)); + AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new Count(true, Any.INSTANCE), + new BitmapUnion(Any.INSTANCE)); } @Override @@ -85,12 +81,12 @@ protected Plan rewriteQueryByView(MatchMode matchMode, // get view and query aggregate and top plan correspondingly Pair> viewTopPlanAndAggPair = splitToTopPlanAndAggregate(viewStructInfo); if (viewTopPlanAndAggPair == null) { - logger.info(currentClassName + "split to view to top plan and agg fail so return null"); + logger.warn(currentClassName + " split to view to top plan and agg fail so return null"); return null; } Pair> queryTopPlanAndAggPair = splitToTopPlanAndAggregate(queryStructInfo); if (queryTopPlanAndAggPair == null) { - logger.info(currentClassName + "split to query to top plan and agg fail so return null"); + logger.warn(currentClassName + " split to query to top plan and agg fail so return null"); return null; } // Firstly, handle query group by expression rewrite @@ -119,7 +115,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode, true); if (rewrittenQueryGroupExpr.isEmpty()) { // can not rewrite, bail out. - logger.info(currentClassName + " can not rewrite expression when not need roll up"); + logger.debug(currentClassName + " can not rewrite expression when not need roll up"); return null; } return new LogicalProject<>( @@ -134,14 +130,14 @@ protected Plan rewriteQueryByView(MatchMode matchMode, viewExpr -> viewExpr.anyMatch(expr -> expr instanceof AggregateFunction && ((AggregateFunction) expr).isDistinct()))) { // if mv aggregate function contains distinct, can not roll up, bail out. - logger.info(currentClassName + " view contains distinct function so can not roll up"); + logger.debug(currentClassName + " view contains distinct function so can not roll up"); return null; } // split the query top plan expressions to group expressions and functions, if can not, bail out. Pair, Set> queryGroupAndFunctionPair = topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair); if (queryGroupAndFunctionPair == null) { - logger.info(currentClassName + " query top plan split to group by and function fail so return null"); + logger.warn(currentClassName + " query top plan split to group by and function fail so return null"); return null; } // Secondly, try to roll up the agg functions @@ -165,11 +161,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode, // try to roll up AggregateFunction queryFunction = (AggregateFunction) topExpression.firstMatch( expr -> expr instanceof AggregateFunction); - Function rollupAggregateFunction = rollup(queryFunction, - queryFunctionShuttled, mvExprToMvScanExprQueryBased); + Function rollupAggregateFunction = rollup(queryFunction, queryFunctionShuttled, + mvExprToMvScanExprQueryBased); if (rollupAggregateFunction == null) { - logger.info(currentClassName + " query function " + queryFunction.getName() - + " can not roll up so return null"); return null; } // key is query need roll up expr, value is mv scan based roll up expr @@ -181,7 +175,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode, queryToViewSlotMapping, false); if (rewrittenFunctionExpression == null) { - logger.info(currentClassName + " roll up expression can not rewrite by view so return null"); + logger.debug(currentClassName + " roll up expression can not rewrite by view so return null"); return null; } finalAggregateExpressions.add((NamedExpression) rewrittenFunctionExpression); @@ -191,7 +185,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode, ExpressionUtils.shuttleExpressionWithLineage(topExpression, queryTopPlan); if (!mvExprToMvScanExprQueryBased.containsKey(queryGroupShuttledExpr)) { // group expr can not rewrite by view - logger.info(currentClassName + logger.debug(currentClassName + " view group expressions can not contains the query group by expression so return null"); return null; } @@ -205,7 +199,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, queryToViewSlotMapping, true); if (rewrittenGroupExpression == null) { - logger.info(currentClassName + " query top expression can not be rewritten by view so return null"); + logger.debug(currentClassName + + " query top expression can not be rewritten by view so return null"); return null; } finalAggregateExpressions.add((NamedExpression) rewrittenGroupExpression); @@ -258,12 +253,15 @@ protected Plan rewriteQueryByView(MatchMode matchMode, private Function rollup(AggregateFunction queryFunction, Expression queryFunctionShuttled, Map mvExprToMvScanExprQueryBased) { + if (!(queryFunction instanceof CouldRollUp)) { + return null; + } Expression rollupParam = null; if (mvExprToMvScanExprQueryBased.containsKey(queryFunctionShuttled)) { - // function can not rewrite by view + // function can rewrite by view rollupParam = mvExprToMvScanExprQueryBased.get(queryFunctionShuttled); } else { - // try to use complex roll up param + // function can not rewrite by view, try to use complex roll up param // eg: query is count(distinct param), mv sql is bitmap_union(to_bitmap(param)) for (Expression mvExprShuttled : mvExprToMvScanExprQueryBased.keySet()) { if (!(mvExprShuttled instanceof Function)) { @@ -278,24 +276,7 @@ private Function rollup(AggregateFunction queryFunction, return null; } // do roll up - Class rollupAggregateFunction = queryFunction.getRollup(); - if (rollupAggregateFunction == null) { - return null; - } - if (Sum.class.isAssignableFrom(rollupAggregateFunction)) { - return new Sum(queryFunction.isDistinct(), rollupParam); - } - if (Max.class.isAssignableFrom(rollupAggregateFunction)) { - return new Max(queryFunction.isDistinct(), rollupParam); - } - if (Min.class.isAssignableFrom(rollupAggregateFunction)) { - return new Min(queryFunction.isDistinct(), rollupParam); - } - if (BitmapUnionCount.class.isAssignableFrom(rollupAggregateFunction)) { - return new BitmapUnionCount(rollupParam); - } - // can rollup return null - return null; + return ((CouldRollUp) queryFunction).constructRollUp(rollupParam); } private Pair, Set> topPlanSplitToGroupAndFunction( @@ -367,37 +348,21 @@ protected boolean checkPattern(StructInfo structInfo) { } private boolean isAggregateFunctionEquivalent(Function queryFunction, Function viewFunction) { - Class queryClazz = queryFunction.getClass(); - Class viewClazz = viewFunction.getClass(); - if (queryClazz.isAssignableFrom(viewClazz)) { + if (queryFunction.equals(viewFunction)) { return true; } - // bitmap roll up - boolean isDistinct = queryFunction instanceof AggregateFunction - && ((AggregateFunction) queryFunction).isDistinct(); - PlaceholderExpression equivalentFunction = AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.get( - PlaceholderExpression.of(queryFunction.getClass(), 0, isDistinct)); + // get query equivalent function + Expression equivalentFunction = null; + for (Map.Entry entry : AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.entrySet()) { + if (entry.getKey().equals(queryFunction)) { + equivalentFunction = entry.getValue(); + } + } // check is have equivalent function or not if (equivalentFunction == null) { return false; } // current compare - if (!viewFunction.getClass().isAssignableFrom(equivalentFunction.getDelegateClazz())) { - return false; - } - if (!viewFunction.children().isEmpty() && !equivalentFunction.children().isEmpty()) { - // children compare, just compare two level, support more later - List equivalentFunctions = equivalentFunction.children(); - if (viewFunction.children().size() != equivalentFunctions.size()) { - return false; - } - for (int i = 0; i < viewFunction.children().size(); i++) { - if (!viewFunction.child(i).getClass().equals( - ((PlaceholderExpression) equivalentFunctions.get(i)).getDelegateClazz())) { - return false; - } - } - } - return true; + return equivalentFunction.equals(viewFunction); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 19b05b71e19f85..7825467628a2f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -60,7 +60,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode, // Can not rewrite, bail out if (expressionsRewritten.isEmpty() || expressionsRewritten.stream().anyMatch(expr -> !(expr instanceof NamedExpression))) { - logger.info(currentClassName + " expression to rewrite is not named expr so return null"); + logger.warn(currentClassName + " expression to rewrite is not named expr so return null"); return null; } // record the group id in materializationContext, and when rewrite again in diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index c4ea2456dad95c..6ef144c286c05b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -31,6 +31,7 @@ import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.executor.Rewriter; +import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; @@ -69,7 +70,7 @@ /** * The abstract class for all materialized view rules */ -public abstract class AbstractMaterializedViewRule { +public abstract class AbstractMaterializedViewRule implements ExplorationRuleFactory { public static final HashSet SUPPORTED_JOIN_TYPE_SET = Sets.newHashSet(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN); protected final String currentClassName = this.getClass().getSimpleName(); @@ -103,7 +104,7 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { logger.info(currentClassName + " this group is already rewritten so skip"); continue; } - MTMV mtmv = materializationContext.getMtmv(); + MTMV mtmv = materializationContext.getMTMV(); MTMVCache mtmvCache = getCacheFromMTMV(mtmv, cascadesContext); if (mtmvCache == null) { logger.info(currentClassName + " mv cache is null so return"); @@ -204,12 +205,18 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { return rewriteResults; } + /** + * Partition will be pruned in query then add the record the partitions to select partitions on + * catalog relation. + * Maybe only just some partitions is valid in materialized view, so we should check if the mv can + * offer the partitions which query used or not. + */ protected boolean checkPartitionIsValid( StructInfo queryInfo, MaterializationContext materializationContext, CascadesContext cascadesContext) { // check partition is valid or not - MTMV mtmv = materializationContext.getMtmv(); + MTMV mtmv = materializationContext.getMTMV(); PartitionInfo mvPartitionInfo = mtmv.getPartitionInfo(); if (PartitionType.UNPARTITIONED.equals(mvPartitionInfo.getType())) { // if not partition, if rewrite success, it means mv is available @@ -222,9 +229,9 @@ protected boolean checkPartitionIsValid( return true; } Optional relatedTableRelation = queryInfo.getRelations().stream() - .filter(relation -> relatedPartitionTable.equals(new BaseTableInfo(relation.getTable())) - && relation instanceof LogicalOlapScan) - .map(relation -> (LogicalOlapScan) relation) + .filter(LogicalOlapScan.class::isInstance) + .filter(relation -> relatedPartitionTable.equals(new BaseTableInfo(relation.getTable()))) + .map(LogicalOlapScan.class::cast) .findFirst(); if (!relatedTableRelation.isPresent()) { logger.warn("mv is partition update, but related table relation is null"); @@ -235,7 +242,7 @@ protected boolean checkPartitionIsValid( try { mvToBasePartitionMap = MTMVUtil.getMvToBasePartitions(mtmv, relatedTable); } catch (AnalysisException e) { - logger.error("mvRewriteSuccess getMvToBasePartitions fail", e); + logger.warn("mvRewriteSuccess getMvToBasePartitions fail", e); return false; } // get mv valid partitions @@ -265,7 +272,8 @@ protected boolean checkPartitionIsValid( if (relatedTableSelectedPartitionToCheck.isEmpty()) { relatedTableSelectedPartitionToCheck.addAll(relatedTable.getPartitionIds()); } - return relatedTalbeValidSet.containsAll(relatedTableSelectedPartitionToCheck); + return !relatedTalbeValidSet.isEmpty() + && relatedTalbeValidSet.containsAll(relatedTableSelectedPartitionToCheck); } private MTMVCache getCacheFromMTMV(MTMV mtmv, CascadesContext cascadesContext) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index dedc775a3ebf95..f61de5cc2b5a99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -95,7 +95,7 @@ public void addMatchedGroup(GroupId groupId) { matchedGroups.add(groupId); } - public MTMV getMtmv() { + public MTMV getMTMV() { return mtmv; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java index 3bd0b1080cc9fe..6a2aa0b7ef2b8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; @@ -31,7 +30,7 @@ /** * This is responsible for aggregate rewriting according to different pattern * */ -public class MaterializedViewAggregateRule extends AbstractMaterializedViewAggregateRule implements RewriteRuleFactory { +public class MaterializedViewAggregateRule extends AbstractMaterializedViewAggregateRule { public static final MaterializedViewAggregateRule INSTANCE = new MaterializedViewAggregateRule(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java index 406afdc48fe77b..ee69ee16b1ee30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; @@ -32,7 +31,7 @@ /** * This is responsible for join pattern such as filter on join */ -public class MaterializedViewFilterJoinRule extends AbstractMaterializedViewJoinRule implements RewriteRuleFactory { +public class MaterializedViewFilterJoinRule extends AbstractMaterializedViewJoinRule { public static final MaterializedViewFilterJoinRule INSTANCE = new MaterializedViewFilterJoinRule(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java index c7e869e04db61c..a253d010a230d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; @@ -33,8 +32,7 @@ /** * This is responsible for join pattern such as filter on project on join */ -public class MaterializedViewFilterProjectJoinRule extends AbstractMaterializedViewJoinRule - implements RewriteRuleFactory { +public class MaterializedViewFilterProjectJoinRule extends AbstractMaterializedViewJoinRule { public static final MaterializedViewFilterProjectJoinRule INSTANCE = new MaterializedViewFilterProjectJoinRule(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java index 16a66e959e485c..b2b313b8470fe2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; @@ -31,7 +30,7 @@ /** * This is responsible for join pattern such as only join */ -public class MaterializedViewOnlyJoinRule extends AbstractMaterializedViewJoinRule implements RewriteRuleFactory { +public class MaterializedViewOnlyJoinRule extends AbstractMaterializedViewJoinRule { public static final MaterializedViewOnlyJoinRule INSTANCE = new MaterializedViewOnlyJoinRule(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java index e9a31f45535c84..c73e3e0b039e82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -30,8 +29,7 @@ import java.util.List; /**MaterializedViewProjectAggregateRule*/ -public class MaterializedViewProjectAggregateRule extends AbstractMaterializedViewAggregateRule implements - RewriteRuleFactory { +public class MaterializedViewProjectAggregateRule extends AbstractMaterializedViewAggregateRule { public static final MaterializedViewProjectAggregateRule INSTANCE = new MaterializedViewProjectAggregateRule(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java index e624b7f6a008f7..41ae132db9462f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; @@ -33,8 +32,7 @@ /** * This is responsible for join pattern such as project on filter on join */ -public class MaterializedViewProjectFilterJoinRule extends AbstractMaterializedViewJoinRule - implements RewriteRuleFactory { +public class MaterializedViewProjectFilterJoinRule extends AbstractMaterializedViewJoinRule { public static final MaterializedViewProjectFilterJoinRule INSTANCE = new MaterializedViewProjectFilterJoinRule(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java index 3a81b9ae626236..bfda1e578055b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -32,7 +31,7 @@ /** * This is responsible for join pattern such as project on join * */ -public class MaterializedViewProjectJoinRule extends AbstractMaterializedViewJoinRule implements RewriteRuleFactory { +public class MaterializedViewProjectJoinRule extends AbstractMaterializedViewJoinRule { public static final MaterializedViewProjectJoinRule INSTANCE = new MaterializedViewProjectJoinRule(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java index c5909822adbb18..c9624c176b9fd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java @@ -18,14 +18,13 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import java.util.List; /** * This is responsible for single table rewriting according to different pattern * */ -public class MaterializedViewScanRule extends AbstractMaterializedViewRule implements RewriteRuleFactory { +public class MaterializedViewScanRule extends AbstractMaterializedViewRule { @Override public List buildRules() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Any.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Any.java new file mode 100644 index 00000000000000..3ee74d417a67e6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Any.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions; + +import org.apache.doris.nereids.trees.TreeNode; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * This represents any expression, it means it equals any expression + */ +public class Any extends Expression { + + public static final Any INSTANCE = new Any(ImmutableList.of()); + + private Any(Expression... children) { + super(children); + } + + private Any(List children) { + super(children); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return null; + } + + @Override + public boolean nullable() { + return false; + } + + @Override + public boolean equals(Object o) { + return true; + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean deepEquals(TreeNode that) { + return true; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/CouldRollUp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/CouldRollUp.java new file mode 100644 index 00000000000000..9e118ba49761a2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/CouldRollUp.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions; + +import org.apache.doris.nereids.trees.expressions.Expression; + +/** + * Could roll up trait, if a function could roll up in aggregate, it will implement the interface + */ +public interface CouldRollUp { + + /** + * construct the roll up function with custom param + */ + Function constructRollUp(Expression param, Expression... varParams); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java index ce80dec27415fe..a7e523dfdb549e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; -import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.typecoercion.ExpectsInputTypes; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; @@ -78,10 +77,6 @@ public boolean isDistinct() { return distinct; } - public Class getRollup() { - return null; - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java index e1fc003249a9be..c051c887917c01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; +import org.apache.doris.nereids.trees.expressions.functions.CouldRollUp; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.window.SupportWindowAnalytic; @@ -37,7 +38,7 @@ /** count agg function. */ public class Count extends AggregateFunction - implements ExplicitlyCastableSignature, AlwaysNotNullable, SupportWindowAnalytic { + implements ExplicitlyCastableSignature, AlwaysNotNullable, SupportWindowAnalytic, CouldRollUp { public static final List SIGNATURES = ImmutableList.of( // count(*) @@ -145,11 +146,11 @@ public List getSignatures() { } @Override - public Class getRollup() { + public Function constructRollUp(Expression param, Expression... varParams) { if (this.isDistinct()) { - return BitmapUnionCount.class; + return new BitmapUnionCount(param); } else { - return Sum.class; + return new Sum(param); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java index b32345b46518a1..8ae719e9477c9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.CouldRollUp; import org.apache.doris.nereids.trees.expressions.functions.CustomSignature; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.window.SupportWindowAnalytic; @@ -35,7 +36,7 @@ /** max agg function. */ public class Max extends NullableAggregateFunction - implements UnaryExpression, CustomSignature, SupportWindowAnalytic { + implements UnaryExpression, CustomSignature, SupportWindowAnalytic, CouldRollUp { public Max(Expression child) { this(false, false, child); } @@ -83,7 +84,7 @@ public R accept(ExpressionVisitor visitor, C context) { } @Override - public Class getRollup() { - return Max.class; + public Function constructRollUp(Expression param, Expression... varParams) { + return new Max(this.distinct, this.alwaysNullable, param); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java index 097b1246d1fd0f..94ff00fb255f1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.CouldRollUp; import org.apache.doris.nereids.trees.expressions.functions.CustomSignature; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.window.SupportWindowAnalytic; @@ -35,7 +36,7 @@ /** min agg function. */ public class Min extends NullableAggregateFunction - implements UnaryExpression, CustomSignature, SupportWindowAnalytic { + implements UnaryExpression, CustomSignature, SupportWindowAnalytic, CouldRollUp { public Min(Expression child) { this(false, false, child); @@ -84,7 +85,7 @@ public R accept(ExpressionVisitor visitor, C context) { } @Override - public Class getRollup() { - return Min.class; + public Function constructRollUp(Expression param, Expression... varParams) { + return new Min(this.distinct, this.alwaysNullable, param); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java index b99f836e09e8b2..6a18dd165c1dea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java @@ -21,7 +21,9 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ComputePrecisionForSum; +import org.apache.doris.nereids.trees.expressions.functions.CouldRollUp; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.window.SupportWindowAnalytic; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -45,7 +47,8 @@ * AggregateFunction 'sum'. This class is generated by GenerateFunction. */ public class Sum extends NullableAggregateFunction - implements UnaryExpression, ExplicitlyCastableSignature, ComputePrecisionForSum, SupportWindowAnalytic { + implements UnaryExpression, ExplicitlyCastableSignature, ComputePrecisionForSum, SupportWindowAnalytic, + CouldRollUp { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).args(BooleanType.INSTANCE), @@ -111,7 +114,7 @@ public List getSignatures() { } @Override - public Class getRollup() { - return Sum.class; + public Function constructRollUp(Expression param, Expression... varParams) { + return new Sum(this.distinct, param); } } diff --git a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy index bc509c5f42bf81..0b2faee2c5185b 100644 --- a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy @@ -98,35 +98,41 @@ suite("partition_mv_rewrite") { """ - def mv_def_sql = "select l_shipdate, o_orderdate, l_partkey,\n" + - "l_suppkey, sum(o_totalprice) as sum_total\n" + - "from lineitem\n" + - "left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate\n" + - "group by\n" + - "l_shipdate,\n" + - "o_orderdate,\n" + - "l_partkey,\n" + - "l_suppkey;" - - def all_partition_sql = "select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total\n" + - "from lineitem\n" + - "left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate\n" + - "group by\n" + - "l_shipdate,\n" + - "o_orderdate,\n" + - "l_partkey,\n" + - "l_suppkey;" - - - def partition_sql = "select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total\n" + - "from lineitem\n" + - "left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate\n" + - "where (l_shipdate>= '2023-10-18' and l_shipdate <= '2023-10-19')\n" + - "group by\n" + - "l_shipdate,\n" + - "o_orderdate,\n" + - "l_partkey,\n" + - "l_suppkey;" + def mv_def_sql = """ + select l_shipdate, o_orderdate, l_partkey, + l_suppkey, sum(o_totalprice) as sum_total + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + + def all_partition_sql = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + + + def partition_sql = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + where (l_shipdate>= '2023-10-18' and l_shipdate <= '2023-10-19') + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ sql """DROP MATERIALIZED VIEW IF EXISTS mv_10086""" sql """DROP TABLE IF EXISTS mv_10086""" @@ -172,7 +178,7 @@ suite("partition_mv_rewrite") { REFRESH MATERIALIZED VIEW mv_10086; """ // wait partition is valid - sleep(5000) + waitingMTMVTaskFinished(job_name) explain { sql("${all_partition_sql}")