diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 4e3c2a650e520b8..c027a6db52bbdee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -25,9 +25,12 @@ import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanSplitContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -37,6 +40,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -56,7 +60,6 @@ protected Plan rewriteQueryByView(MatchMode matchMode, SlotMapping queryToViewSlotMapping, Plan tempRewritedPlan, MaterializationContext materializationContext) { - // get view and query aggregate and top plan correspondingly Pair> viewTopPlanAndAggPair = splitToTopPlanAndAggregate(viewStructInfo); if (viewTopPlanAndAggPair == null) { @@ -66,25 +69,27 @@ protected Plan rewriteQueryByView(MatchMode matchMode, if (queryTopPlanAndAggPair == null) { return null; } - // Firstly, handle query group by expression rewrite LogicalAggregate queryAggregate = queryTopPlanAndAggPair.value(); Plan queryTopPlan = queryTopPlanAndAggPair.key(); // query and view have the same dimension, try to rewrite rewrittenQueryGroupExpr LogicalAggregate viewAggregate = viewTopPlanAndAggPair.value(); + Plan viewTopPlan = viewTopPlanAndAggPair.key(); boolean needRollUp = queryAggregate.getGroupByExpressions().size() != viewAggregate.getGroupByExpressions().size(); if (queryAggregate.getGroupByExpressions().size() == viewAggregate.getGroupByExpressions().size()) { - // todo consider alias - List viewGroupByExpressionQueryBased = ExpressionUtils.replace( - viewAggregate.getGroupByExpressions(), - queryToViewSlotMapping.inverse().toSlotReferenceMap()); - needRollUp = !queryAggregate.getGroupByExpressions().equals(viewGroupByExpressionQueryBased); + List queryGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( + queryAggregate.getGroupByExpressions(), queryTopPlan); + List viewGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( + viewAggregate.getGroupByExpressions(), viewTopPlan) + .stream() + .map(expr -> ExpressionUtils.replace(expr, queryToViewSlotMapping.inverse().toSlotReferenceMap())) + .collect(Collectors.toList()); + needRollUp = !queryGroupShuttledExpression.equals(viewGroupShuttledExpression); } if (!needRollUp) { - List queryShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( - queryTopPlan.getOutput(), queryTopPlan); - List rewrittenQueryGroupExpr = rewriteExpression(queryShuttledExpressions, + List rewrittenQueryGroupExpr = rewriteExpression(queryTopPlan.getOutput(), + queryTopPlan, materializationContext.getMvExprToMvScanExprMapping(), queryToViewSlotMapping, true); @@ -97,95 +102,171 @@ protected Plan rewriteQueryByView(MatchMode matchMode, tempRewritedPlan); } // the dimension in query and view are different, try to roll up - // Split query aggregate dimension and agg function List needPullUpExpression = new ArrayList<>(); - // Firstly, find the query top output rewriteFunctionExprList which only use query aggregate function, + // Split query aggregate to group expression and agg function + // Firstly, find the query top output rewrite function expr list which only use query aggregate function, + // This will be used to roll up if (viewAggregate.getOutputExpressions().stream().anyMatch( viewExpr -> viewExpr.anyMatch(expr -> expr instanceof AggregateFunction - && ((AggregateFunction) expr).isDistinct()) - )) { - // if mv function contains distinct, can not roll up. + && ((AggregateFunction) expr).isDistinct()))) { + // if mv aggregate function contains distinct, can not roll up, bail out. return null; } - Set queryAggGroupSet = new HashSet<>(queryAggregate.getGroupByExpressions()); - List queryAggFunctions = queryAggregate.getOutputExpressions().stream() - .filter(expr -> !queryAggGroupSet.contains(expr)) - .collect(Collectors.toList()); - Set queryAggFunctionSet = new HashSet<>(queryAggFunctions); - Pair, List> queryGroupAndFunctionPair - = splitToGroupAndFunction( - queryTopPlanAndAggPair, - queryAggFunctionSet); - // filter the expression which use the child agg function in query top plan, only support to reference the - // aggregate function directly, will support expression later. - List queryTopPlanFunctionList = queryGroupAndFunctionPair.value(); - if (queryTopPlanFunctionList.stream().anyMatch( - topAggFunc -> !(topAggFunc instanceof NamedExpression) - && (!queryAggFunctionSet.contains(topAggFunc) - || !queryAggFunctionSet.contains(topAggFunc.child(0))))) { + // split the query top plan expressions to group expressions and functions, if can not, bail out. + Pair, Set> queryGroupAndFunctionPair + = topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair); + if (queryGroupAndFunctionPair == null) { return null; } - // Secondly, try to roll up the agg functions and add aggregate - Multimap needRollupFunctionExprMap = HashMultimap.create(); + // Secondly, try to roll up the agg functions + // this map will be used to rewrite expression + Multimap needRollupExprMap = HashMultimap.create(); + Multimap groupRewrittenExprMap = HashMultimap.create(); Map mvExprToMvScanExprQueryBased = materializationContext.getMvExprToMvScanExprMapping().keyPermute( queryToViewSlotMapping.inverse()).flattenMap().get(0); - for (Expression needRollUpExpr : queryTopPlanFunctionList) { - Expression needRollupShuttledExpr = ExpressionUtils.shuttleExpressionWithLineage(needRollUpExpr, - queryTopPlan); - if (!mvExprToMvScanExprQueryBased.containsKey(needRollupShuttledExpr)) { - // function can not rewrite by view - return null; - } - AggregateFunction aggregateFunction = (AggregateFunction) needRollUpExpr.firstMatch( - expr -> expr instanceof AggregateFunction); - AggregateFunction rollup = aggregateFunction.getRollup(); - if (rollup == null) { - return null; + + Set queryTopPlanFunctionSet = queryGroupAndFunctionPair.value(); + // try to rewrite, contains both roll up aggregate functions and aggregate group expression + List finalAggregateExpressions = new ArrayList<>(); + List finalGroupExpressions = new ArrayList<>(); + for (Expression topExpression : queryTopPlan.getExpressions()) { + // is agg function, try to roll up and rewrite + if (queryTopPlanFunctionSet.contains(topExpression)) { + Expression needRollupShuttledExpr = ExpressionUtils.shuttleExpressionWithLineage( + topExpression, + queryTopPlan); + if (!mvExprToMvScanExprQueryBased.containsKey(needRollupShuttledExpr)) { + // function can not rewrite by view + return null; + } + // try to roll up + AggregateFunction needRollupAggFunction = (AggregateFunction) topExpression.firstMatch( + expr -> expr instanceof AggregateFunction); + AggregateFunction rollupAggregateFunction = rollup(needRollupAggFunction, + mvExprToMvScanExprQueryBased.get(needRollupShuttledExpr)); + if (rollupAggregateFunction == null) { + return null; + } + // key is query need roll up expr, value is mv scan based roll up expr + needRollupExprMap.put(needRollupShuttledExpr, rollupAggregateFunction); + // rewrite query function expression by mv expression + Expression rewrittenFunctionExpression = rewriteExpression(topExpression, + queryTopPlan, + new ExpressionMapping(needRollupExprMap), + queryToViewSlotMapping, + false); + if (rewrittenFunctionExpression == null) { + return null; + } + finalAggregateExpressions.add((NamedExpression) rewrittenFunctionExpression); + } else { + // try to rewrite group expression + Expression queryGroupShuttledExpr = + ExpressionUtils.shuttleExpressionWithLineage(topExpression, queryTopPlan); + if (!mvExprToMvScanExprQueryBased.containsKey(queryGroupShuttledExpr)) { + // group expr can not rewrite by view + return null; + } + groupRewrittenExprMap.put(queryGroupShuttledExpr, + mvExprToMvScanExprQueryBased.get(queryGroupShuttledExpr)); + // rewrite query group expression by mv expression + Expression rewrittenGroupExpression = rewriteExpression( + topExpression, + queryTopPlan, + new ExpressionMapping(groupRewrittenExprMap), + queryToViewSlotMapping, + true); + if (rewrittenGroupExpression == null) { + return null; + } + finalAggregateExpressions.add((NamedExpression) rewrittenGroupExpression); + finalGroupExpressions.add(rewrittenGroupExpression); } - // key is query need roll up expr, value is mv scan based roll up expr - needRollupFunctionExprMap.put(needRollUpExpr, - rollup.withChildren(mvExprToMvScanExprQueryBased.get(needRollupShuttledExpr))); } - // query group rewrite - Multimap groupRewrittenExprMap = HashMultimap.create(); - List queryTopPlanGroupExprList = queryGroupAndFunctionPair.key(); - for (Expression needRewriteGroupExpr : queryTopPlanGroupExprList) { - Expression queryGroupShuttledExpr = - ExpressionUtils.shuttleExpressionWithLineage(needRewriteGroupExpr, queryTopPlan); - if (!mvExprToMvScanExprQueryBased.containsKey(queryGroupShuttledExpr)) { - // group expr can not rewrite by view - return null; - } - groupRewrittenExprMap.put(needRewriteGroupExpr, mvExprToMvScanExprQueryBased.get(queryGroupShuttledExpr)); + // add project to guarantee group by column ref is slot reference, + // this is necessary because physical createHash will need slotReference later + List copiedFinalGroupExpressions = new ArrayList<>(finalGroupExpressions); + List projectsUnderAggregate = copiedFinalGroupExpressions.stream() + .map(NamedExpression.class::cast) + .collect(Collectors.toList()); + projectsUnderAggregate.addAll(tempRewritedPlan.getOutput()); + LogicalProject mvProject = new LogicalProject<>(projectsUnderAggregate, tempRewritedPlan); + // add agg rewrite + Map projectOutPutExprIdMap = mvProject.getOutput().stream() + .distinct() + .collect(Collectors.toMap(NamedExpression::getExprId, slot -> slot)); + // make the expressions to re reference project output + finalGroupExpressions = finalGroupExpressions.stream() + .map(expr -> { + ExprId exprId = ((NamedExpression) expr).getExprId(); + if (projectOutPutExprIdMap.containsKey(exprId)) { + return projectOutPutExprIdMap.get(exprId); + } + return (NamedExpression) expr; + }) + .collect(Collectors.toList()); + finalAggregateExpressions = finalAggregateExpressions.stream() + .map(expr -> { + ExprId exprId = expr.getExprId(); + if (projectOutPutExprIdMap.containsKey(exprId)) { + return projectOutPutExprIdMap.get(exprId); + } + return expr; + }) + .collect(Collectors.toList()); + LogicalAggregate rewrittenAggregate = new LogicalAggregate(finalGroupExpressions, + finalAggregateExpressions, mvProject); + // record the group id in materializationContext, and when rewrite again in + // the same group, bail out quickly. + if (queryStructInfo.getOriginalPlan().getGroupExpression().isPresent()) { + materializationContext.addMatchedGroup( + queryStructInfo.getOriginalPlan().getGroupExpression().get().getOwnerGroup().getGroupId()); } - // rewrite expression for group and function expr - List rewriteFunctionExprList = rewriteExpression(queryTopPlanFunctionList, - new ExpressionMapping(needRollupFunctionExprMap), - queryToViewSlotMapping, - true); - if (rewriteFunctionExprList == null) { + return rewrittenAggregate; + } + + // only support sum roll up, support other agg functions later. + private AggregateFunction rollup(AggregateFunction originFunction, + Expression mappedExpression) { + Class rollupAggregateFunction = originFunction.getRollup(); + if (rollupAggregateFunction == null) { return null; } - List rewriteGroupExprList = rewriteExpression(queryTopPlanGroupExprList, - new ExpressionMapping(groupRewrittenExprMap), - queryToViewSlotMapping, - true); - if (rewriteGroupExprList == null) { - return null; + if (Sum.class.isAssignableFrom(rollupAggregateFunction)) { + return new Sum(originFunction.isDistinct(), mappedExpression); } - // project rewrite - return new LogicalAggregate(rewriteGroupExprList, rewriteFunctionExprList, tempRewritedPlan); + // can rollup return null + return null; } - private Pair, List> splitToGroupAndFunction( - Pair> topPlanAndAggPair, - Set queryAggGroupFunctionSet) { + private Pair, Set> topPlanSplitToGroupAndFunction( + Pair> topPlanAndAggPair) { + + LogicalAggregate queryAggregate = topPlanAndAggPair.value(); + Set queryAggGroupSet = new HashSet<>(queryAggregate.getGroupByExpressions()); + Set queryAggFunctionSet = queryAggregate.getOutputExpressions().stream() + .filter(expr -> !queryAggGroupSet.contains(expr)) + .collect(Collectors.toSet()); + Plan queryTopPlan = topPlanAndAggPair.key(); - Map> groupByAndFuncitonMap = queryTopPlan.getExpressions() - .stream() - .collect(Collectors.partitioningBy(expression -> expression.anyMatch(expr -> - expr instanceof NamedExpression && queryAggGroupFunctionSet.contains((NamedExpression) expr)))); - return Pair.of(groupByAndFuncitonMap.get(false), groupByAndFuncitonMap.get(true)); + Set topGroupByExpressions = new HashSet<>(); + Set topFunctionExpressions = new HashSet<>(); + queryTopPlan.getExpressions().forEach( + expression -> { + if (expression.anyMatch(expr -> expr instanceof NamedExpression + && queryAggFunctionSet.contains((NamedExpression) expr))) { + topFunctionExpressions.add(expression); + } else { + topGroupByExpressions.add(expression); + } + }); + // only support to reference the aggregate function directly in top, will support expression later. + if (topFunctionExpressions.stream().anyMatch( + topAggFunc -> !(topAggFunc instanceof NamedExpression) && (!queryAggFunctionSet.contains(topAggFunc) + || !queryAggFunctionSet.contains(topAggFunc.child(0))))) { + return null; + } + return Pair.of(topGroupByExpressions, topFunctionExpressions); } private Pair> splitToTopPlanAndAggregate(StructInfo structInfo) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 1660d11d19b0c75..aad1ffa529c5a1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -26,7 +26,6 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.util.ExpressionUtils; import java.util.List; import java.util.stream.Collectors; @@ -43,13 +42,10 @@ protected Plan rewriteQueryByView(MatchMode matchMode, SlotMapping queryToViewSlotMapping, Plan tempRewritedPlan, MaterializationContext materializationContext) { - - List queryShuttleExpression = ExpressionUtils.shuttleExpressionWithLineage( - queryStructInfo.getExpressions(), - queryStructInfo.getOriginalPlan()); // Rewrite top projects, represent the query projects by view List expressionsRewritten = rewriteExpression( - queryShuttleExpression, + queryStructInfo.getExpressions(), + queryStructInfo.getOriginalPlan(), materializationContext.getMvExprToMvScanExprMapping(), queryToViewSlotMapping, true diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 4d118ff6e00e779..18e11d8645964a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -127,6 +128,7 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { // Try to rewrite compensate predicates by using mv scan List rewriteCompensatePredicates = rewriteExpression( compensatePredicates.toList(), + queryPlan, materializationContext.getMvExprToMvScanExprMapping(), queryToViewSlotMapping, true); @@ -164,20 +166,23 @@ protected Plan rewriteQueryByView(MatchMode matchMode, } /** - * Use target expression to represent the source expression. - * Visit the source expression, try to replace the source expression with target expression, if found then - * replace the source expression by target expression map value. - * Note: make the target expression map key to source based according to targetNeedToQueryBased, - * if targetNeedToQueryBased is true, we should not make it source based. + * Use target expression to represent the source expression. Visit the source expression, + * try to replace the source expression with target expression in targetExpressionMapping, if found then + * replace the source expression by target expression mapping value. + * Note: make the target expression map key to source based according to targetExpressionNeedSourceBased, + * if targetExpressionNeedSourceBased is true, we should make it source based. + * the key expression in targetExpressionMapping should be shuttled. with the method + * ExpressionUtils.shuttleExpressionWithLineage. */ protected List rewriteExpression( List sourceExpressionsToWrite, + Plan sourcePlan, ExpressionMapping targetExpressionMapping, SlotMapping sourceToTargetMapping, - boolean targetNeedToQueryBased) { - // Firstly, rewrite the target plan output expression using query with inverse mapping - // then try to use the mv expression to represent the query. if any of source expressions - // can not be represented by mv, return null + boolean targetExpressionNeedSourceBased) { + // Firstly, rewrite the target expression using source with inverse mapping + // then try to use the target expression to represent the query. if any of source expressions + // can not be represented by target expressions, return null. // // example as following: // source target @@ -187,35 +192,58 @@ protected List rewriteExpression( // transform source to: // project(slot 2, 1) // target - // generate mvSql to mvScan targetExpressionMapping, and change mv sql expression to query based - ExpressionMapping expressionMappingKeySourceBased = targetNeedToQueryBased - ? targetExpressionMapping : targetExpressionMapping.keyPermute(sourceToTargetMapping.inverse()); + // generate target to target replacement expression mapping, and change target expression to source based + List sourceShuttledExpressions = + ExpressionUtils.shuttleExpressionWithLineage(sourceExpressionsToWrite, sourcePlan); + ExpressionMapping expressionMappingKeySourceBased = targetExpressionNeedSourceBased + ? targetExpressionMapping.keyPermute(sourceToTargetMapping.inverse()) : targetExpressionMapping; + // target to target replacement expression mapping, because mv is 1:1 so get first element List> flattenExpressionMap = expressionMappingKeySourceBased.flattenMap(); - // view to view scan expression is 1:1 so get first element - Map mvSqlToMvScanMappingQueryBased = flattenExpressionMap.get(0); + Map targetToTargetReplacementMapping = flattenExpressionMap.get(0); List rewrittenExpressions = new ArrayList<>(); - for (Expression expressionToRewrite : sourceExpressionsToWrite) { + for (int index = 0; index < sourceShuttledExpressions.size(); index++) { + Expression expressionToRewrite = sourceShuttledExpressions.get(index); if (expressionToRewrite instanceof Literal) { rewrittenExpressions.add(expressionToRewrite); continue; } final Set slotsToRewrite = expressionToRewrite.collectToSet(expression -> expression instanceof Slot); - boolean wiAlias = expressionToRewrite instanceof NamedExpression; Expression replacedExpression = ExpressionUtils.replace(expressionToRewrite, - mvSqlToMvScanMappingQueryBased, - wiAlias); + targetToTargetReplacementMapping); if (replacedExpression.anyMatch(slotsToRewrite::contains)) { // if contains any slot to rewrite, which means can not be rewritten by target, bail out return ImmutableList.of(); } + Expression sourceExpression = sourceExpressionsToWrite.get(index); + if (sourceExpression instanceof NamedExpression) { + NamedExpression sourceNamedExpression = (NamedExpression) sourceExpression; + replacedExpression = new Alias(sourceNamedExpression.getExprId(), replacedExpression, + sourceNamedExpression.getName()); + } rewrittenExpressions.add(replacedExpression); } return rewrittenExpressions; } + protected Expression rewriteExpression( + Expression sourceExpressionsToWrite, + Plan sourcePlan, + ExpressionMapping targetExpressionMapping, + SlotMapping sourceToTargetMapping, + boolean targetExpressionNeedSourceBased) { + List expressionToRewrite = new ArrayList<>(); + expressionToRewrite.add(sourceExpressionsToWrite); + List rewrittenExpressions = rewriteExpression(expressionToRewrite, sourcePlan, + targetExpressionMapping, sourceToTargetMapping, targetExpressionNeedSourceBased); + if (rewrittenExpressions.isEmpty()) { + return null; + } + return rewrittenExpressions.get(0); + } + /** * Compensate mv predicates by query predicates, compensate predicate result is query based. * Such as a > 5 in mv, and a > 10 in query, the compensatory predicate is a > 10. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 24dad225e999f13..8061b1835e6bd9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.PlannerHook; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.visitor.TableCollector; @@ -90,19 +91,22 @@ private void initMaterializationContext(CascadesContext cascadesContext) { .getDbOrMetaException(mvBaseTableInfo.getDbId()) .getTableOrMetaException(mvBaseTableInfo.getTableId(), TableType.MATERIALIZED_VIEW); - String qualifiedName = materializedView.getQualifiedName(); // generate outside, maybe add partition filter in the future - Plan mvScan = new LogicalOlapScan(cascadesContext.getStatementContext().getNextRelationId(), + LogicalOlapScan mvScan = new LogicalOlapScan( + cascadesContext.getStatementContext().getNextRelationId(), (OlapTable) materializedView, - ImmutableList.of(qualifiedName), - Lists.newArrayList(materializedView.getId()), + ImmutableList.of(materializedView.getQualifiedDbName()), + // this must be empty, or it will be used to sample + Lists.newArrayList(), Lists.newArrayList(), Optional.empty()); + mvScan = mvScan.withMaterializedIndexSelected(PreAggStatus.on(), materializedView.getBaseIndexId()); List mvProjects = mvScan.getOutput().stream().map(NamedExpression.class::cast) .collect(Collectors.toList()); - mvScan = new LogicalProject(mvProjects, mvScan); + // todo should force keep consistency to mv sql plan output + Plan projectScan = new LogicalProject(mvProjects, mvScan); cascadesContext.addMaterializationContext( - MaterializationContext.fromMaterializedView(materializedView, mvScan, cascadesContext)); + MaterializationContext.fromMaterializedView(materializedView, projectScan, cascadesContext)); } catch (MetaNotFoundException metaNotFoundException) { LOG.error(mvBaseTableInfo.toString() + " can not find corresponding materialized view."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 9de2e9ed16f3200..3e1fe99c9c84e00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupId; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.ExpressionUtils; @@ -32,7 +31,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; /** * Maintain the context for query rewrite by materialized view @@ -67,14 +65,11 @@ public MaterializationContext(MTMV mtmv, mvCache = MVCache.from(mtmv, cascadesContext.getConnectContext()); mtmv.setMvCache(mvCache); } - List mvOutputExpressions = mvCache.getMvOutputExpressions(); // mv output expression shuttle, this will be used to expression rewrite - mvOutputExpressions = - ExpressionUtils.shuttleExpressionWithLineage(mvOutputExpressions, mvCache.getLogicalPlan()).stream() - .map(NamedExpression.class::cast) - .collect(Collectors.toList()); this.mvExprToMvScanExprMapping = ExpressionMapping.generate( - mvOutputExpressions, + ExpressionUtils.shuttleExpressionWithLineage( + mvCache.getMvOutputExpressions(), + mvCache.getLogicalPlan()), mvScanPlan.getExpressions()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java index 787a8cb7d9556a5..e9a31f45535c845 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; @@ -37,8 +38,8 @@ public class MaterializedViewProjectAggregateRule extends AbstractMaterializedVi @Override public List buildRules() { return ImmutableList.of( - logicalAggregate(any()).thenApplyMulti(ctx -> { - LogicalAggregate root = ctx.root; + logicalProject(logicalAggregate(any())).thenApplyMulti(ctx -> { + LogicalProject> root = ctx.root; return rewrite(root, ctx.cascadesContext); }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_AGGREGATE, RulePromise.EXPLORE)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 3d2f3b10e7ef443..557ff43b51d994f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -188,15 +188,15 @@ default boolean anyMatch(Predicate> predicate) { /** * iterate top down and test predicate if any matched. Top-down traverse implicitly. * @param predicate predicate - * @return true if all predicate return true + * @return the first node which match the predicate */ default TreeNode firstMatch(Predicate> predicate) { - if (!predicate.test(this)) { + if (predicate.test(this)) { return this; } for (NODE_TYPE child : children()) { - if (!child.anyMatch(predicate)) { - return this; + if (child.anyMatch(predicate)) { + return child; } } return this; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java index 7267d457d8caa4e..61a589daba27819 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java @@ -77,7 +77,7 @@ public boolean isDistinct() { return distinct; } - public AggregateFunction getRollup() { + public Class getRollup() { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java index 976c489637631e4..b99f836e09e8b22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java @@ -111,7 +111,7 @@ public List getSignatures() { } @Override - public AggregateFunction getRollup() { - return this; + public Class getRollup() { + return Sum.class; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 98e610aab9e79a3..c51881d526d0917 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -209,7 +209,7 @@ private void getColumns(Plan plan) { colNames.add(colName); } columns.add(new ColumnDefinition( - colName, slots.get(i).getDataType(), true, + colName, slots.get(i).getDataType(), slots.get(i).nullable(), CollectionUtils.isEmpty(simpleColumnDefinitions) ? null : simpleColumnDefinitions.get(i).getComment())); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index 53877ff091e43b6..edf962838d3cfa0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -210,7 +210,6 @@ public R accept(PlanVisitor visitor, C context) { @Override public List getExpressions() { return new ImmutableList.Builder() - .addAll(groupByExpressions) .addAll(outputExpressions) .build(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java index a3908ac78329bae..770ceafdfe643a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java @@ -117,15 +117,10 @@ public ExpressionReplaceContext(List targetExpressions, this.tableIdentifiers = tableIdentifiers; // collect the named expressions used in target expression and will be replaced later this.exprIdExpressionMap = targetExpressions.stream() - .map(each -> { - // if Alias, shuttle form the child of alias to retain the alias - if (each instanceof Alias && !each.children().isEmpty()) { - return each.child(0).collectToList(NamedExpression.class::isInstance); - } - return each.collectToList(NamedExpression.class::isInstance); - }) + .map(each -> each.collectToList(NamedExpression.class::isInstance)) .flatMap(Collection::stream) .map(NamedExpression.class::cast) + .distinct() .collect(Collectors.toMap(NamedExpression::getExprId, expr -> expr)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 2f00313cde92cbb..f080eace3971a61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -315,24 +315,7 @@ public static Optional extractSlotOrCastOnSlot(Expression expr) { * */ public static Expression replace(Expression expr, Map replaceMap) { - return expr.accept(ExpressionReplacer.INSTANCE, ExpressionReplacerContext.of(replaceMap, false)); - } - - /** - * Replace expression node in the expression tree by `replaceMap` in top-down manner. - * if replaced, create alias - * For example. - * 
-     * input expression: a > 1
-     * replaceMap: a -> b + c
-     *
-     * output:
-     * ((b + c) as a) > 1
-     *
- */ - public static Expression replace(Expression expr, Map replaceMap, - boolean withAlias) { - return expr.accept(ExpressionReplacer.INSTANCE, ExpressionReplacerContext.of(replaceMap, withAlias)); + return expr.accept(ExpressionReplacer.INSTANCE, replaceMap); } /** @@ -370,54 +353,49 @@ public static List rewriteDownShortCircuit( } private static class ExpressionReplacer - extends DefaultExpressionRewriter { + extends DefaultExpressionRewriter> { public static final ExpressionReplacer INSTANCE = new ExpressionReplacer(); private ExpressionReplacer() { } @Override - public Expression visit(Expression expr, ExpressionReplacerContext replacerContext) { - Map replaceMap = replacerContext.getReplaceMap(); - boolean isContained = replaceMap.containsKey(expr); - if (!isContained) { - return super.visit(expr, replacerContext); - } - boolean withAlias = replacerContext.isWithAlias(); - if (!withAlias) { - return replaceMap.get(expr); - } else { + public Expression visit(Expression expr, Map replaceMap) { + if (replaceMap.containsKey(expr)) { Expression replacedExpression = replaceMap.get(expr); - if (replacedExpression instanceof SlotReference) { - replacedExpression = ((SlotReference) (replacedExpression)).withNullable(expr.nullable()); + if (replacedExpression instanceof SlotReference + && replacedExpression.nullable() != expr.nullable()) { + replacedExpression = ((SlotReference) replacedExpression).withNullable(expr.nullable()); } - return new Alias(((NamedExpression) expr).getExprId(), replacedExpression, - ((NamedExpression) expr).getName()); + return replacedExpression; } + return super.visit(expr, replaceMap); } } private static class ExpressionReplacerContext { private final Map replaceMap; - private final boolean withAlias; + // if the key of replaceMap is named expr and withAlias is true, we should + // add alias after replaced + private final boolean withAliasIfKeyNamed; private ExpressionReplacerContext(Map replaceMap, - boolean withAlias) { + boolean withAliasIfKeyNamed) { this.replaceMap = replaceMap; - this.withAlias = withAlias; + this.withAliasIfKeyNamed = withAliasIfKeyNamed; } public static ExpressionReplacerContext of(Map replaceMap, - boolean withAlias) { - return new ExpressionReplacerContext(replaceMap, withAlias); + boolean withAliasIfKeyNamed) { + return new ExpressionReplacerContext(replaceMap, withAliasIfKeyNamed); } public Map getReplaceMap() { return replaceMap; } - public boolean isWithAlias() { - return withAlias; + public boolean isWithAliasIfKeyNamed() { + return withAliasIfKeyNamed; } } diff --git a/regression-test/data/nereids_rules_p0/mv/aggregate_with_roll_up.out b/regression-test/data/nereids_rules_p0/mv/aggregate_with_roll_up.out new file mode 100644 index 000000000000000..79480f0d5052ee9 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/aggregate_with_roll_up.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0 -- +4 99.50 +6 109.20 + diff --git a/regression-test/data/nereids_rules_p0/mv/aggregate_without_roll_up.out b/regression-test/data/nereids_rules_p0/mv/aggregate_without_roll_up.out new file mode 100644 index 000000000000000..be2c772c6de3a46 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/aggregate_without_roll_up.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0 -- +4 1 99.50 +6 2 109.20 + diff --git a/regression-test/data/nereids_rules_p0/mv/inner_join.out b/regression-test/data/nereids_rules_p0/mv/inner_join.out index 999748537661f34..243968db59686fd 100644 --- a/regression-test/data/nereids_rules_p0/mv/inner_join.out +++ b/regression-test/data/nereids_rules_p0/mv/inner_join.out @@ -1,9 +1,21 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !query1_0 -- +4 +6 -- !query1_1 -- +4 +4 +6 +6 -- !query1_2 -- +4 +6 -- !query1_3 -- +-- !query1_4 -- +1 +2 + diff --git a/regression-test/suites/nereids_rules_p0/mv/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/aggregate_with_roll_up.groovy new file mode 100644 index 000000000000000..52c563a9efe3a7e --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/aggregate_with_roll_up.groovy @@ -0,0 +1,127 @@ +suite("aggregate_with_roll_up") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql "SET enable_materialized_view_rewrite=true" + sql "SET enable_nereids_timeout = false" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + O_ORDERKEY INTEGER NOT NULL, + O_CUSTKEY INTEGER NOT NULL, + O_ORDERSTATUS CHAR(1) NOT NULL, + O_TOTALPRICE DECIMALV3(15,2) NOT NULL, + O_ORDERDATE DATE NOT NULL, + O_ORDERPRIORITY CHAR(15) NOT NULL, + O_CLERK CHAR(15) NOT NULL, + O_SHIPPRIORITY INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(O_ORDERKEY, O_CUSTKEY) + DISTRIBUTED BY HASH(O_ORDERKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + L_ORDERKEY INTEGER NOT NULL, + L_PARTKEY INTEGER NOT NULL, + L_SUPPKEY INTEGER NOT NULL, + L_LINENUMBER INTEGER NOT NULL, + L_QUANTITY DECIMALV3(15,2) NOT NULL, + L_EXTENDEDPRICE DECIMALV3(15,2) NOT NULL, + L_DISCOUNT DECIMALV3(15,2) NOT NULL, + L_TAX DECIMALV3(15,2) NOT NULL, + L_RETURNFLAG CHAR(1) NOT NULL, + L_LINESTATUS CHAR(1) NOT NULL, + L_SHIPDATE DATE NOT NULL, + L_COMMITDATE DATE NOT NULL, + L_RECEIPTDATE DATE NOT NULL, + L_SHIPINSTRUCT CHAR(25) NOT NULL, + L_SHIPMODE CHAR(10) NOT NULL, + L_COMMENT VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER) + DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + PS_PARTKEY INTEGER NOT NULL, + PS_SUPPKEY INTEGER NOT NULL, + PS_AVAILQTY INTEGER NOT NULL, + PS_SUPPLYCOST DECIMALV3(15,2) NOT NULL, + PS_COMMENT VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(PS_PARTKEY, PS_SUPPKEY) + DISTRIBUTED BY HASH(PS_PARTKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-11', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');""" + + sql """ + insert into orders values (1, 1, 'ok', 99.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 2, 'ok', 109.2, '2023-12-09', 'c','d',2, 'mm'); + """ + + sql """ + insert into partsupp values (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + def check_rewrite = { mv_sql, query_sql, mv_name -> + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + + def job_name = getJobName(db, mv_name); + waitingMTMVTaskFinished(job_name) + explain { + sql("${query_sql}") + contains "(${mv_name})" + } + } + + // select + from + inner join + group by + def mv1_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY, sum(O_TOTALPRICE) as sum_alias " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + + "group by lineitem.L_LINENUMBER, orders.O_CUSTKEY " + def query1_0 = "select lineitem.L_LINENUMBER, sum(O_TOTALPRICE) as sum_alias " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + + "group by lineitem.L_LINENUMBER" + // query + check_rewrite(mv1_0, query1_0, "mv1_0") + order_qt_query1_0 "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" +} + diff --git a/regression-test/suites/nereids_rules_p0/mv/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/aggregate_without_roll_up.groovy new file mode 100644 index 000000000000000..d11f94e26699e37 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/aggregate_without_roll_up.groovy @@ -0,0 +1,126 @@ +suite("aggregate_without_roll_up") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql "SET enable_materialized_view_rewrite=true" + sql "SET enable_nereids_timeout = false" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + O_ORDERKEY INTEGER NOT NULL, + O_CUSTKEY INTEGER NOT NULL, + O_ORDERSTATUS CHAR(1) NOT NULL, + O_TOTALPRICE DECIMALV3(15,2) NOT NULL, + O_ORDERDATE DATE NOT NULL, + O_ORDERPRIORITY CHAR(15) NOT NULL, + O_CLERK CHAR(15) NOT NULL, + O_SHIPPRIORITY INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(O_ORDERKEY, O_CUSTKEY) + DISTRIBUTED BY HASH(O_ORDERKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + L_ORDERKEY INTEGER NOT NULL, + L_PARTKEY INTEGER NOT NULL, + L_SUPPKEY INTEGER NOT NULL, + L_LINENUMBER INTEGER NOT NULL, + L_QUANTITY DECIMALV3(15,2) NOT NULL, + L_EXTENDEDPRICE DECIMALV3(15,2) NOT NULL, + L_DISCOUNT DECIMALV3(15,2) NOT NULL, + L_TAX DECIMALV3(15,2) NOT NULL, + L_RETURNFLAG CHAR(1) NOT NULL, + L_LINESTATUS CHAR(1) NOT NULL, + L_SHIPDATE DATE NOT NULL, + L_COMMITDATE DATE NOT NULL, + L_RECEIPTDATE DATE NOT NULL, + L_SHIPINSTRUCT CHAR(25) NOT NULL, + L_SHIPMODE CHAR(10) NOT NULL, + L_COMMENT VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER) + DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + PS_PARTKEY INTEGER NOT NULL, + PS_SUPPKEY INTEGER NOT NULL, + PS_AVAILQTY INTEGER NOT NULL, + PS_SUPPLYCOST DECIMALV3(15,2) NOT NULL, + PS_COMMENT VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(PS_PARTKEY, PS_SUPPKEY) + DISTRIBUTED BY HASH(PS_PARTKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-11', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');""" + + sql """ + insert into orders values (1, 1, 'ok', 99.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 2, 'ok', 109.2, '2023-12-09', 'c','d',2, 'mm'); + """ + + sql """ + insert into partsupp values (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + def check_rewrite = { mv_sql, query_sql, mv_name -> + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + + def job_name = getJobName(db, mv_name); + waitingMTMVTaskFinished(job_name) + explain { + sql("${query_sql}") + contains "(${mv_name})" + } + } + + // select + from + inner join + group by + def mv1_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY, sum(O_TOTALPRICE) as sum_alias " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + + "group by lineitem.L_LINENUMBER, orders.O_CUSTKEY " + def query1_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY, sum(O_TOTALPRICE) as sum_alias " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + + "group by lineitem.L_LINENUMBER, orders.O_CUSTKEY " + check_rewrite(mv1_0, query1_0, "mv1_0") + order_qt_query1_0 "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" +} + diff --git a/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy index 9d0bc62f05c22e2..71d2a4eb181cc6d 100644 --- a/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy @@ -95,6 +95,19 @@ suite("inner_join") { ) """ + sql """ insert into lineitem values (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-11', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');""" + + sql """ + insert into orders values (1, 1, 'ok', 99.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 2, 'ok', 109.2, '2023-12-09', 'c','d',2, 'mm'); + """ + + sql """ + insert into partsupp values (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + def check_rewrite = { mv_sql, query_sql, mv_name -> sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" @@ -162,4 +175,15 @@ suite("inner_join") { check_rewrite(mv1_3, query1_3, "mv1_3") order_qt_query1_3 "${query1_3}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3""" + + // select with complex expression + from + inner join + def mv1_4 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + + "from orders " + + "inner join lineitem on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + def query1_4 = "select IFNULL(orders.O_CUSTKEY, 0) as custkey_not_null " + + "from orders " + + "inner join lineitem on orders.O_ORDERKEY = lineitem.L_ORDERKEY" + check_rewrite(mv1_4, query1_4, "mv1_4") + order_qt_query1_4 "${query1_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_4""" }