diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 5a854f085b2f773..0b05f199b7e6684 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -126,7 +126,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, queryTopPlan, materializationContext.getShuttledExprToScanExprMapping(), viewToQuerySlotMapping, - queryStructInfo.getTableBitSet()); + queryStructInfo.getTableBitSet(), + cascadesContext); boolean isRewrittenQueryExpressionValid = true; if (!rewrittenQueryExpressions.isEmpty()) { List projects = new ArrayList<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 28ea4dc2c7e16f4..5b3fb17fc4225de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -49,7 +49,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, queryStructInfo.getTopPlan(), materializationContext.getShuttledExprToScanExprMapping(), targetToSourceMapping, - queryStructInfo.getTableBitSet() + queryStructInfo.getTableBitSet(), + cascadesContext ); // Can not rewrite, bail out if (expressionsRewritten.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 8e9ef1eaa97b7a9..3b9bb0078732b27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -37,6 +37,8 @@ import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.rules.expression.ExpressionOptimization; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; @@ -45,6 +47,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable; import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; @@ -242,7 +245,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca // Try to rewrite compensate predicates by using mv scan List rewriteCompensatePredicates = rewriteExpression(compensatePredicates.toList(), queryPlan, materializationContext.getShuttledExprToScanExprMapping(), - viewToQuerySlotMapping, queryStructInfo.getTableBitSet()); + viewToQuerySlotMapping, queryStructInfo.getTableBitSet(), cascadesContext); if (rewriteCompensatePredicates.isEmpty()) { materializationContext.recordFailReason(queryStructInfo, "Rewrite compensate predicate by view fail", @@ -563,7 +566,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf * then use the corresponding value of mapping to replace it */ protected List rewriteExpression(List sourceExpressionsToWrite, Plan sourcePlan, - ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, BitSet sourcePlanBitSet) { + ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, BitSet sourcePlanBitSet, + CascadesContext cascadesContext) { // Firstly, rewrite the target expression using source with inverse mapping // then try to use the target expression to represent the query. if any of source expressions // can not be represented by target expressions, return null. @@ -582,18 +586,47 @@ protected List rewriteExpression(List sourceEx rewrittenExpressions.add(expressionShuttledToRewrite); continue; } - final Set slotsToRewrite = + final Set slotsToRewrite = expressionShuttledToRewrite.collectToSet(expression -> expression instanceof Slot); final Set variants = expressionShuttledToRewrite.collectToSet(expression -> expression instanceof SlotReference - && ((SlotReference) expression).getDataType() instanceof VariantType); + && ((SlotReference) expression).getDataType() instanceof VariantType); extendMappingByVariant(variants, targetToTargetReplacementMappingQueryBased); Expression replacedExpression = ExpressionUtils.replace(expressionShuttledToRewrite, targetToTargetReplacementMappingQueryBased); - if (replacedExpression.anyMatch(slotsToRewrite::contains)) { - // if contains any slot to rewrite, which means can not be rewritten by target, bail out - return ImmutableList.of(); + Set replacedExpressionSlotQueryUsed = replacedExpression.collect(slotsToRewrite::contains); + if (!replacedExpressionSlotQueryUsed.isEmpty()) { + // if contains any slot to rewrite, which means can not be rewritten by target, + // shuttled query expr is slot#0 > '2024-01-01' but mv plan output is date_trunc(slot#0, 'day') + // which would try to rewrite + // slotDateTruncMap is {date_trunc(slot#0, 'day') : mv_scan_date_trunc_slot#10} + Map slotDateTruncMap = new HashMap<>(); + targetToTargetReplacementMappingQueryBased.keySet().forEach(expr -> { + if (expr instanceof DateTrunc && expr.child(0) instanceof Slot) { + slotDateTruncMap.put((Slot) expr.child(0), (DateTrunc) expr); + } + }); + + if (slotDateTruncMap.isEmpty()) { + //mv date_trunc slot can not offer slot for query, + // can not try to rewrite by date_trunc, bail out + return ImmutableList.of(); + } + Expression replacedWithDateTrunc = + ExpressionUtils.replace(expressionShuttledToRewrite, slotDateTruncMap); + // check date_trunc(slot#0, 'day') > '2024-01-01' can simplify + replacedWithDateTrunc = new ExpressionOptimization().rewrite(replacedWithDateTrunc, + new ExpressionRewriteContext(cascadesContext)); + if (replacedWithDateTrunc.equals(expressionShuttledToRewrite)) { + // after date_trunc simplify if equals to original expression, could rewritten by mv + replacedExpression = ExpressionUtils.replace(expressionShuttledToRewrite, + targetToTargetReplacementMappingQueryBased, + slotDateTruncMap); + } + if (replacedExpression.anyMatch(slotsToRewrite::contains)) { + return ImmutableList.of(); + } } rewrittenExpressions.add(replacedExpression); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java index 7cd49c94b09ed6d..c149f47360401c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java @@ -50,7 +50,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode, queryStructInfo.getTopPlan(), materializationContext.getShuttledExprToScanExprMapping(), targetToSourceMapping, - queryStructInfo.getTableBitSet() + queryStructInfo.getTableBitSet(), + cascadesContext ); // Can not rewrite, bail out if (expressionsRewritten.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java index 5d45b2430c42b0f..639e416f99fd320 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.rules.expression.rules.NullSafeEqualToEqual; import org.apache.doris.nereids.rules.expression.rules.OrToIn; import org.apache.doris.nereids.rules.expression.rules.SimplifyComparisonPredicate; +import org.apache.doris.nereids.rules.expression.rules.SimplifyDateTrunc; import org.apache.doris.nereids.rules.expression.rules.SimplifyDecimalV3Comparison; import org.apache.doris.nereids.rules.expression.rules.SimplifyInPredicate; import org.apache.doris.nereids.rules.expression.rules.SimplifyRange; @@ -42,6 +43,7 @@ public class ExpressionOptimization extends ExpressionRewrite { public static final List OPTIMIZE_REWRITE_RULES = ImmutableList.of( bottomUp( + SimplifyDateTrunc.INSTANCE, ExtractCommonFactorRule.INSTANCE, DistinctPredicatesRule.INSTANCE, SimplifyComparisonPredicate.INSTANCE, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyDateTrunc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyDateTrunc.java new file mode 100644 index 000000000000000..f65ec84e8cb2b8b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyDateTrunc.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression.rules; + +import org.apache.doris.nereids.rules.expression.AbstractExpressionRewriteRule; +import org.apache.doris.nereids.rules.expression.ExpressionMatchingContext; +import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher; +import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ImmutableList; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * date_trunc(slot#0, 'day') > '2024-01-01' can be simplify to slot#0 > '2024-01-01' + * */ +public class SimplifyDateTrunc extends AbstractExpressionRewriteRule implements ExpressionPatternRuleFactory { + + public static SimplifyDateTrunc INSTANCE = new SimplifyDateTrunc(); + + @Override + public List> buildRules() { + return ImmutableList.of( + matchesType(ComparisonPredicate.class) + .when(predicate -> + (predicate.getArgument(0) instanceof DateTrunc + && predicate.getArgument(1) instanceof Literal) + || (predicate.getArgument(1) instanceof DateTrunc + && predicate.getArgument(0) instanceof Literal) + ) + .thenApply(SimplifyDateTrunc::rewriteWhenComparison) + ); + } + + private static Expression rewriteWhenComparison(ExpressionMatchingContext context) { + + Expression predicate = context.expr; + DateTrunc dateTrunc = predicate.getArgument(0) instanceof DateTrunc + ? (DateTrunc) predicate.getArgument(0) : (DateTrunc) predicate.getArgument(1); + Literal literal = predicate.getArgument(0) instanceof Literal + ? (Literal) predicate.getArgument(0) : (Literal) predicate.getArgument(1); + Expression argument = dateTrunc.getArgument(0); + if (!(argument instanceof Slot)) { + return predicate; + } + Map exprMapping = new HashMap<>(); + exprMapping.put(argument, literal); + Expression replacedExpr = ExpressionUtils.replace(argument, exprMapping); + Expression evaluatedExpr = FoldConstantRuleOnFE.evaluate(replacedExpr, + new ExpressionRewriteContext(context.cascadesContext)); + if (evaluatedExpr.equals(literal)) { + return predicate.withChildren(ImmutableList.of(argument, literal)); + } + return predicate; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 53ab8a50683ecbd..e6070007962ff91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -403,6 +403,29 @@ public static Expression replace(Expression expr, Map + * input expression: a > 1 + * replaceMap: a -> b + c + * + * output: + * b + c > 1 + * + */ + public static Expression replace(Expression expr, Map replaceMap, + Map transferMap) { + return expr.rewriteDownShortCircuit(e -> { + Expression replacedExpr = replaceMap.get(e); + if (replacedExpr != null) { + return replacedExpr; + } + replacedExpr = replaceMap.get(transferMap.get(e)); + return replacedExpr == null ? e : replacedExpr; + }); + } + public static List replace(List exprs, Map replaceMap) { ImmutableList.Builder result = ImmutableList.builderWithExpectedSize(exprs.size());