Skip to content

Commit

Permalink
[opt](mtmv) Support rewrite when mv has date_trunc but query not
Browse files Browse the repository at this point in the history
  • Loading branch information
seawinde committed Dec 10, 2024
1 parent e52e561 commit e095920
Show file tree
Hide file tree
Showing 5 changed files with 822 additions and 71 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.plans.JoinType;
Expand Down Expand Up @@ -578,6 +579,14 @@ protected List<Expression> rewriteExpression(List<? extends Expression> sourceEx
Map<Expression, Expression> targetToTargetReplacementMappingQueryBased =
flattenExpressionMap.get(0);

// viewExprParamToDateTruncMap is {slot#0 : date_trunc(slot#0, 'day')}
Map<Expression, DateTrunc> viewExprParamToDateTruncMap = new HashMap<>();
targetToTargetReplacementMappingQueryBased.keySet().forEach(expr -> {
if (expr instanceof DateTrunc) {
viewExprParamToDateTruncMap.put(expr.child(0), (DateTrunc) expr);
}
});

List<Expression> rewrittenExpressions = new ArrayList<>();
for (int exprIndex = 0; exprIndex < sourceShuttledExpressions.size(); exprIndex++) {
Expression expressionShuttledToRewrite = sourceShuttledExpressions.get(exprIndex);
Expand All @@ -599,37 +608,36 @@ protected List<Expression> rewriteExpression(List<? extends Expression> sourceEx
// if contains any slot to rewrite, which means can not be rewritten by target,
// expressionShuttledToRewrite is slot#0 > '2024-01-01' but mv plan output is date_trunc(slot#0, 'day')
// which would try to rewrite
// viewExpressionParamToDateTruncMap is {slot#0 : date_trunc(slot#0, 'day')}
Map<Expression, DateTrunc> viewExpressionParamToDateTruncMap = new HashMap<>();
targetToTargetReplacementMappingQueryBased.keySet().forEach(expr -> {
if (expr instanceof DateTrunc) {
viewExpressionParamToDateTruncMap.put(expr.child(0), (DateTrunc) expr);
}
});
Expression queryUsedExpr = expressionShuttledToRewrite.child(0);
Expression queryShuttledExprParam = expressionShuttledToRewrite.child(0);

if (!queryExprToInfoMap.containsKey(sourceExpressionsToWrite.get(exprIndex))
|| !viewExpressionParamToDateTruncMap.containsKey(queryUsedExpr)) {
// mv date_trunc expression can not offer expression for query,
// can not try to rewrite by date_trunc, bail out
Expression queryOriginalExpr = sourceExpressionsToWrite.get(exprIndex);
if (!queryExprToInfoMap.containsKey(queryOriginalExpr)
|| !viewExprParamToDateTruncMap.containsKey(queryShuttledExprParam)) {
// query expr contains expression info or mv out contains date_trunc expression,
// if not, can not try to rewritten by view date_trunc, bail out
return ImmutableList.of();
}

Map<Expression, Expression> datetruncMap = new HashMap<>();
Literal queryLiteral = queryExprToInfoMap.get(expressionShuttledToRewrite) == null
? null : queryExprToInfoMap.get(expressionShuttledToRewrite).literal;
datetruncMap.put(queryUsedExpr, queryLiteral);
Expression replacedWithLiteral = ExpressionUtils.replace(
viewExpressionParamToDateTruncMap.get(queryUsedExpr), datetruncMap);
Expression foldedExpressionWithLiteral = FoldConstantRuleOnFE.evaluate(replacedWithLiteral,
Literal queryUsedLiteral = queryExprToInfoMap.get(queryOriginalExpr).literal;
if (!(queryUsedLiteral instanceof DateLiteral)) {
return ImmutableList.of();
}
datetruncMap.put(queryShuttledExprParam, queryUsedLiteral);
Expression dateTruncWithLiteral = ExpressionUtils.replace(
viewExprParamToDateTruncMap.get(queryShuttledExprParam), datetruncMap);
Expression foldedExpressionWithLiteral = FoldConstantRuleOnFE.evaluate(dateTruncWithLiteral,
new ExpressionRewriteContext(cascadesContext));
if (foldedExpressionWithLiteral.equals(queryLiteral)) {
// after date_trunc simplify if equals to original expression, could rewritten by mv
if (!(foldedExpressionWithLiteral instanceof DateLiteral)) {
return ImmutableList.of();
}
if (((DateLiteral) foldedExpressionWithLiteral).getDouble() == queryUsedLiteral.getDouble()) {
// after date_trunc simplify if equals to original expression, expr could be rewritten by mv
replacedExpression = ExpressionUtils.replace(expressionShuttledToRewrite,
targetToTargetReplacementMappingQueryBased,
viewExpressionParamToDateTruncMap);
viewExprParamToDateTruncMap);
}
if (replacedExpression.anyMatch(slotsToRewrite::contains)) {
// has expression not rewritten successfully, bail out
return ImmutableList.of();
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,9 @@ public boolean isInvalid() {
return Objects.equals(this, INVALID_INSTANCE);
}

/**
* Get expression list in predicates
*/
public List<Expression> toList() {
if (isInvalid()) {
return ImmutableList.of();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -408,8 +408,10 @@ public static Expression replace(Expression expr, Map<? extends Expression, ? ex
* For example.
* <pre>
* input expression: a > 1
* replaceMap: a -> b + c
*
* replaceMap: d -> b + c, transferMap: a -> d
* firstly try to get mapping expression from replaceMap by a, if can not then
* get mapping d from transferMap by a
* and get mapping b + c from replaceMap by d
* output:
* b + c > 1
* </pre>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !query1_0_before --
b 3

-- !query1_0_after --
b 3

-- !query1_1_before --
b 3

-- !query1_1_after --
b 3

-- !query1_2_before --
b 1

-- !query1_2_after --
b 1

-- !query1_3_before --
b 18

-- !query1_3_after --
b 18

-- !query1_4_before --
b 18

-- !query1_4_after --
b 18

-- !query1_5_before --
b 18

-- !query1_5_after --
b 18

-- !query1_6_before --
b 15

-- !query1_6_after --
b 15

-- !query1_7_before --
b 15

-- !query1_7_after --
b 15

-- !query1_8_before --
b 6

-- !query1_8_after --
b 6

-- !query1_9_before --
b 6

-- !query1_9_after --
b 6

-- !query1_10_before --
b 21
d 3

-- !query1_10_after --
b 21
d 3

-- !query1_11_before --
b 21
d 3

-- !query1_11_after --
b 21
d 3

-- !query1_12_before --
b 6

-- !query1_12_after --
b 6

-- !query1_13_before --
b 6

-- !query1_13_after --
b 6

-- !query1_14_before --
b 24
d 3

-- !query1_14_after --
b 24
d 3

-- !query1_15_before --
b 24
d 3

-- !query1_15_after --
b 24
d 3

-- !query2_0_before --
b 6

-- !query2_0_after --
b 6

-- !query2_1_before --

-- !query2_1_after --

-- !query2_2_before --
b 18

-- !query2_2_after --
b 18

-- !query2_3_before --
b 18

-- !query2_3_after --
b 18

-- !query3_0_before --
b 15
d 3

-- !query3_0_after --
b 15
d 3

-- !query3_1_before --
b 15
d 3

-- !query3_1_after --
b 15
d 3

Loading

0 comments on commit e095920

Please sign in to comment.