From 2752e77edd07a64b1d82f0a245c7a4387764bd81 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:07:28 +0800 Subject: [PATCH] [fix](mtmv) Fix enable_materialized_view_nest_rewrite session variable is useless in some scene (#41472) This is brought by #34050 if set `enable_materialized_view_nest_rewrite = false`, as expected, top level materialized view should rewritten fail, but now successfully. Such as first level materialized view def is CREATE MATERIALIZED VIEW level1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS SELECT l_orderkey, l_linenumber, l_partkey, o_orderkey, o_custkey FROM lineitem_2 INNER JOIN orders_2 ON l_orderkey = o_orderkey; second level materialized view def is CREATE MATERIALIZED VIEW level2 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS SELECT l_orderkey, l_linenumber, o_orderkey, sum(l_partkey) AS total_revenue, max(o_custkey) AS max_discount FROM join_mv1 GROUP BY l_orderkey, l_linenumber, o_orderkey; if set `enable_materialized_view_nest_rewrite = false`, only `level1` can rewriten succesfully and chosen by cbo if set `enable_materialized_view_nest_rewrite = true`, both `level1` and `level2` can rewriten succesfully and `level2` should be chosen by cbo. This pr fixed this --- .../apache/doris/nereids/CascadesContext.java | 9 + .../doris/nereids/memo/StructInfoMap.java | 16 +- .../apache/doris/nereids/rules/RuleSet.java | 2 - .../mv/AbstractMaterializedViewRule.java | 13 +- .../mv/MaterializedViewOnlyJoinRule.java | 45 ----- ...MaterializedViewProjectFilterJoinRule.java | 1 + .../exploration/mv/MaterializedViewUtils.java | 2 +- .../mv/direct_query/direct_query.groovy | 1 + .../nested_mtmv_rewrite_switch.groovy | 169 ++++++++++++++++++ 9 files changed, 207 insertions(+), 51 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 73f3ceb385f6ab..a5c966370f030d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -122,6 +122,7 @@ public class CascadesContext implements ScheduleContext { private final Optional parent; private final Set materializationContexts; + private final Set> materializationRewrittenSuccessSet = new HashSet<>(); private boolean isLeadingJoin = false; private boolean isLeadingDisableJoinReorder = false; @@ -368,6 +369,14 @@ public void addMaterializationContext(MaterializationContext materializationCont this.materializationContexts.add(materializationContext); } + public Set> getMaterializationRewrittenSuccessSet() { + return materializationRewrittenSuccessSet; + } + + public void addMaterializationRewrittenSuccess(List materializationQualifier) { + this.materializationRewrittenSuccessSet.add(materializationQualifier); + } + /** * getAndCacheSessionVariable */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index 0c11b5fbb224ba..4aa4f146b874da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.memo; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo; @@ -126,6 +127,9 @@ public void refresh(Group group, CascadesContext cascadesContext) { List> childrenTableMap = new LinkedList<>(); if (groupExpression.children().isEmpty()) { BitSet leaf = constructLeaf(groupExpression, cascadesContext); + if (leaf.isEmpty()) { + break; + } groupExpressionMap.put(leaf, Pair.of(groupExpression, new LinkedList<>())); continue; } @@ -163,9 +167,19 @@ public void refresh(Group group, CascadesContext cascadesContext) { private BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext) { Plan plan = groupExpression.getPlan(); BitSet tableMap = new BitSet(); + boolean enableMaterializedViewNestRewrite = cascadesContext.getConnectContext().getSessionVariable() + .isEnableMaterializedViewNestRewrite(); if (plan instanceof LogicalCatalogRelation) { + TableIf table = ((LogicalCatalogRelation) plan).getTable(); + // If disable materialized view nest rewrite, and mv already rewritten successfully once, doesn't construct + // table id map for nest mv rewrite + if (!enableMaterializedViewNestRewrite + && cascadesContext.getMaterializationRewrittenSuccessSet().contains(table.getFullQualifiers())) { + return tableMap; + + } tableMap.set(cascadesContext.getStatementContext() - .getTableId(((LogicalCatalogRelation) plan).getTable()).asInt()); + .getTableId(table).asInt()); } // one row relation / CTE consumer return tableMap; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 26868665b10806..c96e395c5330ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -39,7 +39,6 @@ import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectJoinRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectScanRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterScanRule; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyJoinRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyScanRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectAggregateRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterAggregateRule; @@ -228,7 +227,6 @@ public class RuleSet { .build(); public static final List MATERIALIZED_VIEW_RULES = planRuleFactories() - .add(MaterializedViewOnlyJoinRule.INSTANCE) .add(MaterializedViewProjectJoinRule.INSTANCE) .add(MaterializedViewFilterJoinRule.INSTANCE) .add(MaterializedViewFilterProjectJoinRule.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 7d84b8ab36b59c..dd17d754244a15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -37,6 +37,7 @@ import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -354,6 +355,13 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca rewrittenPlanOutput, queryPlan.getOutput())); continue; } + // Merge project + rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext, + childContext -> { + Rewriter.getCteChildrenRewriter(childContext, + ImmutableList.of(Rewriter.bottomUp(new MergeProjects()))).execute(); + return childContext.getRewritePlan(); + }, rewrittenPlan, queryPlan); if (!isOutputValid(queryPlan, rewrittenPlan)) { LogicalProperties logicalProperties = rewrittenPlan.getLogicalProperties(); materializationContext.recordFailReason(queryStructInfo, @@ -363,7 +371,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca logicalProperties, queryPlan.getLogicalProperties())); continue; } - recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext); + recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext, cascadesContext); trySetStatistics(materializationContext, cascadesContext); rewriteResults.add(rewrittenPlan); // if rewrite successfully, try to regenerate mv scan because it maybe used again @@ -852,8 +860,9 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon return checkQueryPattern(structInfo, cascadesContext); } - protected void recordIfRewritten(Plan plan, MaterializationContext context) { + protected void recordIfRewritten(Plan plan, MaterializationContext context, CascadesContext cascadesContext) { context.setSuccess(true); + cascadesContext.addMaterializationRewrittenSuccess(context.generateMaterializationIdentifier()); if (plan.getGroupExpression().isPresent()) { context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId(), true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java deleted file mode 100644 index 2735ca87fe9c39..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; - -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * This is responsible for join pattern such as only join - */ -public class MaterializedViewOnlyJoinRule extends AbstractMaterializedViewJoinRule { - - public static final MaterializedViewOnlyJoinRule INSTANCE = new MaterializedViewOnlyJoinRule(); - - @Override - public List buildRules() { - return ImmutableList.of(logicalJoin(any().when(LogicalPlan.class::isInstance), - any().when(LogicalPlan.class::isInstance)).thenApplyMultiNoThrow(ctx -> { - LogicalJoin root = ctx.root; - return rewrite(root, ctx.cascadesContext); - }).toRule(RuleType.MATERIALIZED_VIEW_ONLY_JOIN)); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java index d82f838ea6b66d..05f54ac3401d55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java @@ -31,6 +31,7 @@ /** * This is responsible for join pattern such as project on filter on join + * Needed because variant data type would have filter on join directly, such as query query3_5 in variant_mv.groovy */ public class MaterializedViewProjectFilterJoinRule extends AbstractMaterializedViewJoinRule { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 6af72b1e81db3f..7997363a8e9b94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -212,8 +212,8 @@ public static List extractStructInfo(Plan plan, Plan originalPlan, C structInfosBuilder.add(structInfo); } } - return structInfosBuilder.build(); } + return structInfosBuilder.build(); } // if plan doesn't belong to any group, construct it directly return ImmutableList.of(StructInfo.of(plan, originalPlan, cascadesContext)); diff --git a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy index 6ebc97896e5e6c..f4fc91db26c3b8 100644 --- a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy @@ -20,6 +20,7 @@ suite("direct_query_mv") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" sql "set runtime_filter_mode=OFF" + sql """set enable_materialized_view_nest_rewrite = true; """ sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy new file mode 100644 index 00000000000000..f9a84fa5250179 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("nested_mtmv_rewrite_switch") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "SET enable_materialized_view_rewrite=true" + + sql """ + drop table if exists orders_2 + """ + sql """ + CREATE TABLE `orders_2` ( + `o_orderkey` BIGINT, + `o_custkey` int, + `o_orderstatus` VARCHAR(1), + `o_totalprice` DECIMAL(15, 2), + `o_orderpriority` VARCHAR(15), + `o_clerk` VARCHAR(15), + `o_shippriority` int, + `o_comment` VARCHAR(79), + `o_orderdate` DATE + ) ENGINE=olap + + PROPERTIES ( + "replication_num" = "1" + + ); + """ + + sql """ + drop table if exists lineitem_2 + """ + sql """ + CREATE TABLE `lineitem_2` ( + `l_orderkey` BIGINT, + `l_linenumber` INT, + `l_partkey` INT, + `l_suppkey` INT, + `l_quantity` DECIMAL(15, 2), + `l_extendedprice` DECIMAL(15, 2), + `l_discount` DECIMAL(15, 2), + `l_tax` DECIMAL(15, 2), + `l_returnflag` VARCHAR(1), + `l_linestatus` VARCHAR(1), + `l_commitdate` DATE, + `l_receiptdate` DATE, + `l_shipinstruct` VARCHAR(25), + `l_shipmode` VARCHAR(10), + `l_comment` VARCHAR(44), + `l_shipdate` DATE + ) ENGINE=olap + + PROPERTIES ( + "replication_num" = "1" + + ); + """ + + sql """ + insert into orders_2 values + (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), + (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'), + (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'), + (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'), + (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'), + (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'), + (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'), + (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'), + (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), + (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); + """ + + sql""" + insert into lineitem_2 values + (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'), + (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'); + """ + + sql """analyze table orders_2 with sync;""" + sql """analyze table lineitem_2 with sync;""" + + + def compare_res = { def stmt -> + sql "SET enable_materialized_view_rewrite=false" + def origin_res = sql stmt + logger.info("origin_res: " + origin_res) + sql "SET enable_materialized_view_rewrite=true" + def mv_origin_res = sql stmt + logger.info("mv_origin_res: " + mv_origin_res) + assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size())) + for (int row = 0; row < mv_origin_res.size(); row++) { + assertTrue(mv_origin_res[row].size() == origin_res[row].size()) + for (int col = 0; col < mv_origin_res[row].size(); col++) { + assertTrue(mv_origin_res[row][col] == origin_res[row][col]) + } + } + } + + + // create base first level mv + create_async_mv(db, "join_mv1", """ + SELECT l_orderkey, l_linenumber, l_partkey, o_orderkey, o_custkey + FROM lineitem_2 INNER JOIN orders_2 + ON l_orderkey = o_orderkey; + """) + + // create second level mv based on first level mv + create_async_mv(db, "agg_mv2", """ + SELECT + l_orderkey, + l_linenumber, + o_orderkey, + sum(l_partkey) AS total_revenue, + max(o_custkey) AS max_discount + FROM join_mv1 + GROUP BY l_orderkey, l_linenumber, o_orderkey; + """) + + // create third level mv based on second level mv + create_async_mv(db, "join_agg_mv3", """ + SELECT + l_orderkey, + sum(total_revenue) AS total_revenue, + max(max_discount) AS max_discount + FROM agg_mv2 + GROUP BY l_orderkey; + """) + + def query = """ + SELECT l_orderkey, sum(l_partkey) AS total_revenue, max(o_custkey) AS max_discount FROM lineitem_2 INNER JOIN orders_2 ON l_orderkey = o_orderkey GROUP BY l_orderkey + """ + + sql """set enable_materialized_view_nest_rewrite = false;""" + // Just first level mv rewrite successfully, second and third level mv should rewriten fail + mv_rewrite_fail(query, "agg_mv2") + mv_rewrite_fail(query, "join_agg_mv3") + mv_rewrite_success(query, "join_mv1") + compare_res(query + " order by 1,2,3") + + + sql """set enable_materialized_view_nest_rewrite = true;""" + // All mv rewrite successfully but only thirst level mv can be chosen by cbo + mv_rewrite_success_without_check_chosen(query, "join_mv1") + mv_rewrite_success_without_check_chosen(query, "agg_mv2") + mv_rewrite_success(query, "join_agg_mv3") + compare_res(query + " order by 1,2,3") + + +}