From 6ea6ff5dee6eba0478e46d141a34de532e29c31f Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Sun, 24 Dec 2023 09:00:49 +0800 Subject: [PATCH] [fix](nereids) Fix query mv rewrite fail when mv cache build quickly (#28876) --- .../mv/AbstractMaterializedViewRule.java | 23 ++----------- .../mv/MaterializationContext.java | 8 +++++ .../aggregate_with_roll_up.groovy | 33 +++++++++++++++++-- .../aggregate_without_roll_up.groovy | 17 +++++++--- 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 6dcf1885563691..90ebe567c4628e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -26,7 +26,6 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.BaseTableInfo; -import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.CascadesContext; @@ -104,13 +103,8 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { logger.debug(currentClassName + " this group is already rewritten so skip"); continue; } - MTMV mtmv = materializationContext.getMTMV(); - MTMVCache mtmvCache = getCacheFromMTMV(mtmv); - if (mtmvCache == null) { - logger.warn(currentClassName + " mv cache is null so return"); - return rewriteResults; - } - List viewStructInfos = extractStructInfo(mtmvCache.getLogicalPlan(), cascadesContext); + List viewStructInfos = extractStructInfo(materializationContext.getMvPlan(), + cascadesContext); if (viewStructInfos.size() > 1) { // view struct info should only have one logger.warn(currentClassName + " the num of view struct info is more then one so return"); @@ -200,7 +194,7 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { CascadesContext rewrittenPlanContext = CascadesContext.initContext(cascadesContext.getStatementContext(), rewrittenPlan, cascadesContext.getCurrentJobContext().getRequiredProperties()); - Rewriter.getWholeTreeRewriter(cascadesContext).execute(); + Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute(); rewrittenPlan = rewrittenPlanContext.getRewritePlan(); logger.debug(currentClassName + "rewrite by materialized view success"); rewriteResults.add(rewrittenPlan); @@ -289,17 +283,6 @@ protected boolean checkPartitionIsValid( && relatedTalbeValidSet.containsAll(relatedTableSelectedPartitionToCheck); } - private MTMVCache getCacheFromMTMV(MTMV mtmv) { - MTMVCache cache; - try { - cache = mtmv.getOrGenerateCache(); - } catch (AnalysisException analysisException) { - logger.warn(this.getClass().getSimpleName() + " get mtmv cache analysisException", analysisException); - return null; - } - return cache; - } - /** * Rewrite query by view, for aggregate or join rewriting should be different inherit class implementation */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 19b1f5be8bb10a..153688ecc2d3dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -51,6 +51,8 @@ public class MaterializationContext { // generate form mv scan plan private ExpressionMapping mvExprToMvScanExprMapping; private boolean available = true; + // the mv plan from cache at present, record it to make sure query rewrite by mv is right when cache change. + private Plan mvPlan; /** * MaterializationContext, this contains necessary info for query rewriting by mv @@ -81,6 +83,8 @@ public MaterializationContext(MTMV mtmv, mtmvCache.getMvOutputExpressions(), mtmvCache.getLogicalPlan()), mvScanPlan.getExpressions()); + // copy the plan from cache, which the plan in cache may change + this.mvPlan = mtmvCache.getLogicalPlan(); } public Set getMatchedGroups() { @@ -119,6 +123,10 @@ public boolean isAvailable() { return available; } + public Plan getMvPlan() { + return mvPlan; + } + /** * MaterializationContext fromMaterializedView */ diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index e67086deba0668..b56922d5db6470 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -42,7 +42,11 @@ suite("aggregate_with_roll_up") { O_COMMENT VARCHAR(79) NOT NULL ) DUPLICATE KEY(o_orderkey, o_custkey) - PARTITION BY RANGE(o_orderdate) (PARTITION `day_2` VALUES LESS THAN ('2023-12-30')) + PARTITION BY RANGE(o_orderdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" @@ -73,7 +77,10 @@ suite("aggregate_with_roll_up") { l_comment VARCHAR(44) NOT NULL ) DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) - PARTITION BY RANGE(l_shipdate) (PARTITION `day_1` VALUES LESS THAN ('2023-12-30')) + PARTITION BY RANGE(l_shipdate) ( + PARTITION `day_1` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_2` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_3` VALUES LESS THAN ("2023-12-30")) DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" @@ -144,6 +151,26 @@ suite("aggregate_with_roll_up") { } } + def check_rewrite_with_mv_partition = { mv_sql, query_sql, mv_name, partition_column -> + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (${partition_column}) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + + def job_name = getJobName(db, mv_name); + waitingMTMVTaskFinished(job_name) + explain { + sql("${query_sql}") + contains "(${mv_name})" + } + } + def check_rewrite_with_force_analyze = { mv_sql, query_sql, mv_name -> sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" @@ -283,7 +310,7 @@ suite("aggregate_with_roll_up") { "l_partkey, " + "l_suppkey" order_qt_query15_0_before "${query15_0}" - check_rewrite(mv15_0, query15_0, "mv15_0") + check_rewrite_with_mv_partition(mv15_0, query15_0, "mv15_0", "l_shipdate") order_qt_query15_0_after "${query15_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv15_0""" diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index b76763752fbd36..72e80eadbf93f0 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -43,11 +43,15 @@ suite("aggregate_without_roll_up") { o_comment VARCHAR(79) NOT NULL ) DUPLICATE KEY(o_orderkey, o_custkey) - PARTITION BY RANGE(o_orderdate) (PARTITION `day_2` VALUES LESS THAN ('2023-12-30')) + PARTITION BY RANGE(o_orderdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" - ) + ); """ sql """ @@ -74,7 +78,11 @@ suite("aggregate_without_roll_up") { l_comment VARCHAR(44) NOT NULL ) DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) - PARTITION BY RANGE(l_shipdate) (PARTITION `day_1` VALUES LESS THAN ('2023-12-30')) + PARTITION BY RANGE(l_shipdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" @@ -100,7 +108,8 @@ suite("aggregate_without_roll_up") { ) """ - sql """ insert into lineitem values + sql """ + insert into lineitem values (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),