From ba675e4dbabff01d2dfd1080d9a5edc85895d013 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Wed, 6 Nov 2024 14:52:47 +0800 Subject: [PATCH] [test](mtmv) Auto calc the value of sync_cbo_rewrite variable to makesure the same test code in different version (#42279) Auto calc the value of sync_cbo_rewrite variable to make sure the same test code in different version. Because the query rewrite by mv code is a little different from 2.1 to 3.0 and master. But should keep same regression test code in different version. So add the methoed `enable_sync_mv_cost_based_rewrite` to get `enable_sync_mv_cost_based_rewrite` session variable, then decide to regression test logic by the variable auto. --- .../apache/doris/nereids/NereidsPlanner.java | 4 +- .../mv/MaterializationContext.java | 3 +- .../doris/regression/suite/Suite.groovy | 67 +++++++++++++------ 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 7ea92ee73b384f..9ceecde9950dd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -595,11 +595,11 @@ public String getExplainString(ExplainOptions explainOptions) { ExplainLevel explainLevel = getExplainLevel(explainOptions); String plan = ""; String mvSummary = ""; - if (this.getPhysicalPlan() != null && cascadesContext != null) { + if ((this.getPhysicalPlan() != null || this.getOptimizedPlan() != null) && cascadesContext != null) { mvSummary = cascadesContext.getMaterializationContexts().isEmpty() ? "" : "\n\n========== MATERIALIZATIONS ==========\n" + MaterializationContext.toSummaryString(cascadesContext.getMaterializationContexts(), - this.getPhysicalPlan()); + this.getPhysicalPlan() == null ? this.getOptimizedPlan() : this.getPhysicalPlan()); } switch (explainLevel) { case PARSED_PLAN: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 609125280ded4b..df535d59d87399 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -35,7 +35,6 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; -import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.statistics.ColumnStatistic; @@ -348,7 +347,7 @@ public String toString() { * ToSummaryString, this contains only summary info. */ public static String toSummaryString(List materializationContexts, - PhysicalPlan physicalPlan) { + Plan physicalPlan) { if (materializationContexts.isEmpty()) { return ""; } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index cb3cdeb869e355..2693797d96725f 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -1688,6 +1688,17 @@ class Suite implements GroovyInterceptable { return result.values().toList() } + // enable_sync_mv_cost_based_rewrite is true or not + boolean enable_sync_mv_cost_based_rewrite () { + def showVariable = "show variables like 'enable_sync_mv_cost_based_rewrite';" + List> result = sql(showVariable) + logger.info("enable_sync_mv_cost_based_rewrite = " + result) + if (result.isEmpty()) { + return false; + } + return Boolean.parseBoolean(result.get(0).get(1)); + } + // Given tables to decide whether the table partition row count statistic is ready or not boolean is_partition_statistics_ready(db, tables) { boolean isReady = true; @@ -1767,12 +1778,14 @@ class Suite implements GroovyInterceptable { // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite // is_partition_statistics_ready is the bool value which identifying if partition row count is valid or not // if true, check if chosen by cbo or doesn't check - def mv_rewrite_success = { query_sql, mv_name, sync_cbo_rewrite = true, is_partition_statistics_ready = true -> + void mv_rewrite_success(query_sql, mv_name, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite(), + is_partition_statistics_ready = true) { logger.info("query_sql = " + query_sql + ", mv_name = " + mv_name + ", sync_cbo_rewrite = " +sync_cbo_rewrite + ", is_partition_statistics_ready = " + is_partition_statistics_ready) if (!is_partition_statistics_ready) { // If partition statistics is no ready, degrade to without check cbo chosen - return mv_rewrite_success_without_check_chosen(query_sql, mv_name, sync_cbo_rewrite) + mv_rewrite_success_without_check_chosen(query_sql, mv_name, sync_cbo_rewrite) + return } if (!sync_cbo_rewrite) { explain { @@ -1791,12 +1804,14 @@ class Suite implements GroovyInterceptable { // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite // is_partition_statistics_ready is the bool value which identifying if partition row count is valid or not // if true, check if chosen by cbo or doesn't check - def mv_rewrite_all_success = { query_sql, mv_names, sync_cbo_rewrite = true, is_partition_statistics_ready = true -> + void mv_rewrite_all_success( query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite(), + is_partition_statistics_ready = true) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names + ", sync_cbo_rewrite = " +sync_cbo_rewrite + ", is_partition_statistics_ready = " + is_partition_statistics_ready) if (!is_partition_statistics_ready) { // If partition statistics is no ready, degrade to without check cbo chosen - return mv_rewrite_all_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite) + mv_rewrite_all_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite) + return } if (!sync_cbo_rewrite) { explain { @@ -1827,12 +1842,14 @@ class Suite implements GroovyInterceptable { // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite // is_partition_statistics_ready is the bool value which identifying if partition row count is valid or not // if true, check if chosen by cbo or doesn't check - def mv_rewrite_any_success = { query_sql, mv_names, sync_cbo_rewrite = true, is_partition_statistics_ready = true -> + void mv_rewrite_any_success(query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite(), + is_partition_statistics_ready = true) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names + ", sync_cbo_rewrite = " +sync_cbo_rewrite + ", is_partition_statistics_ready = " + is_partition_statistics_ready) if (!is_partition_statistics_ready) { // If partition statistics is no ready, degrade to without check cbo chosen - return mv_rewrite_any_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite) + mv_rewrite_any_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite) + return } if (!sync_cbo_rewrite) { explain { @@ -1861,7 +1878,8 @@ class Suite implements GroovyInterceptable { // multi mv part in rewrite process, all rewrte success without check if chosen by cbo // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_all_success_without_check_chosen = { query_sql, mv_names, sync_cbo_rewrite = true -> + void mv_rewrite_all_success_without_check_chosen(query_sql, mv_names, + sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()){ logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) if (!sync_cbo_rewrite) { explain { @@ -1869,7 +1887,9 @@ class Suite implements GroovyInterceptable { check { result -> boolean success = true; for (String mv_name : mv_names) { - success = success && result.contains("(${mv_name})") + def splitResult = result.split("MaterializedViewRewriteFail") + def each_result = splitResult.length == 2 ? splitResult[0].contains(mv_name) : false + success = success && (result.contains("(${mv_name})") || each_result) } Assert.assertEquals(true, success) } @@ -1891,7 +1911,8 @@ class Suite implements GroovyInterceptable { // multi mv part in rewrite process, any of them rewrte success without check if chosen by cbo or not // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_any_success_without_check_chosen = { query_sql, mv_names, sync_cbo_rewrite = true -> + void mv_rewrite_any_success_without_check_chosen(query_sql, mv_names, + sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) if (!sync_cbo_rewrite) { explain { @@ -1899,7 +1920,9 @@ class Suite implements GroovyInterceptable { check { result -> boolean success = false; for (String mv_name : mv_names) { - success = success || result.contains("(${mv_name})") + def splitResult = result.split("MaterializedViewRewriteFail") + def each_result = splitResult.length == 2 ? splitResult[0].contains(mv_name) : false + success = success || (result.contains("(${mv_name})") || each_result) } Assert.assertEquals(true, success) } @@ -1908,7 +1931,7 @@ class Suite implements GroovyInterceptable { } explain { sql(" memo plan ${query_sql}") - check {result -> + check { result -> boolean success = false for (String mv_name : mv_names) { success = success || result.contains("${mv_name} chose") || result.contains("${mv_name} not chose") @@ -1920,12 +1943,16 @@ class Suite implements GroovyInterceptable { // multi mv part in rewrite process, rewrte success without check if chosen by cbo or not // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_success_without_check_chosen = { query_sql, mv_name, sync_cbo_rewrite = true -> + void mv_rewrite_success_without_check_chosen(query_sql, mv_name, + sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_name = " + mv_name) if (!sync_cbo_rewrite) { explain { sql("${query_sql}") - contains("(${mv_name})") + check { result -> + def splitResult = result.split("MaterializedViewRewriteFail") + result.contains("(${mv_name})") || (splitResult.length == 2 ? splitResult[0].contains(mv_name) : false) + } } return } @@ -1939,12 +1966,12 @@ class Suite implements GroovyInterceptable { // single mv part in rewrite process, rewrte fail // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_fail = { query_sql, mv_name, sync_cbo_rewrite = true -> + void mv_rewrite_fail(query_sql, mv_name, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_name = " + mv_name) if (!sync_cbo_rewrite) { explain { sql("${query_sql}") - nonContains("(${mv_name})") + notContains("(${mv_name})") } return } @@ -1956,7 +1983,7 @@ class Suite implements GroovyInterceptable { // multi mv part in rewrite process, all rewrte fail // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_all_fail = {query_sql, mv_names, sync_cbo_rewrite = true -> + void mv_rewrite_all_fail(query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) if (!sync_cbo_rewrite) { explain { @@ -1987,7 +2014,7 @@ class Suite implements GroovyInterceptable { // multi mv part in rewrite process, any rewrte fail // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_any_fail = {query_sql, mv_names, sync_cbo_rewrite = true -> + void mv_rewrite_any_fail (query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) if (!sync_cbo_rewrite) { explain { @@ -2026,7 +2053,7 @@ class Suite implements GroovyInterceptable { """ def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) - mv_rewrite_success(query_sql, mv_name) + mv_rewrite_success(query_sql, mv_name, true) } def async_mv_rewrite_success_without_check_chosen = { db, mv_sql, query_sql, mv_name -> @@ -2042,7 +2069,7 @@ class Suite implements GroovyInterceptable { def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) - mv_rewrite_success_without_check_chosen(query_sql, mv_name) + mv_rewrite_success_without_check_chosen(query_sql, mv_name, true) } @@ -2059,7 +2086,7 @@ class Suite implements GroovyInterceptable { def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) - mv_rewrite_fail(query_sql, mv_name) + mv_rewrite_fail(query_sql, mv_name, true) } def token = context.config.metaServiceToken