diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index 56061c75b9cee28..2895ad73e14297b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -51,14 +51,18 @@ public class MTMVCache { // The materialized view plan which should be optimized by the same rules to query // and will remove top sink and unused sort private final Plan logicalPlan; - // The original plan of mv def sql + // The original rewritten plan of mv def sql private final Plan originalPlan; + // The analyzed plan of mv def sql, which is used by tableCollector,should not be optimized by rbo + private final Plan analyzedPlan; private final Statistics statistics; private final StructInfo structInfo; - public MTMVCache(Plan logicalPlan, Plan originalPlan, Statistics statistics, StructInfo structInfo) { + public MTMVCache(Plan logicalPlan, Plan originalPlan, Plan analyzedPlan, + Statistics statistics, StructInfo structInfo) { this.logicalPlan = logicalPlan; this.originalPlan = originalPlan; + this.analyzedPlan = analyzedPlan; this.statistics = statistics; this.structInfo = structInfo; } @@ -71,6 +75,10 @@ public Plan getOriginalPlan() { return originalPlan; } + public Plan getAnalyzedPlan() { + return analyzedPlan; + } + public Statistics getStatistics() { return statistics; } @@ -117,7 +125,7 @@ public Plan visitLogicalResultSink(LogicalResultSink logicalResu Optional structInfoOptional = MaterializationContext.constructStructInfo(mvPlan, originPlan, planner.getCascadesContext(), new BitSet()); - return new MTMVCache(mvPlan, originPlan, needCost + return new MTMVCache(mvPlan, originPlan, planner.getAnalyzedPlan(), needCost ? planner.getCascadesContext().getMemo().getRoot().getStatistics() : null, structInfoOptional.orElseGet(() -> null)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 3a41c873ba827c2..4f732e384e74818 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -41,7 +41,6 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; import java.util.List; import java.util.Optional; @@ -97,32 +96,32 @@ public static MTMVRelation generateMTMVRelation(MTMV mtmv, ConnectContext ctx) { // Should not make table without data to empty relation when analyze the related table, // so add disable rules Plan plan = getAnalyzePlanBySql(mtmv.getQuerySql(), ctx); - return generateMTMVRelation(plan); + return generateMTMVRelation(plan, ctx); } - public static MTMVRelation generateMTMVRelation(Plan plan) { - return new MTMVRelation(getBaseTables(plan, true, true), - getBaseTables(plan, false, true), - getBaseViews(plan, true, true)); + public static MTMVRelation generateMTMVRelation(Plan plan, ConnectContext connectContext) { + return new MTMVRelation(getBaseTables(plan, true, true, connectContext), + getBaseTables(plan, false, true, connectContext), + getBaseViews(plan, true, true, connectContext)); } private static Set getBaseTables(Plan plan, boolean expandMaterializedView, - boolean expandView) { + boolean expandView, ConnectContext connectContext) { TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( com.google.common.collect.Sets - .newHashSet(TableType.values()), expandMaterializedView, expandView); + .newHashSet(TableType.values()), connectContext, expandMaterializedView, expandView); plan.accept(TableCollector.INSTANCE, collectorContext); Set collectedTables = collectorContext.getCollectedTables(); return transferTableIfToInfo(collectedTables); } private static Set getBaseViews(Plan plan, boolean expandMaterializedView, - boolean expandView) { + boolean expandView, ConnectContext connectContext) { TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( com.google.common.collect.Sets - .newHashSet(TableType.VIEW), expandMaterializedView, expandView); + .newHashSet(TableType.VIEW), connectContext, expandMaterializedView, expandView); plan.accept(TableCollector.INSTANCE, collectorContext); Set collectedTables = collectorContext.getCollectedTables(); return transferTableIfToInfo(collectedTables); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index bad966252a09af8..4402ea34fafa410 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -80,13 +80,13 @@ public void initMaterializationContext(CascadesContext cascadesContext) { */ protected void doInitMaterializationContext(CascadesContext cascadesContext) { // Only collect the table or mv which query use directly, to avoid useless mv partition in rewrite + // Keep use one connection context when in query, if new connect context, + // the ConnectionContext.get() will change TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), + cascadesContext.getConnectContext(), false, true); try { Plan rewritePlan = cascadesContext.getRewritePlan(); - // Keep use one connection context when in query, if new connect context, - // the ConnectionContext.get() will change - collectorContext.setConnectContext(cascadesContext.getConnectContext()); rewritePlan.accept(TableCollector.INSTANCE, collectorContext); } catch (Exception e) { LOG.warn(String.format("MaterializationContext init table collect fail, current queryId is %s", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index a659c2f9990a3fe..9c2226bd8beffef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -343,7 +343,7 @@ public Plan visitLogicalResultSink(LogicalResultSink logicalResu ImmutableList.of(Rewriter.custom(RuleType.ELIMINATE_SORT, EliminateSort::new))).execute(); return childContext.getRewritePlan(); }, mvPlan, originPlan); - return new MTMVCache(mvPlan, originPlan, + return new MTMVCache(mvPlan, originPlan, planner.getAnalyzedPlan(), planner.getCascadesContext().getMemo().getRoot().getStatistics(), null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 61c0fb33b2c84e7..fb2e6dea49ae631 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -252,8 +252,6 @@ public void analyzeQuery(ConnectContext ctx, Map mvProperties) t NereidsPlanner planner = new NereidsPlanner(statementContext); // this is for expression column name infer when not use alias LogicalSink logicalSink = new UnboundResultSink<>(logicalQuery); - // must disable constant folding by be, because be constant folding may return wrong type - ctx.getSessionVariable().disableConstantFoldingByBEOnce(); // Should not make table without data to empty relation when analyze the related table, // so add disable rules Set tempDisableRules = ctx.getSessionVariable().getDisableNereidsRuleNames(); @@ -326,7 +324,7 @@ private void analyzeKeys() { // Should use analyzed plan for collect views and tables private void getRelation(NereidsPlanner planner) { - this.relation = MTMVPlanUtil.generateMTMVRelation(planner.getAnalyzedPlan()); + this.relation = MTMVPlanUtil.generateMTMVRelation(planner.getAnalyzedPlan(), planner.getConnectContext()); } private PartitionDesc generatePartitionDesc(ConnectContext ctx) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java index c3111945f968173..5e536a5042313c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java @@ -21,8 +21,8 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.View; +import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.MTMVCache; -import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalView; @@ -88,9 +88,15 @@ private void expandMvAndCollect(MTMV mtmv, TableCollectorContext context) { return; } // Make sure use only one connection context when in query to avoid ConnectionContext.get() wrong - MTMVCache expandedMv = MTMVCache.from(mtmv, context.getConnectContext() == null - ? MTMVPlanUtil.createMTMVContext(mtmv) : context.getConnectContext(), false); - expandedMv.getLogicalPlan().accept(this, context); + MTMVCache expandedMvCache; + try { + expandedMvCache = mtmv.getOrGenerateCache(context.getConnectContext()); + } catch (AnalysisException exception) { + LOG.warn(String.format("expandMvAndCollect getOrGenerateCache fail, mtmv name is %s", mtmv.getName()), + exception); + expandedMvCache = MTMVCache.from(mtmv, context.getConnectContext(), false); + } + expandedMvCache.getAnalyzedPlan().accept(this, context); } /** @@ -103,11 +109,13 @@ public static final class TableCollectorContext { // if expand the mv or not private final boolean expandMaterializedView; private final boolean expandView; - private ConnectContext connectContext; + private final ConnectContext connectContext; - public TableCollectorContext(Set targetTableTypes, boolean expandMaterializedView, + public TableCollectorContext(Set targetTableTypes, + ConnectContext connectContext, boolean expandMaterializedView, boolean expandView) { this.targetTableTypes = targetTableTypes; + this.connectContext = connectContext; this.expandMaterializedView = expandMaterializedView; this.expandView = expandView; } @@ -120,16 +128,12 @@ public Set getTargetTableTypes() { return targetTableTypes; } - public boolean isExpandMaterializedView() { - return expandMaterializedView; - } - public ConnectContext getConnectContext() { return connectContext; } - public void setConnectContext(ConnectContext connectContext) { - this.connectContext = connectContext; + public boolean isExpandMaterializedView() { + return expandMaterializedView; } public boolean isExpandView() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java index c87aa254273b823..11656fd11a666ec 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java @@ -27,20 +27,17 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Random; import org.apache.doris.nereids.trees.expressions.functions.scalar.UnixTimestamp; import org.apache.doris.nereids.trees.expressions.functions.scalar.Uuid; +import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.visitor.TableCollector; import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; import org.apache.doris.nereids.util.PlanChecker; -import org.apache.doris.qe.SessionVariable; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.Sets; -import mockit.Mock; -import mockit.MockUp; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.BitSet; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -55,7 +52,8 @@ public class PlanVisitorTest extends TestWithFeService { protected void runBeforeAll() throws Exception { createDatabase("visitor_test"); useDatabase("visitor_test"); - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + // This test the method which used in create mtmv, so need to use the same disable rules + connectContext.getSessionVariable().setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); createTable("CREATE TABLE `table1` (\n" + " `c1` varchar(20) NULL,\n" @@ -148,7 +146,8 @@ public void test1() { Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Random); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true, true); + Sets.newHashSet(TableType.OLAP), connectContext, + true, true); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -177,7 +176,8 @@ public void test2() { Assertions.assertTrue(nondeterministicFunctionSet.get(1) instanceof Random); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true, true); + Sets.newHashSet(TableType.OLAP), connectContext, + true, true); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -192,14 +192,6 @@ public void test2() { @Test public void testCollectTable() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - BitSet disableNereidsRules = connectContext.getSessionVariable().getDisableNereidsRules(); - new MockUp() { - @Mock - public BitSet getDisableNereidsRules() { - return disableNereidsRules; - } - }; PlanChecker.from(connectContext) .checkExplain("SELECT view2.* FROM " + "view2 " @@ -216,7 +208,8 @@ public BitSet getDisableNereidsRules() { // view 1 contains table1 and table2, view2 contains view1 and table5 // mv1 contains table1 and table3 TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true, true); + Sets.newHashSet(TableType.OLAP), connectContext, + true, true); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -230,7 +223,8 @@ public BitSet getDisableNereidsRules() { expectedTables); collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.VIEW), true, true); + Sets.newHashSet(TableType.VIEW), connectContext, + true, true); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); expectedTables = new HashSet<>(); expectedTables.add("view1"); @@ -243,7 +237,8 @@ public BitSet getDisableNereidsRules() { // Expand view and materialized view, only all type table collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(), true, true); + Sets.newHashSet(), connectContext, + true, true); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -263,7 +258,8 @@ public BitSet getDisableNereidsRules() { // view 1 contains table1 and table2, view2 contains view1 and table5 // mv1 contains table1 and table3 collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), false, true); + Sets.newHashSet(TableType.OLAP), connectContext, + false, true); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -276,7 +272,8 @@ public BitSet getDisableNereidsRules() { expectedTables); collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.VIEW), false, true); + Sets.newHashSet(TableType.VIEW), connectContext, + false, true); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); expectedTables = new HashSet<>(); expectedTables.add("view1"); @@ -289,7 +286,8 @@ public BitSet getDisableNereidsRules() { // Expand view but not materialized view, collect all type table collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(), false, true); + Sets.newHashSet(), connectContext, + false, true); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -308,7 +306,8 @@ public BitSet getDisableNereidsRules() { // view 1 contains table1 and table2, view2 contains view1 and table5 // mv1 contains table1 and table3 collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true, false); + Sets.newHashSet(TableType.OLAP), connectContext, + true, false); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -321,7 +320,8 @@ public BitSet getDisableNereidsRules() { expectedTables); collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.VIEW), true, false); + Sets.newHashSet(TableType.VIEW), connectContext, + true, false); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); expectedTables = new HashSet<>(); expectedTables.add("view2"); @@ -333,7 +333,8 @@ public BitSet getDisableNereidsRules() { // Expand materialized view but not view, collect all type table collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(), true, false); + Sets.newHashSet(), connectContext, + true, false); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -351,7 +352,8 @@ public BitSet getDisableNereidsRules() { // view 1 contains table1 and table2, view2 contains view1 and table5 // mv1 contains table1 and table3 collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), false, false); + Sets.newHashSet(TableType.OLAP), connectContext, + false, false); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); expectedTables = new HashSet<>(); expectedTables.add("table2"); @@ -362,7 +364,8 @@ public BitSet getDisableNereidsRules() { expectedTables); collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.VIEW), false, false); + Sets.newHashSet(TableType.VIEW), connectContext, + false, false); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); expectedTables = new HashSet<>(); expectedTables.add("view2"); @@ -374,7 +377,8 @@ public BitSet getDisableNereidsRules() { // Not expand materialized view and view, collect all type table collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(), false, false); + Sets.newHashSet(), connectContext, + false, false); analyzedPlan.accept(TableCollector.INSTANCE, collectorContext); expectedTables = new HashSet<>(); expectedTables.add("table2"); @@ -390,14 +394,6 @@ public BitSet getDisableNereidsRules() { @Test public void test3() throws Exception { - connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); - BitSet disableNereidsRules = connectContext.getSessionVariable().getDisableNereidsRules(); - new MockUp() { - @Mock - public BitSet getDisableNereidsRules() { - return disableNereidsRules; - } - }; PlanChecker.from(connectContext) .checkPlannerResult("SELECT mv1.*, uuid() FROM mv1 " + "INNER JOIN view1 on mv1.c1 = view1.c2 " @@ -412,7 +408,8 @@ public BitSet getDisableNereidsRules() { Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Uuid); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true, true); + Sets.newHashSet(TableType.OLAP), connectContext, + true, true); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -426,7 +423,7 @@ public BitSet getDisableNereidsRules() { TableCollectorContext collectorContextWithNoExpand = new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.OLAP), - false, true); + connectContext, false, true); physicalPlan.accept(TableCollector.INSTANCE, collectorContextWithNoExpand); Set expectedTablesWithNoExpand = new HashSet<>(); expectedTablesWithNoExpand.add("table1"); @@ -438,7 +435,8 @@ public BitSet getDisableNereidsRules() { expectedTablesWithNoExpand); TableCollectorContext mvCollectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.MATERIALIZED_VIEW), true, true); + Sets.newHashSet(TableType.MATERIALIZED_VIEW), connectContext, + true, true); physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContext); Set expectedMvs = new HashSet<>(); expectedMvs.add("mv1"); @@ -450,8 +448,8 @@ public BitSet getDisableNereidsRules() { TableCollectorContext mvCollectorContextWithNoExpand = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.MATERIALIZED_VIEW), false, - true); + Sets.newHashSet(TableType.MATERIALIZED_VIEW), connectContext, + false, true); physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContextWithNoExpand); Set expectedMvsWithNoExpand = new HashSet<>(); expectedMvsWithNoExpand.add("mv1"); @@ -463,8 +461,8 @@ public BitSet getDisableNereidsRules() { TableCollectorContext allTableTypeWithExpand = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.values()), true, - true); + Sets.newHashSet(TableType.values()), connectContext, + true, true); physicalPlan.accept(TableCollector.INSTANCE, allTableTypeWithExpand); // when collect in plan with expand, should collect table which is expended Set expectedTablesWithExpand = new HashSet<>(); diff --git a/regression-test/suites/nereids_rules_p0/mv/with_cte/with_cte.groovy b/regression-test/suites/nereids_rules_p0/mv/with_cte/with_cte.groovy index 9e12b69fdbe4e75..ead91155ec8a51f 100644 --- a/regression-test/suites/nereids_rules_p0/mv/with_cte/with_cte.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/with_cte/with_cte.groovy @@ -195,5 +195,5 @@ suite("rewrite_with_cte") { // Test mv rewrite when cte async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") order_qt_query1_0_after "${query1_0}" -// sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" }