diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 5806bda8e4832a..bdb5b5e56bdd37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -280,16 +280,16 @@ private Plan planWithoutLock( } int nth = cascadesContext.getConnectContext().getSessionVariable().getNthOptimizedPlan(); PhysicalPlan physicalPlan = chooseNthPlan(getRoot(), requireProperties, nth); - double cost1 = cost; - - - cascadesContext.plan = cascadesContext.cteplan; - optimize(); - PhysicalPlan physicalPlan2 = chooseNthPlan(getRoot(), requireProperties, nth); - double cost2 = cost; - - if (cost2 < cost1) { - physicalPlan = physicalPlan2; + if (cascadesContext.cteplan != null) { + double cost1 = cost; + cascadesContext.plan = cascadesContext.cteplan; + optimize(); + PhysicalPlan physicalPlan2 = chooseNthPlan(getRoot(), requireProperties, nth); + double cost2 = cost; + + if (cost2 < cost1) { + physicalPlan = physicalPlan2; + } } physicalPlan = postProcess(physicalPlan); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/CteExtractor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/CteExtractor.java index 8205a6054fd1af..2bce261a5c9d8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/CteExtractor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/CteExtractor.java @@ -2,15 +2,18 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.analysis.CheckAfterRewrite; +import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule; import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; import org.apache.doris.nereids.rules.exploration.mv.MaterializationContext; +import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.rules.exploration.mv.StructInfo; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.copier.DeepCopierContext; import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier; import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Plan; @@ -18,8 +21,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.BitSet; @@ -70,8 +75,8 @@ public List extract() { Plan target1 = targetPlans.get(1); Optional optStruct1 = MaterializationContext.constructStructInfo(target1, target1, cascadesContext, new BitSet()); StructInfo queryStructInfo = optStruct1.get(); - List queryToViewTableMappings = RelationMapping.generate(optStruct0.get().getRelations(), - optStruct1.get().getRelations()); + List queryToViewTableMappings = RelationMapping.generate(optStruct1.get().getRelations(), + optStruct0.get().getRelations()); RelationMapping queryToViewTableMapping = queryToViewTableMappings.get(0); SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse(); @@ -80,6 +85,17 @@ public List extract() { ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext); if (!comparisonResult.isInvalid()) { + SplitPredicate compensatePredicates = AbstractMaterializedViewRule.predicatesCompensate(queryStructInfo, viewStructInfo, + viewToQuerySlotMapping, comparisonResult, cascadesContext); + if (compensatePredicates.isInvalid()) { + throw new RuntimeException("Unsupported comparison result: " + comparisonResult); + } + // List rewriteCompensatePredicates = AbstractMaterializedViewRule.rewriteExpression(compensatePredicates.toList(), + // target1, materializationContext.getShuttledExprToScanExprMapping(), + // viewToQuerySlotMapping, queryStructInfo.getTableBitSet()); + Expression origin = compensatePredicates.getRangePredicate(); + List originList = Lists.newArrayList(origin); + List rewrittens = ExpressionUtils.shuttleExpressionWithLineage(originList, target1, Sets.newHashSet(), Sets.newHashSet(), new BitSet()); CTEId cteId = StatementScopeIdGenerator.newCTEId(); Plan producerBody = LogicalPlanDeepCopier.INSTANCE .deepCopy((LogicalPlan) target0, new DeepCopierContext()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java index 28f1ed405e82ef..b04fa70cec1e5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java @@ -42,6 +42,15 @@ import javax.annotation.Nullable; /** + * agg(group by[A])->filter(A>1)->scan(T) + * + * expressions:[ + * set1[A], + * set2(A>1) + * ] + * + * filter(A>1)->agg(group by[A])->scan(T) + * * HyperGraph Node. */ public class StructInfoNode extends AbstractNode { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index e6f384502d620f..5fd40b96b727d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -559,7 +559,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf * @param targetExpressionMapping target expression mapping, if finding the expression in key set of the mapping * then use the corresponding value of mapping to replace it */ - protected List rewriteExpression(List sourceExpressionsToWrite, Plan sourcePlan, + public static List rewriteExpression(List sourceExpressionsToWrite, Plan sourcePlan, ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, BitSet sourcePlanBitSet) { // Firstly, rewrite the target expression using source with inverse mapping // then try to use the target expression to represent the query. if any of source expressions @@ -607,7 +607,7 @@ protected List rewriteExpression(List sourceEx * then we can add payload['issue']['number']#20 -> element_at(element_at(payload#10, 'issue'), 'number') * to targetToTargetReplacementMappingQueryBased * */ - private void extendMappingByVariant(Set queryVariants, + public static void extendMappingByVariant(Set queryVariants, Map targetToTargetReplacementMappingQueryBased) { if (queryVariants.isEmpty()) { return; @@ -709,7 +709,7 @@ public static NamedExpression normalizeExpression( * For another example as following: * predicate a = b in mv, and a = b and c = d in query, the compensatory predicate is c = d */ - protected SplitPredicate predicatesCompensate( + public static SplitPredicate predicatesCompensate( StructInfo queryStructInfo, StructInfo viewStructInfo, SlotMapping viewToQuerySlotMapping, @@ -791,7 +791,7 @@ protected SplitPredicate predicatesCompensate( /** * Check the queryPredicates contains the required nullable slot */ - private boolean containsNullRejectSlot(Set> requireNoNullableViewSlot, + public static boolean containsNullRejectSlot(Set> requireNoNullableViewSlot, Set queryPredicates, SlotMapping queryToViewMapping, StructInfo queryStructInfo, diff --git a/regression-test/suites/query_p0/common_cte/common_cte.groovy b/regression-test/suites/query_p0/common_cte/common_cte.groovy index a97c214c0c488c..4d593d027c270d 100644 --- a/regression-test/suites/query_p0/common_cte/common_cte.groovy +++ b/regression-test/suites/query_p0/common_cte/common_cte.groovy @@ -3,23 +3,57 @@ // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance +// License); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("common_cte") { - qt_1 " explain physical plan select * from (select count() a from lineitem union all select count() b from lineitem) T;" +suite(common_cte) { + sql set enable_common_cte=true; + qt_1 explain verbose select * from (select count() a from lineitem union all select count() b from lineitem) T; } + +explain verbose select * from (select count() a from lineitem union all select count() b from lineitem where orderkey > 0) T + + set enable_common_cte=false; explain verbose with x as (select count() a from lineitem) -select * from (select * from x union all select * from x) T; \ No newline at end of file +select * from (select * from x union all select * from x) T; + + +CREATE MATERIALIZED VIEW vlo +BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL +DISTRIBUTED BY RANDOM BUCKETS 2 +PROPERTIES ('replication_num' = '1') +AS select lineitem.L_LINENUMBER, t2.O_CUSTKEY, t2.O_ORDERSTATUS + from lineitem + inner join + (select * from orders where O_ORDERSTATUS = 'o') t2 + on lineitem.L_ORDERKEY = t2.O_ORDERKEY ; + + +select lineitem.L_LINENUMBER +from lineitem +inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY +where orders.O_ORDERSTATUS = 'o' + +------------- +CREATE MATERIALIZED VIEW v_filter +BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL +DISTRIBUTED BY RANDOM BUCKETS 2 +PROPERTIES ('replication_num' = '1') +AS select o_custkey, o_orderkey + from orders where O_ORDERSTATUS = 'o' or O_ORDERSTATUS = 'x' ; + +explain +select o_custkey, o_orderkey + from orders where O_ORDERSTATUS = 'o'; \ No newline at end of file