From 4122e0a1d9fdd1eae2e779e5fd60bafd38167bab Mon Sep 17 00:00:00 2001 From: seawinde Date: Fri, 29 Dec 2023 18:12:14 +0800 Subject: [PATCH] [feature] (nereids) Support join derivation when query rewrite by materialized view --- .../mv/AbstractMaterializedViewRule.java | 152 +++++++++++++----- 1 file changed, 109 insertions(+), 43 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index c63e2d85af35ab8..dc403430ab01653 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -50,8 +50,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.TypeUtils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -62,6 +64,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -149,7 +152,7 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { queryStructInfo.addPredicates(pulledUpExpressions); } SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo, - queryToViewSlotMapping); + queryToViewSlotMapping, comparisonResult); // Can not compensate, bail out if (compensatePredicates.isEmpty()) { logger.debug(currentClassName + " predicate compensate fail so continue"); @@ -385,34 +388,89 @@ protected Expression rewriteExpression( protected SplitPredicate predicatesCompensate( StructInfo queryStructInfo, StructInfo viewStructInfo, - SlotMapping queryToViewSlotMapping + SlotMapping queryToViewSlotMapping, + ComparisonResult comparisonResult ) { + // viewEquivalenceClass to query based + SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse(); + final Set equalCompensateConjunctions = compensateEquivalence( + queryStructInfo, + viewStructInfo, + viewToQuerySlotMapping, + comparisonResult); + // range compensate + final Set rangeCompensatePredicates = compensateRangePredicate( + queryStructInfo, + viewStructInfo, + viewToQuerySlotMapping, + comparisonResult); + // residual compensate + final Set residualCompensatePredicates = compensateResidualPredicate( + queryStructInfo, + viewStructInfo, + viewToQuerySlotMapping, + comparisonResult); + // if the join type in query and mv plan is different, we should check and add filter on mv to make + // the mv join type is accord with query + Set> viewNoNullableSlot = comparisonResult.getViewNoNullableSlot(); + // extract + if (!viewNoNullableSlot.isEmpty()) { + Set queryUsedRejectNullSlotsViewBased = ExpressionUtils.extractConjunction( + queryStructInfo.getSplitPredicate().getRangePredicate()).stream() + .map(expression -> { + if (TypeUtils.isNotNull(expression).isPresent()) { + return ImmutableList.of(TypeUtils.isNotNull(expression).get()); + } else { + Set slotRefrenceSet = + expression.collectToSet(expr -> expr instanceof SlotReference); + if (slotRefrenceSet.size() != 1) { + return null; + } + return slotRefrenceSet.iterator().next(); + } + }) + .filter(Objects::nonNull) + .map(expr -> ExpressionUtils.replace((Expression) expr, + queryToViewSlotMapping.toSlotReferenceMap())) + .collect(Collectors.toSet()); + + if (viewNoNullableSlot.stream().anyMatch( + set -> Sets.intersection(set, queryUsedRejectNullSlotsViewBased).isEmpty())) { + return SplitPredicate.empty(); + } + } + return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions), + rangeCompensatePredicates.isEmpty() ? BooleanLiteral.of(true) + : ExpressionUtils.and(rangeCompensatePredicates), + residualCompensatePredicates.isEmpty() ? BooleanLiteral.of(true) + : ExpressionUtils.and(residualCompensatePredicates)); + } + + protected Set compensateEquivalence(StructInfo queryStructInfo, + StructInfo viewStructInfo, + SlotMapping viewToQuerySlotMapping, + ComparisonResult comparisonResult) { EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass(); EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass(); - // viewEquivalenceClass to query based - Map viewToQuerySlotMapping = queryToViewSlotMapping.inverse() - .toSlotReferenceMap(); - EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMapping); + Map viewToQuerySlotMap = viewToQuerySlotMapping.toSlotReferenceMap(); + EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMap); if (viewEquivalenceClassQueryBased == null) { - logger.info(currentClassName + " permute view equivalence class by query fail so return empty"); - return SplitPredicate.empty(); + return ImmutableSet.of(); } - final List equalCompensateConjunctions = new ArrayList<>(); + final Set equalCompensateConjunctions = new HashSet<>(); if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) { equalCompensateConjunctions.add(BooleanLiteral.of(true)); } if (queryEquivalenceClass.isEmpty() && !viewEquivalenceClass.isEmpty()) { - logger.info(currentClassName + " view has equivalence class but query not so return empty"); - return SplitPredicate.empty(); + return ImmutableSet.of(); } EquivalenceClassSetMapping queryToViewEquivalenceMapping = EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased); // can not map all target equivalence class, can not compensate if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size() < viewEquivalenceClass.getEquivalenceSetList().size()) { - logger.info(currentClassName + " view has more equivalence than query so return empty"); - return SplitPredicate.empty(); + return ImmutableSet.of(); } // do equal compensate Set> mappedQueryEquivalenceSet = @@ -441,36 +499,20 @@ protected SplitPredicate predicatesCompensate( } } ); - // TODO range predicates and residual predicates compensate, Simplify implementation. - SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate(); - SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate(); - - // range compensate - List rangeCompensate = new ArrayList<>(); - Expression queryRangePredicate = querySplitPredicate.getRangePredicate(); - Expression viewRangePredicate = viewSplitPredicate.getRangePredicate(); - Expression viewRangePredicateQueryBased = - ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping); + return equalCompensateConjunctions; + } - Set queryRangeSet = - Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate)); - Set viewRangeQueryBasedSet = - Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased)); - // query range predicate can not contain all view range predicate when view have range predicate, bail out - if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE) - && !queryRangeSet.containsAll(viewRangeQueryBasedSet)) { - logger.info(currentClassName + " query range predicate set can not contains all view range predicate"); - return SplitPredicate.empty(); - } - queryRangeSet.removeAll(viewRangeQueryBasedSet); - rangeCompensate.addAll(queryRangeSet); - // residual compensate - List residualCompensate = new ArrayList<>(); + protected Set compensateResidualPredicate(StructInfo queryStructInfo, + StructInfo viewStructInfo, + SlotMapping viewToQuerySlotMapping, + ComparisonResult comparisonResult) { + SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate(); + SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate(); Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate(); Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate(); Expression viewResidualPredicateQueryBased = - ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping); + ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping.toSlotReferenceMap()); Set queryResidualSet = Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate)); Set viewResidualQueryBasedSet = @@ -481,14 +523,38 @@ protected SplitPredicate predicatesCompensate( && !queryResidualSet.containsAll(viewResidualQueryBasedSet)) { logger.info( currentClassName + " query residual predicate set can not contains all view residual predicate"); - return SplitPredicate.empty(); + return ImmutableSet.of(); } queryResidualSet.removeAll(viewResidualQueryBasedSet); - residualCompensate.addAll(queryResidualSet); + return queryResidualSet; + } - return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions), - rangeCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(rangeCompensate), - residualCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(residualCompensate)); + + protected Set compensateRangePredicate(StructInfo queryStructInfo, + StructInfo viewStructInfo, + SlotMapping viewToQuerySlotMapping, + ComparisonResult comparisonResult) { + // TODO range predicates and residual predicates compensate, Simplify implementation. + SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate(); + SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate(); + + Expression queryRangePredicate = querySplitPredicate.getRangePredicate(); + Expression viewRangePredicate = viewSplitPredicate.getRangePredicate(); + Expression viewRangePredicateQueryBased = + ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping.toSlotReferenceMap()); + + Set queryRangeSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate)); + Set viewRangeQueryBasedSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased)); + // query range predicate can not contain all view range predicate when view have range predicate, bail out + if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE) + && !queryRangeSet.containsAll(viewRangeQueryBasedSet)) { + logger.info(currentClassName + " query range predicate set can not contains all view range predicate"); + return ImmutableSet.of(); + } + queryRangeSet.removeAll(viewRangeQueryBasedSet); + return queryRangeSet; } /**