Skip to content

Commit

Permalink
[feature] (nereids) Support join derivation when query rewrite by mat…
Browse files Browse the repository at this point in the history 
…erialized view
  • Loading branch information
@seawinde
seawinde committed Dec 29, 2023
1 parent d2dc12b commit 4122e0a
Showing 1 changed file with 109 additions and 43 deletions.
152 changes: 109 additions & 43 deletions ...main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,10 @@
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.TypeUtils;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -62,6 +64,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -149,7 +152,7 @@ protected List<Plan> rewrite(Plan queryPlan, CascadesContext cascadesContext) {
queryStructInfo.addPredicates(pulledUpExpressions);
}
SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo,
queryToViewSlotMapping);
queryToViewSlotMapping, comparisonResult);
// Can not compensate, bail out
if (compensatePredicates.isEmpty()) {
logger.debug(currentClassName + " predicate compensate fail so continue");
Expand Down Expand Up @@ -385,34 +388,89 @@ protected Expression rewriteExpression(
protected SplitPredicate predicatesCompensate(
StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping queryToViewSlotMapping
SlotMapping queryToViewSlotMapping,
ComparisonResult comparisonResult
) {
// viewEquivalenceClass to query based
SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse();
final Set<Expression> equalCompensateConjunctions = compensateEquivalence(
queryStructInfo,
viewStructInfo,
viewToQuerySlotMapping,
comparisonResult);
// range compensate
final Set<Expression> rangeCompensatePredicates = compensateRangePredicate(
queryStructInfo,
viewStructInfo,
viewToQuerySlotMapping,
comparisonResult);
// residual compensate
final Set<Expression> residualCompensatePredicates = compensateResidualPredicate(
queryStructInfo,
viewStructInfo,
viewToQuerySlotMapping,
comparisonResult);
// if the join type in query and mv plan is different, we should check and add filter on mv to make
// the mv join type is accord with query
Set<Set<Slot>> viewNoNullableSlot = comparisonResult.getViewNoNullableSlot();
// extract
if (!viewNoNullableSlot.isEmpty()) {
Set<Expression> queryUsedRejectNullSlotsViewBased = ExpressionUtils.extractConjunction(
queryStructInfo.getSplitPredicate().getRangePredicate()).stream()
.map(expression -> {
if (TypeUtils.isNotNull(expression).isPresent()) {
return ImmutableList.of(TypeUtils.isNotNull(expression).get());
} else {
Set<Object> slotRefrenceSet =
expression.collectToSet(expr -> expr instanceof SlotReference);
if (slotRefrenceSet.size() != 1) {
return null;
}
return slotRefrenceSet.iterator().next();
}
})
.filter(Objects::nonNull)
.map(expr -> ExpressionUtils.replace((Expression) expr,
queryToViewSlotMapping.toSlotReferenceMap()))
.collect(Collectors.toSet());

if (viewNoNullableSlot.stream().anyMatch(
set -> Sets.intersection(set, queryUsedRejectNullSlotsViewBased).isEmpty())) {
return SplitPredicate.empty();
}
}
return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions),
rangeCompensatePredicates.isEmpty() ? BooleanLiteral.of(true)
: ExpressionUtils.and(rangeCompensatePredicates),
residualCompensatePredicates.isEmpty() ? BooleanLiteral.of(true)
: ExpressionUtils.and(residualCompensatePredicates));
}

protected Set<Expression> compensateEquivalence(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass();
EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass();
// viewEquivalenceClass to query based
Map<SlotReference, SlotReference> viewToQuerySlotMapping = queryToViewSlotMapping.inverse()
.toSlotReferenceMap();
EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMapping);
Map<SlotReference, SlotReference> viewToQuerySlotMap = viewToQuerySlotMapping.toSlotReferenceMap();
EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMap);
if (viewEquivalenceClassQueryBased == null) {
logger.info(currentClassName + " permute view equivalence class by query fail so return empty");
return SplitPredicate.empty();
return ImmutableSet.of();
}
final List<Expression> equalCompensateConjunctions = new ArrayList<>();
final Set<Expression> equalCompensateConjunctions = new HashSet<>();
if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) {
equalCompensateConjunctions.add(BooleanLiteral.of(true));
}
if (queryEquivalenceClass.isEmpty()
&& !viewEquivalenceClass.isEmpty()) {
logger.info(currentClassName + " view has equivalence class but query not so return empty");
return SplitPredicate.empty();
return ImmutableSet.of();
}
EquivalenceClassSetMapping queryToViewEquivalenceMapping =
EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased);
// can not map all target equivalence class, can not compensate
if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size()
< viewEquivalenceClass.getEquivalenceSetList().size()) {
logger.info(currentClassName + " view has more equivalence than query so return empty");
return SplitPredicate.empty();
return ImmutableSet.of();
}
// do equal compensate
Set<Set<SlotReference>> mappedQueryEquivalenceSet =
Expand Down Expand Up @@ -441,36 +499,20 @@ protected SplitPredicate predicatesCompensate(
}
}
);
// TODO range predicates and residual predicates compensate, Simplify implementation.
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();

// range compensate
List<Expression> rangeCompensate = new ArrayList<>();
Expression queryRangePredicate = querySplitPredicate.getRangePredicate();
Expression viewRangePredicate = viewSplitPredicate.getRangePredicate();
Expression viewRangePredicateQueryBased =
ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping);
return equalCompensateConjunctions;
}

Set<Expression> queryRangeSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate));
Set<Expression> viewRangeQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased));
// query range predicate can not contain all view range predicate when view have range predicate, bail out
if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE)
&& !queryRangeSet.containsAll(viewRangeQueryBasedSet)) {
logger.info(currentClassName + " query range predicate set can not contains all view range predicate");
return SplitPredicate.empty();
}
queryRangeSet.removeAll(viewRangeQueryBasedSet);
rangeCompensate.addAll(queryRangeSet);

// residual compensate
List<Expression> residualCompensate = new ArrayList<>();
protected Set<Expression> compensateResidualPredicate(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();
Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate();
Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate();
Expression viewResidualPredicateQueryBased =
ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping);
ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping.toSlotReferenceMap());
Set<Expression> queryResidualSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate));
Set<Expression> viewResidualQueryBasedSet =
Expand All @@ -481,14 +523,38 @@ protected SplitPredicate predicatesCompensate(
&& !queryResidualSet.containsAll(viewResidualQueryBasedSet)) {
logger.info(
currentClassName + " query residual predicate set can not contains all view residual predicate");
return SplitPredicate.empty();
return ImmutableSet.of();
}
queryResidualSet.removeAll(viewResidualQueryBasedSet);
residualCompensate.addAll(queryResidualSet);
return queryResidualSet;
}

return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions),
rangeCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(rangeCompensate),
residualCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(residualCompensate));

protected Set<Expression> compensateRangePredicate(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
// TODO range predicates and residual predicates compensate, Simplify implementation.
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();

Expression queryRangePredicate = querySplitPredicate.getRangePredicate();
Expression viewRangePredicate = viewSplitPredicate.getRangePredicate();
Expression viewRangePredicateQueryBased =
ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping.toSlotReferenceMap());

Set<Expression> queryRangeSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate));
Set<Expression> viewRangeQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased));
// query range predicate can not contain all view range predicate when view have range predicate, bail out
if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE)
&& !queryRangeSet.containsAll(viewRangeQueryBasedSet)) {
logger.info(currentClassName + " query range predicate set can not contains all view range predicate");
return ImmutableSet.of();
}
queryRangeSet.removeAll(viewRangeQueryBasedSet);
return queryRangeSet;
}

/**
Expand Down

0 comments on commit 4122e0a

Please sign in to comment.