Skip to content

Commit

Permalink
[fix](mtmv) Fix rewritten by mv fail when filter expression in query …
Browse files Browse the repository at this point in the history 
…has alias but actual same
  • Loading branch information
@seawinde
seawinde committed Nov 27, 2024
1 parent aca0eaa commit f2fe5ad
Show file tree
Hide file tree
Showing 7 changed files with 269 additions and 137 deletions.
4 changes: 2 additions & 2 deletions ...main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,8 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
() -> String.format("matchMode is %s", matchMode));
return rewriteResults;
}
List<RelationMapping> queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(),
viewStructInfo.getRelations());
List<RelationMapping> queryToViewTableMappings = RelationMapping.EMPTY_INSTANCE.generate(
queryStructInfo.getRelations(), viewStructInfo.getRelations());
// if any relation in query and view can not map, bail out.
if (queryToViewTableMappings == null) {
materializationContext.recordFailReason(queryStructInfo,
Expand Down
19 changes: 16 additions & 3 deletions ...ore/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.JoinUtils;

import com.google.common.collect.BiMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
Expand Down Expand Up @@ -75,9 +76,9 @@ public class HyperGraphComparator {
private final Map<Edge, List<? extends Expression>> pullUpViewExprWithEdge = new HashMap<>();
private final LogicalCompatibilityContext logicalCompatibilityContext;
// this records the slots which needs to reject null
// the key is the target join which should reject null, the value is a pair, the first value of the pair is the
// join type, the second value is also a pair which left represents the slots in the left of join that should
// reject null, right represents the slots in the right of join that should reject null.
// the key is the view join edge which should reject null, the value is a pair, the first value of the pair is the
// query join type, the second value is also a pair which left represents the slots in the left of view join that
// should reject null, right represents the slots in the right of view join that should reject null.
private final Map<JoinEdge, Pair<JoinType, Pair<Set<Slot>, Set<Slot>>>> inferredViewEdgeWithCond = new HashMap<>();
private List<JoinEdge> viewJoinEdgesAfterInferring;
private List<FilterEdge> viewFilterEdgesAfterInferring;
Expand All @@ -104,6 +105,18 @@ public HyperGraphComparator(HyperGraph queryHyperGraph, HyperGraph viewHyperGrap
*/
public static ComparisonResult isLogicCompatible(HyperGraph queryHG, HyperGraph viewHG,
LogicalCompatibilityContext ctx) {
ComparisonResult logicCompatible;
if (ctx.getQueryToViewFilterEdgeExpressionMappingList().size() > 1) {
for (BiMap<Expression, Expression> expressionBiMap : ctx.getQueryToViewFilterEdgeExpressionMappingList()) {
LogicalCompatibilityContext eachCtx = ctx.withQueryToViewFilterEdgeExpressionMapping(expressionBiMap);
logicCompatible =
new HyperGraphComparator(queryHG, viewHG, eachCtx).isLogicCompatible();
if (logicCompatible.isInvalid()) {
continue;
}
return logicCompatible;
}
}
return new HyperGraphComparator(queryHG, viewHG, ctx).isLogicCompatible();
}

Expand Down
96 changes: 44 additions & 52 deletions .../main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,24 +20,21 @@
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.Mapping.MappedRelation;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.plans.ObjectId;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;

import com.google.common.base.Suppliers;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;

Expand All @@ -51,8 +48,27 @@ public class LogicalCompatibilityContext {
private final Supplier<BiMap<Expression, Expression>> queryToViewJoinEdgeExpressionMappingSupplier;
private final Supplier<BiMap<Expression, Expression>> queryToViewNodeExpressionMappingSupplier;
private final Supplier<BiMap<Expression, Expression>> queryToViewFilterEdgeExpressionMappingSupplier;
@Deprecated
private BiMap<Expression, Expression> queryToViewAllExpressionMapping;
private final List<BiMap<Expression, Expression>> queryToViewFilterEdgeExpressionMappingList;

private LogicalCompatibilityContext(
BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping,
BiMap<Integer, Integer> queryToViewNodeIDMapping,
ObjectId planNodeId,
BiMap<Expression, Expression> queryToViewJoinEdgeExpressionMapping,
BiMap<Expression, Expression> queryToViewNodeExpressionMapping,
BiMap<Expression, Expression> queryToViewFilterEdgeExpressionMapping,
List<BiMap<Expression, Expression>> queryToViewFilterEdgeExpressionMappingList) {
this.queryToViewNodeMapping = queryToViewNodeMapping;
this.queryToViewNodeIDMapping = queryToViewNodeIDMapping;
this.planNodeId = planNodeId;
this.queryToViewJoinEdgeExpressionMappingSupplier =
Suppliers.memoize(() -> queryToViewJoinEdgeExpressionMapping);
this.queryToViewNodeExpressionMappingSupplier =
Suppliers.memoize(() -> queryToViewNodeExpressionMapping);
this.queryToViewFilterEdgeExpressionMappingSupplier =
Suppliers.memoize(() -> queryToViewFilterEdgeExpressionMapping);
this.queryToViewFilterEdgeExpressionMappingList = queryToViewFilterEdgeExpressionMappingList;
}

/**
* LogicalCompatibilityContext
Expand All @@ -62,17 +78,17 @@ private LogicalCompatibilityContext(BiMap<StructInfoNode, StructInfoNode> queryT
StructInfo viewStructInfo) {

this.queryToViewJoinEdgeExpressionMappingSupplier =
Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping,
Suppliers.memoize(() -> ExpressionMapping.generateExpressionMappingFirst(viewToQuerySlotMapping,
queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE),
viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE)));

this.queryToViewNodeExpressionMappingSupplier =
Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping,
Suppliers.memoize(() -> ExpressionMapping.generateExpressionMappingFirst(viewToQuerySlotMapping,
queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE),
viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE)));

this.queryToViewFilterEdgeExpressionMappingSupplier =
Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping,
Suppliers.memoize(() -> ExpressionMapping.generateExpressionMappingFirst(viewToQuerySlotMapping,
queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE),
viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE)));

Expand All @@ -82,6 +98,11 @@ private LogicalCompatibilityContext(BiMap<StructInfoNode, StructInfoNode> queryT

this.planNodeId = queryStructInfo.getTopPlan().getGroupExpression()
.map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1));

this.queryToViewFilterEdgeExpressionMappingList = ExpressionMapping.EMPTY_INSTANCE
.generateExpressionMappingCombine(viewToQuerySlotMapping,
queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE),
viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE));
}

public BiMap<StructInfoNode, StructInfoNode> getQueryToViewNodeMapping() {
Expand Down Expand Up @@ -134,50 +155,21 @@ public static LogicalCompatibilityContext from(RelationMapping relationMapping,
viewStructInfo);
}

private static BiMap<Expression, Expression> generateExpressionMapping(
Map<SlotReference, SlotReference> viewToQuerySlotMapping,
Map<Expression, Expression> queryShuttledExprToExprMap,
Map<Expression, Expression> viewShuttledExprToExprMap) {
final Map<Expression, Expression> viewEdgeToConjunctsMapQueryBased = new HashMap<>();
BiMap<Expression, Expression> queryToViewEdgeMapping = HashBiMap.create();
if (queryShuttledExprToExprMap == null || viewShuttledExprToExprMap == null
|| queryShuttledExprToExprMap.isEmpty() || viewShuttledExprToExprMap.isEmpty()) {
return queryToViewEdgeMapping;
}
viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> {
viewEdgeToConjunctsMapQueryBased.put(
orderSlotAsc(ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping)), expr);
});
queryShuttledExprToExprMap.forEach((exprSet, edge) -> {
Expression viewExpr = viewEdgeToConjunctsMapQueryBased.get(orderSlotAsc(exprSet));
if (viewExpr != null) {
queryToViewEdgeMapping.put(edge, viewExpr);
}
});
return queryToViewEdgeMapping;
}

private static Expression orderSlotAsc(Expression expression) {
return expression.accept(ExpressionSlotOrder.INSTANCE, null);
/**
* Construct LogicalCompatibilityContext with queryToViewFilterEdgeExpressionMapping
*/
public LogicalCompatibilityContext withQueryToViewFilterEdgeExpressionMapping(
BiMap<Expression, Expression> queryToViewFilterEdgeExpressionMapping) {
return new LogicalCompatibilityContext(this.queryToViewNodeMapping,
this.queryToViewNodeIDMapping,
this.planNodeId,
this.queryToViewJoinEdgeExpressionMappingSupplier.get(),
this.queryToViewNodeExpressionMappingSupplier.get(), queryToViewFilterEdgeExpressionMapping,
this.queryToViewFilterEdgeExpressionMappingList);
}

private static final class ExpressionSlotOrder extends DefaultExpressionRewriter<Void> {
public static final ExpressionSlotOrder INSTANCE = new ExpressionSlotOrder();

@Override
public Expression visitEqualTo(EqualTo equalTo, Void context) {
if (!(equalTo.getArgument(0) instanceof NamedExpression)
|| !(equalTo.getArgument(1) instanceof NamedExpression)) {
return equalTo;
}
NamedExpression left = (NamedExpression) equalTo.getArgument(0);
NamedExpression right = (NamedExpression) equalTo.getArgument(1);
if (right.getExprId().asInt() < left.getExprId().asInt()) {
return new EqualTo(right, left);
} else {
return equalTo;
}
}
public List<BiMap<Expression, Expression>> getQueryToViewFilterEdgeExpressionMappingList() {
return queryToViewFilterEdgeExpressionMappingList;
}

@Override
Expand Down
20 changes: 12 additions & 8 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,10 @@
import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer;
import org.apache.doris.nereids.util.ExpressionUtils;

import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;

import java.util.ArrayList;
Expand Down Expand Up @@ -110,8 +112,10 @@ public class StructInfo {
private SplitPredicate splitPredicate;
private EquivalenceClass equivalenceClass;
// Key is the expression shuttled and the value is the origin expression
// Sometimes origin expressions are different and shuttled expression is same
// Such as origin expressions are l_partkey#0 > 1 and l_partkey#10 > 1 and shuttled expression is l_partkey#10 > 1
// this is for building LogicalCompatibilityContext later.
private final Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap;
private final Map<ExpressionPosition, Multimap<Expression, Expression>> shuttledExpressionsToExpressionsMap;
// Record the exprId and the corresponding expr map, this is used by expression shuttled
private final Map<ExprId, Expression> namedExprIdAndExprMapping;
private final List<? extends Expression> planOutputShuttledExpressions;
Expand All @@ -123,7 +127,7 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG
Plan bottomPlan, List<CatalogRelation> relations,
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap,
@Nullable Predicates predicates,
Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap,
Map<ExpressionPosition, Multimap<Expression, Expression>> shuttledExpressionsToExpressionsMap,
Map<ExprId, Expression> namedExprIdAndExprMapping,
BitSet tableIdSet,
SplitPredicate splitPredicate,
Expand Down Expand Up @@ -168,7 +172,7 @@ public StructInfo withTableBitSet(BitSet tableBitSet) {

private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
Plan topPlan,
Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap,
Map<ExpressionPosition, Multimap<Expression, Expression>> shuttledExpressionsToExpressionsMap,
Map<ExprId, Expression> namedExprIdAndExprMapping,
List<CatalogRelation> relations,
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap,
Expand Down Expand Up @@ -311,7 +315,7 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable
// collect struct info fromGraph
List<CatalogRelation> relationList = new ArrayList<>();
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap = new LinkedHashMap<>();
Map<ExpressionPosition, Map<Expression, Expression>> shuttledHashConjunctsToConjunctsMap =
Map<ExpressionPosition, Multimap<Expression, Expression>> shuttledHashConjunctsToConjunctsMap =
new LinkedHashMap<>();
Map<ExprId, Expression> namedExprIdAndExprMapping = new LinkedHashMap<>();
BitSet tableBitSet = new BitSet();
Expand Down Expand Up @@ -401,18 +405,18 @@ public Map<RelationId, StructInfoNode> getRelationIdStructInfoNodeMap() {
return relationIdStructInfoNodeMap;
}

public Map<ExpressionPosition, Map<Expression, Expression>> getShuttledExpressionsToExpressionsMap() {
public Map<ExpressionPosition, Multimap<Expression, Expression>> getShuttledExpressionsToExpressionsMap() {
return shuttledExpressionsToExpressionsMap;
}

private static void putShuttledExpressionsToExpressionsMap(
Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap,
Map<ExpressionPosition, Multimap<Expression, Expression>> shuttledExpressionsToExpressionsMap,
ExpressionPosition expressionPosition,
Expression key, Expression value) {
Map<Expression, Expression> expressionExpressionMap = shuttledExpressionsToExpressionsMap.get(
Multimap<Expression, Expression> expressionExpressionMap = shuttledExpressionsToExpressionsMap.get(
expressionPosition);
if (expressionExpressionMap == null) {
expressionExpressionMap = new LinkedHashMap<>();
expressionExpressionMap = HashMultimap.create();
shuttledExpressionsToExpressionsMap.put(expressionPosition, expressionExpressionMap);
}
expressionExpressionMap.put(key, value);
Expand Down
Loading

0 comments on commit f2fe5ad

Please sign in to comment.