Skip to content

Commit

Permalink
(fix)[mtmv] merge pr
Browse files Browse the repository at this point in the history
  • Loading branch information
@seawinde
seawinde committed Dec 2, 2024
1 parent 51ad321 commit 7baaf18
Show file tree
Hide file tree
Showing 9 changed files with 987 additions and 116 deletions.
27 changes: 27 additions & 0 deletions ...e-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.jobs.joinorder.hypergraph;

/**
* This is the common base class for all
* */
public interface HyperElement {

// Get the references nodes
long getReferenceNodes();
}
11 changes: 9 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.doris.nereids.jobs.joinorder.hypergraph.edge;

import org.apache.doris.common.Pair;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
Expand All @@ -32,7 +33,7 @@
/**
* Edge in HyperGraph
*/
public abstract class Edge {
public abstract class Edge implements HyperElement {
private final int index;
private final double selectivity;

Expand All @@ -42,7 +43,9 @@ public abstract class Edge {
// added by the graph simplifier.
private final long leftRequiredNodes;
private final long rightRequiredNodes;
// The nodes needed which to prevent wrong association or l-association
private long leftExtendedNodes;
// The nodes needed which to prevent wrong association or r-association
private long rightExtendedNodes;

// record the left child edges and right child edges in origin plan tree
Expand All @@ -53,8 +56,11 @@ public abstract class Edge {
private final BitSet curOperatorEdges = new BitSet();
// record all sub nodes behind in this operator. It's T function in paper
private final long subTreeNodes;

// The edges which prevents association or l-association when join edge
// and prevents push down or pull up when filter edge in the left of edge
private final Set<JoinEdge> leftRejectEdges;
// The edges which prevents association or r-association
// and prevents push down or pull up when filter edge in the right of edge
private final Set<JoinEdge> rightRejectEdges;

/**
Expand Down Expand Up @@ -187,6 +193,7 @@ public boolean isSub(Edge edge) {
return LongBitmap.isSubset(getReferenceNodes(), otherBitmap);
}

@Override
public long getReferenceNodes() {
return LongBitmap.newBitmapUnion(leftExtendedNodes, rightExtendedNodes);
}
Expand Down
8 changes: 7 additions & 1 deletion ...e/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.doris.nereids.jobs.joinorder.hypergraph.node;

import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge;
Expand All @@ -33,7 +34,7 @@
/**
* HyperGraph Node.
*/
public class AbstractNode {
public class AbstractNode implements HyperElement {
protected final int index;
protected final List<JoinEdge> joinEdges;
protected final List<FilterEdge> filterEdges;
Expand Down Expand Up @@ -65,6 +66,11 @@ public List<Edge> getEdges() {
.build();
}

@Override
public long getReferenceNodes() {
return getNodeMap();
}

public int getIndex() {
return index;
}
Expand Down
164 changes: 134 additions & 30 deletions ...ore/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@

import org.apache.doris.common.Pair;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.ConflictRulesMaker;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
Expand All @@ -35,18 +37,23 @@
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.JoinUtils;

import com.google.common.collect.BiMap;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
Expand Down Expand Up @@ -75,9 +82,9 @@ public class HyperGraphComparator {
private final Map<Edge, List<? extends Expression>> pullUpViewExprWithEdge = new HashMap<>();
private final LogicalCompatibilityContext logicalCompatibilityContext;
// this records the slots which needs to reject null
// the key is the target join which should reject null, the value is a pair, the first value of the pair is the
// join type, the second value is also a pair which left represents the slots in the left of join that should
// reject null, right represents the slots in the right of join that should reject null.
// the key is the view join edge which should reject null, the value is a pair, the first value of the pair is the
// query join type, the second value is also a pair which left represents the slots in the left of view join that
// should reject null, right represents the slots in the right of view join that should reject null.
private final Map<JoinEdge, Pair<JoinType, Pair<Set<Slot>, Set<Slot>>>> inferredViewEdgeWithCond = new HashMap<>();
private List<JoinEdge> viewJoinEdgesAfterInferring;
private List<FilterEdge> viewFilterEdgesAfterInferring;
Expand Down Expand Up @@ -245,9 +252,17 @@ private boolean compareNodeWithExpr(StructInfoNode query, StructInfoNode view) {
}
int size = queryExprSetList.size();
for (int i = 0; i < size; i++) {
Set<Expression> mappingQueryExprSet = queryExprSetList.get(i).stream()
.map(logicalCompatibilityContext::getViewNodeExprFromQuery)
.collect(Collectors.toSet());
Set<Expression> queryExpressions = queryExprSetList.get(i);
Set<Expression> mappingQueryExprSet = new HashSet<>();
for (Expression queryExpression : queryExpressions) {
Optional<Expression> mappingViewExprByQueryExpr = getMappingViewExprByQueryExpr(queryExpression, query,
this.logicalCompatibilityContext,
ExpressionPosition.NODE);
if (!mappingViewExprByQueryExpr.isPresent()) {
return false;
}
mappingQueryExprSet.add(mappingViewExprByQueryExpr.get());
}
if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) {
return false;
}
Expand Down Expand Up @@ -403,33 +418,87 @@ private Map<Edge, Edge> constructQueryToViewJoinMapWithExpr() {
if (edgeMap.containsKey(entry.getValue())) {
continue;
}
Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(entry.getKey());
Expression viewExpr = getMappingViewExprByQueryExpr(entry.getKey(),
entry.getValue(),
logicalCompatibilityContext,
ExpressionPosition.JOIN_EDGE).orElse(null);
if (viewExprToEdge.containsKey(viewExpr)) {
edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr)));
}
}
return edgeMap;
}

// Such as the filter as following, their expression is same, but should be different filter edge
// Only construct edge that can mapping, the edges which can not mapping would be handled by buildComparisonRes
// LogicalJoin[569]
// |--LogicalProject[567]
// | +--LogicalFilter[566] ( predicates=(l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1))) )
// | +--LogicalJoin[565]
// | |--LogicalProject[562]
// | | +--LogicalOlapScan
// | +--LogicalProject[564]
// | +--LogicalFilter[563] ( predicates=(l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1))))
// | +--LogicalOlapScan
// +--LogicalProject[568]
// +--LogicalOlapScan
private Map<Edge, Edge> constructQueryToViewFilterMapWithExpr() {
Map<Expression, Edge> viewExprToEdge = getViewFilterEdges().stream()
.flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
.collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
Map<Expression, Edge> queryExprToEdge = getQueryFilterEdges().stream()
Multimap<Expression, Edge> viewExprToEdge = HashMultimap.create();
getViewFilterEdges().stream()
.flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
.collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
.forEach(pair -> viewExprToEdge.put(pair.key(), pair.value()));

HashMap<Edge, Edge> edgeMap = new HashMap<>();
for (Entry<Expression, Edge> entry : queryExprToEdge.entrySet()) {
if (edgeMap.containsKey(entry.getValue())) {
continue;
Multimap<Expression, Edge> queryExprToEdge = HashMultimap.create();
getQueryFilterEdges().stream()
.flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
.forEach(pair -> queryExprToEdge.put(pair.key(), pair.value()));

HashMap<Edge, Edge> queryToViewEdgeMap = new HashMap<>();
for (Entry<Expression, Collection<Edge>> entry : queryExprToEdge.asMap().entrySet()) {
Expression queryExprViewBased = null;
for (Edge queryEdge : entry.getValue()) {
queryExprViewBased = getMappingViewExprByQueryExpr(entry.getKey(),
queryEdge,
logicalCompatibilityContext,
ExpressionPosition.FILTER_EDGE).orElse(null);
if (queryExprViewBased == null) {
continue;
}
Collection<Edge> viewEdges = viewExprToEdge.get(queryExprViewBased);
if (viewEdges.isEmpty()) {
continue;
}
for (Edge viewEdge : viewEdges) {
if (!isSubTreeNodesEquals(queryEdge, viewEdge, logicalCompatibilityContext)) {
// Such as query filter edge is <{1} --FILTER-- {}> but view filter edge is
// <{0, 1} --FILTER-- {}>, though they are all
// l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1)) but they are different actually
continue;
}
queryToViewEdgeMap.put(queryEdge, viewEdge);
}
}
Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(entry.getKey());
if (viewExprToEdge.containsKey(viewExpr)) {
edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr)));
}
return queryToViewEdgeMap;
}

private static boolean isSubTreeNodesEquals(Edge queryEdge, Edge viewEdge,
LogicalCompatibilityContext logicalCompatibilityContext) {
if (!(queryEdge instanceof FilterEdge) || !(viewEdge instanceof FilterEdge)) {
return false;
}
// subTreeNodes should be equal
BiMap<Integer, Integer> queryToViewNodeIdMapping =
logicalCompatibilityContext.getQueryToViewNodeIDMapping();
List<Integer> queryNodeIndexViewBasedList = new ArrayList<>();
for (int queryNodeIndex : LongBitmap.getIterator(queryEdge.getSubTreeNodes())) {
Integer queryNodeIndexViewBased = queryToViewNodeIdMapping.get(queryNodeIndex);
if (queryNodeIndexViewBased == null) {
return false;
}
queryNodeIndexViewBasedList.add(queryNodeIndexViewBased);
}
return edgeMap;
return LongBitmap.newBitmap(queryNodeIndexViewBasedList) == viewEdge.getSubTreeNodes();
}

private void refreshViewEdges() {
Expand Down Expand Up @@ -463,17 +532,17 @@ private boolean compareEdgeWithNode(Edge query, Edge view) {
}

private boolean compareFilterEdgeWithNode(FilterEdge query, FilterEdge view) {
return rewriteQueryNodeMap(query.getReferenceNodes()) == view.getReferenceNodes();
return getViewNodesByQuery(query.getReferenceNodes()) == view.getReferenceNodes();
}

private boolean compareJoinEdgeWithNode(JoinEdge query, JoinEdge view) {
boolean res = false;
if (query.getJoinType().swap() == view.getJoinType()) {
res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getRightExtendedNodes()
&& rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getLeftExtendedNodes();
res |= getViewNodesByQuery(query.getLeftExtendedNodes()) == view.getRightExtendedNodes()
&& getViewNodesByQuery(query.getRightExtendedNodes()) == view.getLeftExtendedNodes();
}
res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getLeftExtendedNodes()
&& rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getRightExtendedNodes();
res |= getViewNodesByQuery(query.getLeftExtendedNodes()) == view.getLeftExtendedNodes()
&& getViewNodesByQuery(query.getRightExtendedNodes()) == view.getRightExtendedNodes();
return res;
}

Expand All @@ -496,8 +565,8 @@ private boolean compareJoinEdgeOrInfer(JoinEdge query, JoinEdge view) {
}

private boolean tryInferEdge(JoinEdge query, JoinEdge view) {
if (rewriteQueryNodeMap(query.getLeftRequiredNodes()) != view.getLeftRequiredNodes()
|| rewriteQueryNodeMap(query.getRightRequiredNodes()) != view.getRightRequiredNodes()) {
if (getViewNodesByQuery(query.getLeftRequiredNodes()) != view.getLeftRequiredNodes()
|| getViewNodesByQuery(query.getRightRequiredNodes()) != view.getRightRequiredNodes()) {
return false;
}
if (!query.getJoinType().equals(view.getJoinType())) {
Expand All @@ -518,7 +587,7 @@ private boolean tryInferEdge(JoinEdge query, JoinEdge view) {
return true;
}

private long rewriteQueryNodeMap(long bitmap) {
private long getViewNodesByQuery(long bitmap) {
long newBitmap = LongBitmap.newBitmap();
for (int i : LongBitmap.getIterator(bitmap)) {
int newIdx = getQueryToViewNodeIdMap().getOrDefault(i, 0);
Expand All @@ -527,14 +596,46 @@ private long rewriteQueryNodeMap(long bitmap) {
return newBitmap;
}

private Optional<Expression> getMappingViewExprByQueryExpr(Expression queryExpression,
HyperElement queryExpressionBelongedHyperElement,
LogicalCompatibilityContext context,
ExpressionPosition expressionPosition) {
Expression queryShuttledExpr;
Collection<Pair<Expression, HyperElement>> viewExpressions;
if (ExpressionPosition.JOIN_EDGE.equals(expressionPosition)) {
queryShuttledExpr = context.getQueryJoinShuttledExpr(queryExpression);
viewExpressions = context.getViewJoinExprFromQuery(queryShuttledExpr);
} else if (ExpressionPosition.FILTER_EDGE.equals(expressionPosition)) {
queryShuttledExpr = context.getQueryFilterShuttledExpr(queryExpression);
viewExpressions = context.getViewFilterExprFromQuery(queryShuttledExpr);
} else {
queryShuttledExpr = context.getQueryNodeShuttledExpr(queryExpression);
viewExpressions = context.getViewNodeExprFromQuery(queryShuttledExpr);
}
if (viewExpressions.size() == 1) {
return Optional.of(viewExpressions.iterator().next().key());
}
long queryReferenceNodes = queryExpressionBelongedHyperElement.getReferenceNodes();
long viewReferenceNodes = getViewNodesByQuery(queryReferenceNodes);
for (Pair<Expression, HyperElement> viewExpressionPair : viewExpressions) {
if (viewExpressionPair.value().getReferenceNodes() == viewReferenceNodes) {
return Optional.of(viewExpressionPair.key());
}
}
return Optional.empty();
}

private void compareJoinEdgeWithExpr(Edge query, Edge view) {
Set<? extends Expression> queryExprSet = query.getExpressionSet();
Set<? extends Expression> viewExprSet = view.getExpressionSet();

Set<Expression> exprMappedOfView = new HashSet<>();
List<Expression> residualQueryExpr = new ArrayList<>();
for (Expression queryExpr : queryExprSet) {
Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(queryExpr);
Expression viewExpr = getMappingViewExprByQueryExpr(queryExpr,
query,
logicalCompatibilityContext,
ExpressionPosition.JOIN_EDGE).orElse(null);
if (viewExprSet.contains(viewExpr)) {
exprMappedOfView.add(viewExpr);
} else {
Expand All @@ -553,7 +654,10 @@ private void compareFilterEdgeWithExpr(Edge query, Edge view) {
Set<Expression> exprMappedOfView = new HashSet<>();
List<Expression> residualQueryExpr = new ArrayList<>();
for (Expression queryExpr : queryExprSet) {
Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(queryExpr);
Expression viewExpr = getMappingViewExprByQueryExpr(queryExpr,
query,
logicalCompatibilityContext,
ExpressionPosition.FILTER_EDGE).orElse(null);
if (viewExprSet.contains(viewExpr)) {
exprMappedOfView.add(viewExpr);
} else {
Expand Down
Loading

0 comments on commit 7baaf18

Please sign in to comment.