Skip to content

Commit

Permalink
fix inner join comment
Browse files Browse the repository at this point in the history
  • Loading branch information
@seawinde
seawinde committed Dec 6, 2023
1 parent d2497b3 commit 2f8e70b
Show file tree
Hide file tree
Showing 6 changed files with 69 additions and 75 deletions.
13 changes: 8 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@
import java.util.List;
import java.util.stream.Collectors;

/**The cache for materialized view cache */
/**
* The cache for materialized view cache
*/
public class MVCache {

// the materialized view plan which should be optimized by the same rules to query
Expand Down Expand Up @@ -67,12 +69,13 @@ public static MVCache from(MTMV mtmv, ConnectContext connectContext) {
new OriginStatement(mtmv.getQuerySql(), 0));
NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext);

planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN);
Plan mvAnalyzedPlan = planner.getAnalyzedPlan();
Plan mvRewrittenPlan = planner.getRewrittenPlan();
Plan mvRewrittenPlan =
planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN);
Plan mvPlan = mvRewrittenPlan instanceof LogicalResultSink
? (Plan) ((LogicalResultSink) mvRewrittenPlan).child() : mvRewrittenPlan;
List<NamedExpression> mvOutputExpressions = mvAnalyzedPlan.getExpressions().stream()
// use rewritten plan output expression currently, if expression rewrite fail,
// consider to use the analyzed plan for output expressions only
List<NamedExpression> mvOutputExpressions = mvRewrittenPlan.getExpressions().stream()
.map(NamedExpression.class::cast)
.collect(Collectors.toList());
return new MVCache(mvPlan, mvOutputExpressions);
Expand Down
15 changes: 9 additions & 6 deletions .../java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
* This is responsible for common join rewriting
*/
public abstract class AbstractMaterializedViewJoinRule extends AbstractMaterializedViewRule {
private static final HashSet<JoinType> SUPPORTED_JOIN_TYPE_SET =
Sets.newHashSet(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN);

@Override
protected Plan rewriteQueryByView(MatchMode matchMode,
Expand Down Expand Up @@ -72,21 +74,22 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
tempRewritedPlan);
}

// Check join is whether valid or not. Support join's input can not contain aggregate
// Only support project, filter, join, logical relation node and
// join condition should be slot reference equals currently
/**
* Check join is whether valid or not. Support join's input can not contain aggregate
* Only support project, filter, join, logical relation node and
* join condition should be slot reference equals currently
*/
@Override
protected boolean checkPattern(StructInfo structInfo) {
HyperGraph hyperGraph = structInfo.getHyperGraph();
HashSet<JoinType> requiredJoinType = Sets.newHashSet(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN);
for (AbstractNode node : hyperGraph.getNodes()) {
StructInfoNode structInfoNode = (StructInfoNode) node;
if (!structInfoNode.getPlan().accept(StructInfo.JOIN_PATTERN_CHECKER,
requiredJoinType)) {
SUPPORTED_JOIN_TYPE_SET)) {
return false;
}
for (Edge edge : hyperGraph.getEdges()) {
if (!edge.getJoin().accept(StructInfo.JOIN_PATTERN_CHECKER, requiredJoinType)) {
if (!edge.getJoin().accept(StructInfo.JOIN_PATTERN_CHECKER, SUPPORTED_JOIN_TYPE_SET)) {
return false;
}
}
Expand Down
10 changes: 5 additions & 5 deletions ...main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
* Use target output expression to represent the source expression
*/
protected List<Expression> rewriteExpression(
List<? extends Expression> sourceExpressions,
ExpressionMapping expressionMapping,
List<? extends Expression> sourceExpressionsToWrite,
ExpressionMapping mvExpressionToMvScanExpressionMapping,
SlotMapping sourceToTargetMapping) {
// Firstly, rewrite the target plan output expression using query with inverse mapping
// then try to use the mv expression to represent the query. if any of source expressions
Expand All @@ -175,16 +175,16 @@ protected List<Expression> rewriteExpression(
// transform source to:
// project(slot 2, 1)
// target
// generate mvSql to mvScan expressionMapping, and change mv sql expression to query based
// generate mvSql to mvScan mvExpressionToMvScanExpressionMapping, and change mv sql expression to query based
ExpressionMapping expressionMappingKeySourceBased =
expressionMapping.keyPermute(sourceToTargetMapping.inverse());
mvExpressionToMvScanExpressionMapping.keyPermute(sourceToTargetMapping.inverse());
List<Map<? extends Expression, ? extends Expression>> flattenExpressionMap =
expressionMappingKeySourceBased.flattenMap();
// view to view scan expression is 1:1 so get first element
Map<? extends Expression, ? extends Expression> mvSqlToMvScanMappingQueryBased = flattenExpressionMap.get(0);

List<Expression> rewrittenExpressions = new ArrayList<>();
for (Expression expressionToRewrite : sourceExpressions) {
for (Expression expressionToRewrite : sourceExpressionsToWrite) {
if (expressionToRewrite instanceof Literal) {
rewrittenExpressions.add(expressionToRewrite);
continue;
Expand Down
40 changes: 20 additions & 20 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,26 +74,26 @@ public static SplitPredicate splitPredicates(Expression expression) {
* The split different representation for predicate expression, such as equal, range and residual predicate.
*/
public static final class SplitPredicate {
private Optional<Expression> equalPredicates;
private Optional<Expression> rangePredicates;
private Optional<Expression> residualPredicates;

public SplitPredicate(Expression equalPredicates, Expression rangePredicates, Expression residualPredicates) {
this.equalPredicates = Optional.ofNullable(equalPredicates);
this.rangePredicates = Optional.ofNullable(rangePredicates);
this.residualPredicates = Optional.ofNullable(residualPredicates);
private Optional<Expression> equalPredicate;
private Optional<Expression> rangePredicate;
private Optional<Expression> residualPredicate;

public SplitPredicate(Expression equalPredicate, Expression rangePredicate, Expression residualPredicate) {
this.equalPredicate = Optional.ofNullable(equalPredicate);
this.rangePredicate = Optional.ofNullable(rangePredicate);
this.residualPredicate = Optional.ofNullable(residualPredicate);
}

public Expression getEqualPredicate() {
return equalPredicates.orElse(BooleanLiteral.TRUE);
return equalPredicate.orElse(BooleanLiteral.TRUE);
}

public Expression getRangePredicate() {
return rangePredicates.orElse(BooleanLiteral.TRUE);
return rangePredicate.orElse(BooleanLiteral.TRUE);
}

public Expression getResidualPredicate() {
return residualPredicates.orElse(BooleanLiteral.TRUE);
return residualPredicate.orElse(BooleanLiteral.TRUE);
}

public static SplitPredicate empty() {
Expand All @@ -113,24 +113,24 @@ public static SplitPredicate of(Expression equalPredicates,
* isEmpty
*/
public boolean isEmpty() {
return !equalPredicates.isPresent()
&& !rangePredicates.isPresent()
&& !residualPredicates.isPresent();
return !equalPredicate.isPresent()
&& !rangePredicate.isPresent()
&& !residualPredicate.isPresent();
}

public List<Expression> toList() {
return ImmutableList.of(equalPredicates.orElse(BooleanLiteral.TRUE),
rangePredicates.orElse(BooleanLiteral.TRUE),
residualPredicates.orElse(BooleanLiteral.TRUE));
return ImmutableList.of(equalPredicate.orElse(BooleanLiteral.TRUE),
rangePredicate.orElse(BooleanLiteral.TRUE),
residualPredicate.orElse(BooleanLiteral.TRUE));
}

/**
* Check the predicates in SplitPredicate is whether all true or not
*/
public boolean isAlwaysTrue() {
Expression equalExpr = equalPredicates.orElse(BooleanLiteral.TRUE);
Expression rangeExpr = rangePredicates.orElse(BooleanLiteral.TRUE);
Expression residualExpr = residualPredicates.orElse(BooleanLiteral.TRUE);
Expression equalExpr = equalPredicate.orElse(BooleanLiteral.TRUE);
Expression rangeExpr = rangePredicate.orElse(BooleanLiteral.TRUE);
Expression residualExpr = residualPredicate.orElse(BooleanLiteral.TRUE);
return equalExpr instanceof BooleanLiteral
&& rangeExpr instanceof BooleanLiteral
&& residualExpr instanceof BooleanLiteral
Expand Down
17 changes: 9 additions & 8 deletions ...-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,8 @@
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
Expand All @@ -40,10 +39,10 @@
*/
public class PredicatesSplitter {

private List<Expression> equalPredicates = new ArrayList<>();
private List<Expression> rangePredicates = new ArrayList<>();
private List<Expression> residualPredicates = new ArrayList<>();
private List<Expression> conjunctExpressions;
private final List<Expression> equalPredicates = new ArrayList<>();
private final List<Expression> rangePredicates = new ArrayList<>();
private final List<Expression> residualPredicates = new ArrayList<>();
private final List<Expression> conjunctExpressions;

private final PredicateExtract instance = new PredicateExtract();

Expand All @@ -54,7 +53,9 @@ public PredicatesSplitter(Expression target) {
}
}

/**PredicateExtract*/
/**
* PredicateExtract
*/
public class PredicateExtract extends DefaultExpressionVisitor<Void, Expression> {

@Override
Expand All @@ -63,7 +64,7 @@ public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Ex
Expression rightArg = comparisonPredicate.getArgument(1);
boolean leftArgOnlyContainsColumnRef = containOnlyColumnRef(leftArg, true);
boolean rightArgOnlyContainsColumnRef = containOnlyColumnRef(rightArg, true);
if (comparisonPredicate instanceof EqualTo || comparisonPredicate instanceof NullSafeEqual) {
if (comparisonPredicate instanceof EqualPredicate) {
if (leftArgOnlyContainsColumnRef && rightArgOnlyContainsColumnRef) {
equalPredicates.add(comparisonPredicate);
return null;
Expand Down
49 changes: 18 additions & 31 deletions .../src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@

package org.apache.doris.nereids.rules.exploration.mv.mapping;

import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;

import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableBiMap.Builder;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.Lists;
Expand All @@ -40,17 +40,17 @@
*/
public class RelationMapping extends Mapping {

private final BiMap<MappedRelation, MappedRelation> mappedRelationMap;
private final ImmutableBiMap<MappedRelation, MappedRelation> mappedRelationMap;

public RelationMapping(BiMap<MappedRelation, MappedRelation> mappedRelationMap) {
public RelationMapping(ImmutableBiMap<MappedRelation, MappedRelation> mappedRelationMap) {
this.mappedRelationMap = mappedRelationMap;
}

public BiMap<MappedRelation, MappedRelation> getMappedRelationMap() {
return mappedRelationMap;
}

public static RelationMapping of(BiMap<MappedRelation, MappedRelation> mappedRelationMap) {
public static RelationMapping of(ImmutableBiMap<MappedRelation, MappedRelation> mappedRelationMap) {
return new RelationMapping(mappedRelationMap);
}

Expand All @@ -59,27 +59,20 @@ public static RelationMapping of(BiMap<MappedRelation, MappedRelation> mappedRel
*/
public static List<RelationMapping> generate(List<CatalogRelation> sources, List<CatalogRelation> targets) {
// Construct tmp map, key is the table qualifier, value is the corresponding catalog relations
LinkedListMultimap<String, MappedRelation> sourceTableRelationIdMap = LinkedListMultimap.create();
LinkedListMultimap<Long, MappedRelation> sourceTableRelationIdMap = LinkedListMultimap.create();
for (CatalogRelation relation : sources) {
String tableQualifier = getTableQualifier(relation.getTable());
if (tableQualifier == null) {
return null;
}
sourceTableRelationIdMap.put(tableQualifier, MappedRelation.of(relation.getRelationId(), relation));
sourceTableRelationIdMap.put(getTableQualifier(relation.getTable()),
MappedRelation.of(relation.getRelationId(), relation));
}
LinkedListMultimap<String, MappedRelation> targetTableRelationIdMap = LinkedListMultimap.create();
LinkedListMultimap<Long, MappedRelation> targetTableRelationIdMap = LinkedListMultimap.create();
for (CatalogRelation relation : targets) {
String tableQualifier = getTableQualifier(relation.getTable());
if (tableQualifier == null) {
return null;
}
targetTableRelationIdMap.put(tableQualifier, MappedRelation.of(relation.getRelationId(), relation));
targetTableRelationIdMap.put(getTableQualifier(relation.getTable()),
MappedRelation.of(relation.getRelationId(), relation));
}

Set<String> sourceTableKeySet = sourceTableRelationIdMap.keySet();
Set<Long> sourceTableKeySet = sourceTableRelationIdMap.keySet();
List<List<Pair<MappedRelation, MappedRelation>>> mappedRelations = new ArrayList<>();

for (String sourceTableQualifier : sourceTableKeySet) {
for (Long sourceTableQualifier : sourceTableKeySet) {
List<MappedRelation> sourceMappedRelations = sourceTableRelationIdMap.get(sourceTableQualifier);
List<MappedRelation> targetMappedRelations = targetTableRelationIdMap.get(sourceTableQualifier);
if (targetMappedRelations.isEmpty()) {
Expand All @@ -104,23 +97,17 @@ public static List<RelationMapping> generate(List<CatalogRelation> sources, List

return Lists.cartesianProduct(mappedRelations).stream()
.map(mappedRelationList -> {
BiMap<MappedRelation, MappedRelation> relationMappedRelationBiMap =
HashBiMap.create(mappedRelationCount);
Builder<MappedRelation, MappedRelation> mapBuilder = ImmutableBiMap.builder();
for (int relationIndex = 0; relationIndex < mappedRelationCount; relationIndex++) {
relationMappedRelationBiMap.put(mappedRelationList.get(relationIndex).key(),
mapBuilder.put(mappedRelationList.get(relationIndex).key(),
mappedRelationList.get(relationIndex).value());
}
return RelationMapping.of(relationMappedRelationBiMap);
return RelationMapping.of(mapBuilder.build());
})
.collect(ImmutableList.toImmutableList());
}

private static String getTableQualifier(TableIf tableIf) {
String tableName = tableIf.getName();
DatabaseIf database = tableIf.getDatabase();
if (database == null) {
return null;
}
return database.getFullName() + ":" + tableName;
private static Long getTableQualifier(TableIf tableIf) {
return tableIf.getId();
}
}

0 comments on commit 2f8e70b

Please sign in to comment.