Skip to content

Commit

Permalink
fix some comments and optimize some code
Browse files Browse the repository at this point in the history
  • Loading branch information
@seawinde
seawinde committed Dec 19, 2023
1 parent 60ef06c commit a67dcd2
Show file tree
Hide file tree
Showing 24 changed files with 214 additions and 171 deletions.
1 change: 0 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ public MTMVCache(Plan logicalPlan, List<NamedExpression> mvOutputExpressions) {
public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) {
LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql());
// TODO: connect context set current db when create mv by use database
// view should also disable the predicate infer and join eliminate.
StatementContext mvSqlStatementContext = new StatementContext(connectContext,
new OriginStatement(mtmv.getQuerySql(), 0));
NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext);
Expand Down
3 changes: 1 addition & 2 deletions fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,7 @@ public class MTMVRelationManager implements MTMVHookService {
private Map<BaseTableInfo, Set<BaseTableInfo>> tableMTMVs = Maps.newConcurrentMap();

public Set<BaseTableInfo> getMtmvsByBaseTable(BaseTableInfo table) {
Set<BaseTableInfo> baseTableInfos = tableMTMVs.get(table);
return baseTableInfos == null ? ImmutableSet.of() : baseTableInfos;
return tableMTMVs.getOrDefault(table, ImmutableSet.of());
}

public Set<MTMV> getAvailableMTMVs(List<BaseTableInfo> tableInfos) {
Expand Down
24 changes: 2 additions & 22 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
* @see PlaceholderCollector
*/
public class PlaceholderExpression extends Expression implements AlwaysNotNullable {
protected boolean distinct;
private final Class<? extends Expression> delegateClazz;
/**
* 1 based
Expand All @@ -46,23 +45,10 @@ public PlaceholderExpression(List<Expression> children, Class<? extends Expressi
this.position = position;
}

public PlaceholderExpression(List<Expression> children, Class<? extends Expression> delegateClazz, int position,
boolean distinct) {
super(children);
this.delegateClazz = Objects.requireNonNull(delegateClazz, "delegateClazz should not be null");
this.position = position;
this.distinct = distinct;
}

public static PlaceholderExpression of(Class<? extends Expression> delegateClazz, int position) {
return new PlaceholderExpression(ImmutableList.of(), delegateClazz, position);
}

public static PlaceholderExpression of(Class<? extends Expression> delegateClazz, int position,
boolean distinct) {
return new PlaceholderExpression(ImmutableList.of(), delegateClazz, position, distinct);
}

@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visit(this, context);
Expand All @@ -76,10 +62,6 @@ public int getPosition() {
return position;
}

public boolean isDistinct() {
return distinct;
}

@Override
public boolean equals(Object o) {
if (this == o) {
Expand All @@ -92,13 +74,11 @@ public boolean equals(Object o) {
return false;
}
PlaceholderExpression that = (PlaceholderExpression) o;
return position == that.position
&& Objects.equals(delegateClazz, that.delegateClazz)
&& distinct == that.distinct;
return position == that.position && Objects.equals(delegateClazz, that.delegateClazz);
}

@Override
public int hashCode() {
return Objects.hash(super.hashCode(), delegateClazz, position, distinct);
return Objects.hash(super.hashCode(), delegateClazz, position);
}
}
2 changes: 0 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@
import org.apache.doris.nereids.rules.rewrite.MergeGenerates;
import org.apache.doris.nereids.rules.rewrite.MergeLimits;
import org.apache.doris.nereids.rules.rewrite.MergeProjects;
import org.apache.doris.nereids.rules.rewrite.OrExpansion;
import org.apache.doris.nereids.rules.rewrite.PushDownAliasThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownExpressionsInHashCondition;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughAggregation;
Expand Down Expand Up @@ -126,7 +125,6 @@ public class RuleSet {
.add(PushDownProjectThroughSemiJoin.INSTANCE)
.add(TransposeAggSemiJoin.INSTANCE)
.add(TransposeAggSemiJoinProject.INSTANCE)
.add(OrExpansion.INSTANCE)
.build();

public static final List<RuleFactory> PUSH_DOWN_FILTERS = ImmutableList.of(
Expand Down
99 changes: 32 additions & 67 deletions .../org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.doris.nereids.rules.exploration.mv;

import org.apache.doris.common.Pair;
import org.apache.doris.nereids.analyzer.PlaceholderExpression;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.AbstractNode;
Expand All @@ -30,21 +29,18 @@
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.functions.Any;
import org.apache.doris.nereids.trees.expressions.functions.CouldRollUp;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion;
import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.ExpressionUtils;

import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
Expand All @@ -64,15 +60,15 @@
*/
public abstract class AbstractMaterializedViewAggregateRule extends AbstractMaterializedViewRule {

protected static final Map<PlaceholderExpression, PlaceholderExpression>
protected static final Map<Expression, Expression>
AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP = new HashMap<>();
protected final String currentClassName = this.getClass().getSimpleName();

private final Logger logger = LogManager.getLogger(this.getClass());

static {
AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(
PlaceholderExpression.of(Count.class, 0, true),
new PlaceholderExpression(ImmutableList.of(), BitmapUnion.class, 0));
AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new Count(true, Any.INSTANCE),
new BitmapUnion(Any.INSTANCE));
}

@Override
Expand All @@ -85,12 +81,12 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
// get view and query aggregate and top plan correspondingly
Pair<Plan, LogicalAggregate<Plan>> viewTopPlanAndAggPair = splitToTopPlanAndAggregate(viewStructInfo);
if (viewTopPlanAndAggPair == null) {
logger.info(currentClassName + "split to view to top plan and agg fail so return null");
logger.warn(currentClassName + " split to view to top plan and agg fail so return null");
return null;
}
Pair<Plan, LogicalAggregate<Plan>> queryTopPlanAndAggPair = splitToTopPlanAndAggregate(queryStructInfo);
if (queryTopPlanAndAggPair == null) {
logger.info(currentClassName + "split to query to top plan and agg fail so return null");
logger.warn(currentClassName + " split to query to top plan and agg fail so return null");
return null;
}
// Firstly, handle query group by expression rewrite
Expand Down Expand Up @@ -119,7 +115,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
true);
if (rewrittenQueryGroupExpr.isEmpty()) {
// can not rewrite, bail out.
logger.info(currentClassName + " can not rewrite expression when not need roll up");
logger.debug(currentClassName + " can not rewrite expression when not need roll up");
return null;
}
return new LogicalProject<>(
Expand All @@ -134,14 +130,14 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
viewExpr -> viewExpr.anyMatch(expr -> expr instanceof AggregateFunction
&& ((AggregateFunction) expr).isDistinct()))) {
// if mv aggregate function contains distinct, can not roll up, bail out.
logger.info(currentClassName + " view contains distinct function so can not roll up");
logger.debug(currentClassName + " view contains distinct function so can not roll up");
return null;
}
// split the query top plan expressions to group expressions and functions, if can not, bail out.
Pair<Set<? extends Expression>, Set<? extends Expression>> queryGroupAndFunctionPair
= topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair);
if (queryGroupAndFunctionPair == null) {
logger.info(currentClassName + " query top plan split to group by and function fail so return null");
logger.warn(currentClassName + " query top plan split to group by and function fail so return null");
return null;
}
// Secondly, try to roll up the agg functions
Expand All @@ -165,11 +161,9 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
// try to roll up
AggregateFunction queryFunction = (AggregateFunction) topExpression.firstMatch(
expr -> expr instanceof AggregateFunction);
Function rollupAggregateFunction = rollup(queryFunction,
queryFunctionShuttled, mvExprToMvScanExprQueryBased);
Function rollupAggregateFunction = rollup(queryFunction, queryFunctionShuttled,
mvExprToMvScanExprQueryBased);
if (rollupAggregateFunction == null) {
logger.info(currentClassName + " query function " + queryFunction.getName()
+ " can not roll up so return null");
return null;
}
// key is query need roll up expr, value is mv scan based roll up expr
Expand All @@ -181,7 +175,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
queryToViewSlotMapping,
false);
if (rewrittenFunctionExpression == null) {
logger.info(currentClassName + " roll up expression can not rewrite by view so return null");
logger.debug(currentClassName + " roll up expression can not rewrite by view so return null");
return null;
}
finalAggregateExpressions.add((NamedExpression) rewrittenFunctionExpression);
Expand All @@ -191,7 +185,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
ExpressionUtils.shuttleExpressionWithLineage(topExpression, queryTopPlan);
if (!mvExprToMvScanExprQueryBased.containsKey(queryGroupShuttledExpr)) {
// group expr can not rewrite by view
logger.info(currentClassName
logger.debug(currentClassName
+ " view group expressions can not contains the query group by expression so return null");
return null;
}
Expand All @@ -205,7 +199,8 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
queryToViewSlotMapping,
true);
if (rewrittenGroupExpression == null) {
logger.info(currentClassName + " query top expression can not be rewritten by view so return null");
logger.debug(currentClassName
+ " query top expression can not be rewritten by view so return null");
return null;
}
finalAggregateExpressions.add((NamedExpression) rewrittenGroupExpression);
Expand Down Expand Up @@ -258,12 +253,15 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
private Function rollup(AggregateFunction queryFunction,
Expression queryFunctionShuttled,
Map<Expression, Expression> mvExprToMvScanExprQueryBased) {
if (!(queryFunction instanceof CouldRollUp)) {
return null;
}
Expression rollupParam = null;
if (mvExprToMvScanExprQueryBased.containsKey(queryFunctionShuttled)) {
// function can not rewrite by view
// function can rewrite by view
rollupParam = mvExprToMvScanExprQueryBased.get(queryFunctionShuttled);
} else {
// try to use complex roll up param
// function can not rewrite by view, try to use complex roll up param
// eg: query is count(distinct param), mv sql is bitmap_union(to_bitmap(param))
for (Expression mvExprShuttled : mvExprToMvScanExprQueryBased.keySet()) {
if (!(mvExprShuttled instanceof Function)) {
Expand All @@ -278,24 +276,7 @@ private Function rollup(AggregateFunction queryFunction,
return null;
}
// do roll up
Class<? extends Function> rollupAggregateFunction = queryFunction.getRollup();
if (rollupAggregateFunction == null) {
return null;
}
if (Sum.class.isAssignableFrom(rollupAggregateFunction)) {
return new Sum(queryFunction.isDistinct(), rollupParam);
}
if (Max.class.isAssignableFrom(rollupAggregateFunction)) {
return new Max(queryFunction.isDistinct(), rollupParam);
}
if (Min.class.isAssignableFrom(rollupAggregateFunction)) {
return new Min(queryFunction.isDistinct(), rollupParam);
}
if (BitmapUnionCount.class.isAssignableFrom(rollupAggregateFunction)) {
return new BitmapUnionCount(rollupParam);
}
// can rollup return null
return null;
return ((CouldRollUp) queryFunction).constructRollUp(rollupParam);
}

private Pair<Set<? extends Expression>, Set<? extends Expression>> topPlanSplitToGroupAndFunction(
Expand Down Expand Up @@ -367,37 +348,21 @@ protected boolean checkPattern(StructInfo structInfo) {
}

private boolean isAggregateFunctionEquivalent(Function queryFunction, Function viewFunction) {
Class<? extends Function> queryClazz = queryFunction.getClass();
Class<? extends Function> viewClazz = viewFunction.getClass();
if (queryClazz.isAssignableFrom(viewClazz)) {
if (queryFunction.equals(viewFunction)) {
return true;
}
// bitmap roll up
boolean isDistinct = queryFunction instanceof AggregateFunction
&& ((AggregateFunction) queryFunction).isDistinct();
PlaceholderExpression equivalentFunction = AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.get(
PlaceholderExpression.of(queryFunction.getClass(), 0, isDistinct));
// get query equivalent function
Expression equivalentFunction = null;
for (Map.Entry<Expression, Expression> entry : AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.entrySet()) {
if (entry.getKey().equals(queryFunction)) {
equivalentFunction = entry.getValue();
}
}
// check is have equivalent function or not
if (equivalentFunction == null) {
return false;
}
// current compare
if (!viewFunction.getClass().isAssignableFrom(equivalentFunction.getDelegateClazz())) {
return false;
}
if (!viewFunction.children().isEmpty() && !equivalentFunction.children().isEmpty()) {
// children compare, just compare two level, support more later
List<Expression> equivalentFunctions = equivalentFunction.children();
if (viewFunction.children().size() != equivalentFunctions.size()) {
return false;
}
for (int i = 0; i < viewFunction.children().size(); i++) {
if (!viewFunction.child(i).getClass().equals(
((PlaceholderExpression) equivalentFunctions.get(i)).getDelegateClazz())) {
return false;
}
}
}
return true;
return equivalentFunction.equals(viewFunction);
}
}
2 changes: 1 addition & 1 deletion .../java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
// Can not rewrite, bail out
if (expressionsRewritten.isEmpty()
|| expressionsRewritten.stream().anyMatch(expr -> !(expr instanceof NamedExpression))) {
logger.info(currentClassName + " expression to rewrite is not named expr so return null");
logger.warn(currentClassName + " expression to rewrite is not named expr so return null");
return null;
}
// record the group id in materializationContext, and when rewrite again in
Expand Down
Loading

0 comments on commit a67dcd2

Please sign in to comment.