Skip to content

Commit

Permalink
no stats left zigzag
Browse files Browse the repository at this point in the history
  • Loading branch information
@englefly
englefly committed Nov 9, 2023
1 parent 7df60a4 commit f3e8999
Show file tree
Hide file tree
Showing 11 changed files with 86 additions and 41 deletions.
21 changes: 0 additions & 21 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,27 +293,12 @@ public Cost visitPhysicalHashJoin(
// use totalInstanceNumber to the power of 2 as the default factor value
buildSideFactor = Math.pow(totalInstanceNumber, 0.5);
}
// TODO: since the outputs rows may expand a lot, penalty on it will cause bc never be chosen.
// will refine this in next generation cost model.
if (!context.isStatsReliable()) {
// forbid broadcast join when stats is unknown
return CostV1.of(context.getSessionVariable(), rightRowCount * buildSideFactor + 1 / leftRowCount,
rightRowCount,
0
);
}
return CostV1.of(context.getSessionVariable(),
leftRowCount + rightRowCount * buildSideFactor + outputRowCount * probeSideFactor,
rightRowCount,
0
);
}
if (!context.isStatsReliable()) {
return CostV1.of(context.getSessionVariable(),
rightRowCount + 1 / leftRowCount,
rightRowCount,
0);
}
return CostV1.of(context.getSessionVariable(), leftRowCount + rightRowCount + outputRowCount,
rightRowCount,
0
Expand All @@ -328,12 +313,6 @@ public Cost visitPhysicalNestedLoopJoin(
Preconditions.checkState(context.arity() == 2);
Statistics leftStatistics = context.getChildStatistics(0);
Statistics rightStatistics = context.getChildStatistics(1);
if (!context.isStatsReliable()) {
return CostV1.of(context.getSessionVariable(),
rightStatistics.getRowCount() + 1 / leftStatistics.getRowCount(),
rightStatistics.getRowCount(),
0);
}
return CostV1.of(context.getSessionVariable(),
leftStatistics.getRowCount() * rightStatistics.getRowCount(),
rightStatistics.getRowCount(),
Expand Down
16 changes: 12 additions & 4 deletions ...core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,20 @@ public void execute() {
}

private List<Rule> getExplorationRules() {
if (groupExpression.getOwnerGroup() == null) {
System.out.println(groupExpression);
}
boolean isDisableJoinReorder = context.getCascadesContext().getConnectContext().getSessionVariable()
.isDisableJoinReorder()
|| context.getCascadesContext().getMemo().getGroupExpressionsSize() > context.getCascadesContext()
.getConnectContext().getSessionVariable().memoMaxGroupExpressionSize;
.isDisableJoinReorder()
|| context.getCascadesContext().getMemo().getGroupExpressionsSize() > context.getCascadesContext()
.getConnectContext().getSessionVariable().memoMaxGroupExpressionSize;
boolean isDpHyp = context.getCascadesContext().getStatementContext().isDpHyp();
boolean isOtherJoinReorder = context.getCascadesContext().getStatementContext().isOtherJoinReorder();
boolean isEnableBushyTree = context.getCascadesContext().getConnectContext().getSessionVariable()
.isEnableBushyTree();
.isEnableBushyTree();
boolean isLeftZigZagTree = context.getCascadesContext().getConnectContext()
.getSessionVariable().isEnableLeftZigZag()
|| !(groupExpression.getOwnerGroup() != null && groupExpression.getOwnerGroup().isStatsReliable());
int joinNumBushyTree = context.getCascadesContext().getConnectContext()
.getSessionVariable().getMaxJoinNumBushyTree();
if (isDisableJoinReorder) {
Expand All @@ -77,6 +83,8 @@ private List<Rule> getExplorationRules() {
} else {
return Collections.emptyList();
}
} else if (isLeftZigZagTree) {
return getRuleSet().getLeftZigZagTreeJoinReorder();
} else if (isEnableBushyTree) {
return getRuleSet().getBushyTreeJoinReorder();
} else if (context.getCascadesContext().getStatementContext().getMaxNAryInnerJoin() <= joinNumBushyTree) {
Expand Down
11 changes: 11 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,13 @@ public class RuleSet {
.add(new LogicalDeferMaterializeResultSinkToPhysicalDeferMaterializeResultSink())
.build();

public static final List<Rule> LEFT_ZIG_ZAG_TREE_JOIN_REORDER = planRuleFactories()
.add(JoinCommute.LEFT_ZIG_ZAG)
.add(InnerJoinLAsscom.LEFT_ZIG_ZAG)
.add(InnerJoinLAsscomProject.LEFT_ZIG_ZAG)
.addAll(OTHER_REORDER_RULES)
.build();

public static final List<Rule> ZIG_ZAG_TREE_JOIN_REORDER = planRuleFactories()
.add(JoinCommute.ZIG_ZAG)
.add(InnerJoinLAsscom.INSTANCE)
Expand Down Expand Up @@ -220,6 +227,10 @@ public List<Rule> getZigZagTreeJoinReorder() {
return ZIG_ZAG_TREE_JOIN_REORDER_RULES;
}

public List<Rule> getLeftZigZagTreeJoinReorder() {
return LEFT_ZIG_ZAG_TREE_JOIN_REORDER;
}

public List<Rule> getBushyTreeJoinReorder() {
return BUSHY_TREE_JOIN_REORDER_RULES;
}
Expand Down
29 changes: 23 additions & 6 deletions ...-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscom.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,13 @@
* Rule for change inner join LAsscom (associative and commutive).
*/
public class InnerJoinLAsscom extends OneExplorationRuleFactory {
public static final InnerJoinLAsscom INSTANCE = new InnerJoinLAsscom();
public static final InnerJoinLAsscom INSTANCE = new InnerJoinLAsscom(false);
public static final InnerJoinLAsscom LEFT_ZIG_ZAG = new InnerJoinLAsscom(true);
private boolean leftZigZag = false;

public InnerJoinLAsscom(boolean leftZigZag) {
this.leftZigZag = leftZigZag;
}

/*
* topJoin newTopJoin
Expand All @@ -48,7 +54,7 @@ public class InnerJoinLAsscom extends OneExplorationRuleFactory {
@Override
public Rule build() {
return innerLogicalJoin(innerLogicalJoin(), group())
.when(topJoin -> checkReorder(topJoin, topJoin.left()))
.when(topJoin -> checkReorder(topJoin, topJoin.left(), leftZigZag))
.whenNot(join -> join.hasJoinHint() || join.left().hasJoinHint())
.whenNot(join -> join.isMarkJoin() || join.left().isMarkJoin())
.then(topJoin -> {
Expand Down Expand Up @@ -85,11 +91,22 @@ public Rule build() {
}).toRule(RuleType.LOGICAL_INNER_JOIN_LASSCOM);
}

/**
* trigger rule condition
*/
public static boolean checkReorder(LogicalJoin<? extends Plan, GroupPlan> topJoin,
LogicalJoin<GroupPlan, GroupPlan> bottomJoin) {
return !bottomJoin.getJoinReorderContext().hasCommuteZigZag()
&& !topJoin.getJoinReorderContext().hasLAsscom()
&& (!bottomJoin.isMarkJoin() && !topJoin.isMarkJoin());
LogicalJoin<GroupPlan, GroupPlan> bottomJoin, boolean leftZigZag) {
if (leftZigZag) {
double bRows = bottomJoin.right().getGroup().getStatistics().getRowCount();
double cRows = topJoin.right().getGroup().getStatistics().getRowCount();
return bRows < cRows && !bottomJoin.getJoinReorderContext().hasCommuteZigZag()
&& !topJoin.getJoinReorderContext().hasLAsscom()
&& (!bottomJoin.isMarkJoin() && !topJoin.isMarkJoin());
} else {
return !bottomJoin.getJoinReorderContext().hasCommuteZigZag()
&& !topJoin.getJoinReorderContext().hasLAsscom()
&& (!bottomJoin.isMarkJoin() && !topJoin.isMarkJoin());
}
}

/**
Expand Down
10 changes: 8 additions & 2 deletions ...rc/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,13 @@
* Rule for change inner join LAsscom (associative and commutive).
*/
public class InnerJoinLAsscomProject extends OneExplorationRuleFactory {
public static final InnerJoinLAsscomProject INSTANCE = new InnerJoinLAsscomProject();
public static final InnerJoinLAsscomProject INSTANCE = new InnerJoinLAsscomProject(false);
public static final InnerJoinLAsscomProject LEFT_ZIG_ZAG = new InnerJoinLAsscomProject(true);
private final boolean enableLeftZigZag;

public InnerJoinLAsscomProject(boolean enableLeftZigZag) {
this.enableLeftZigZag = enableLeftZigZag;
}
/*
* topJoin newTopJoin
* / \ / \
Expand All @@ -51,10 +56,11 @@ public class InnerJoinLAsscomProject extends OneExplorationRuleFactory {
* / \ / \
* A B A C
*/

@Override
public Rule build() {
return innerLogicalJoin(logicalProject(innerLogicalJoin()), group())
.when(topJoin -> InnerJoinLAsscom.checkReorder(topJoin, topJoin.left().child()))
.when(topJoin -> InnerJoinLAsscom.checkReorder(topJoin, topJoin.left().child(), enableLeftZigZag))
.whenNot(join -> join.hasJoinHint() || join.left().child().hasJoinHint())
.whenNot(join -> join.isMarkJoin() || join.left().child().isMarkJoin())
.when(join -> join.left().isAllSlots())
Expand Down
14 changes: 13 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
public class JoinCommute extends OneExplorationRuleFactory {

public static final JoinCommute LEFT_DEEP = new JoinCommute(SwapType.LEFT_DEEP, false);
public static final JoinCommute LEFT_ZIG_ZAG = new JoinCommute(SwapType.LEFT_ZIG_ZAG, false);
public static final JoinCommute ZIG_ZAG = new JoinCommute(SwapType.ZIG_ZAG, false);
public static final JoinCommute BUSHY = new JoinCommute(SwapType.BUSHY, false);
public static final JoinCommute NON_INNER = new JoinCommute(SwapType.BUSHY, true);
Expand Down Expand Up @@ -73,7 +74,8 @@ public Rule build() {
}

enum SwapType {
LEFT_DEEP, ZIG_ZAG, BUSHY
LEFT_DEEP, ZIG_ZAG, BUSHY,
LEFT_ZIG_ZAG
}

/**
Expand All @@ -88,6 +90,12 @@ public static boolean check(SwapType swapType, LogicalJoin<GroupPlan, GroupPlan>
return false;
}

if (swapType == SwapType.LEFT_ZIG_ZAG) {
double leftRows = join.left().getGroup().getStatistics().getRowCount();
double rightRows = join.right().getGroup().getStatistics().getRowCount();
return leftRows < rightRows && isZigZagJoin(join);
}

return true;
}

Expand All @@ -101,6 +109,10 @@ public static boolean isNotBottomJoin(LogicalJoin<GroupPlan, GroupPlan> join) {
return containJoin(join.left()) || containJoin(join.right());
}

public static boolean isZigZagJoin(LogicalJoin<GroupPlan, GroupPlan> join) {
return !containJoin(join.left()) || !containJoin(join.right());
}

private static boolean containJoin(GroupPlan groupPlan) {
// TODO: tmp way to judge containJoin
List<Slot> output = groupPlan.getOutput();
Expand Down
2 changes: 1 addition & 1 deletion ...ava/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public class SemiJoinSemiJoinTransposeProject extends OneExplorationRuleFactory
public Rule build() {
return logicalJoin(logicalProject(logicalJoin()), group())
.when(this::typeChecker)
.when(topSemi -> InnerJoinLAsscom.checkReorder(topSemi, topSemi.left().child()))
.when(topSemi -> InnerJoinLAsscom.checkReorder(topSemi, topSemi.left().child(), false))
.whenNot(join -> join.hasJoinHint() || join.left().child().hasJoinHint())
.whenNot(join -> join.isMarkJoin() || join.left().child().isMarkJoin())
.when(join -> join.left().isAllSlots())
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ private static Statistics estimateNestLoopJoin(Statistics leftStats, Statistics

private static Statistics estimateInnerJoin(Statistics leftStats, Statistics rightStats, Join join) {
if (hashJoinConditionContainsUnknownColumnStats(leftStats, rightStats, join)) {
double rowCount = leftStats.getRowCount() + rightStats.getRowCount();
double rowCount = Math.max(leftStats.getRowCount(), rightStats.getRowCount());
rowCount = Math.max(1, rowCount);
return new StatisticsBuilder()
.setRowCount(rowCount)
Expand Down
12 changes: 12 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ public class SessionVariable implements Serializable, Writable {

public static final String ENABLE_DPHYP_OPTIMIZER = "enable_dphyp_optimizer";

public static final String ENABLE_LEFT_ZIG_ZAG = "enable_left_zig_zag";
public static final String NTH_OPTIMIZED_PLAN = "nth_optimized_plan";

public static final String ENABLE_NEREIDS_PLANNER = "enable_nereids_planner";
Expand Down Expand Up @@ -909,6 +910,17 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) {
@VariableMgr.VarAttr(name = NTH_OPTIMIZED_PLAN)
private int nthOptimizedPlan = 1;

public boolean isEnableLeftZigZag() {
return enableLeftZigZag;
}

public void setEnableLeftZigZag(boolean enableLeftZigZag) {
this.enableLeftZigZag = enableLeftZigZag;
}

@VariableMgr.VarAttr(name = ENABLE_LEFT_ZIG_ZAG)
private boolean enableLeftZigZag = false;

/**
* as the new optimizer is not mature yet, use this var
* to control whether to use new optimizer, remove it when
Expand Down
8 changes: 4 additions & 4 deletions ...src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifierTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,10 @@ void testStarQuery() {
.add(Pair.of(17L, 2L)) // 04 - 1
.add(Pair.of(17L, 4L)) // 04 - 2
.add(Pair.of(17L, 8L)) // 04 - 3
.add(Pair.of(25L, 2L)) // 034 - 1
.add(Pair.of(25L, 4L)) // 034 - 2
.add(Pair.of(29L, 2L)) // 0234 - 1
.build(); // 0-4-3-2-1 : big left deep tree
.add(Pair.of(19L, 8L)) // 041 - 2
.add(Pair.of(21L, 2L)) // 042 - 1
.add(Pair.of(23L, 8L)) // 0134 - 2
.build(); // 0-4-3-1-2 : big left deep tree
for (Pair<Long, Long> step : steps) {
if (!graphSimplifier.applySimplificationStep()) {
break;
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/test/java/org/apache/doris/nereids/memo/RankTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ void test() {
shape.add(memo.unrank(memo.rank(i + 1).first).shape(""));
}
System.out.println(shape);
Assertions.assertEquals(4, shape.size());
Assertions.assertEquals(1, shape.size());
Assertions.assertEquals(bestPlan.shape(""), memo.unrank(memo.rank(1).first).shape(""));
}
}

0 comments on commit f3e8999

Please sign in to comment.