Skip to content

Commit

Permalink
mv poc code
Browse files Browse the repository at this point in the history
  • Loading branch information
@seawinde
seawinde committed Nov 8, 2023
1 parent dde3d26 commit 6f0b29b
Show file tree
Hide file tree
Showing 21 changed files with 459 additions and 238 deletions.
9 changes: 4 additions & 5 deletions ...e-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,9 +238,8 @@ public PhysicalPlanTranslator(PlanTranslatorContext context, StatsErrorEstimator
public PlanFragment translatePlan(PhysicalPlan physicalPlan) {
PlanFragment rootFragment = physicalPlan.accept(this, context);
List<Expr> outputExprs = Lists.newArrayList();
physicalPlan.getOutput().stream().forEach(slotReference -> outputExprs.add(
context.findSlotRef((SlotReference) slotReference) == null ? context.findSlotRef(
slotReference.getExprId()) : context.findSlotRef((SlotReference) slotReference)));
physicalPlan.getOutput().stream().map(Slot::getExprId)
.forEach(exprId -> outputExprs.add(context.findSlotRef(exprId)));
rootFragment.setOutputExprs(outputExprs);
Collections.reverse(context.getPlanFragments());
// TODO: maybe we need to trans nullable directly? and then we could remove call computeMemLayout
Expand Down Expand Up @@ -961,7 +960,7 @@ public PlanFragment visitPhysicalCTEConsumer(PhysicalCTEConsumer cteConsumer,
Slot consumerSlot = cteConsumer.getProducerToConsumerSlotMap().get(producerSlot);
SlotRef slotRef = context.findSlotRef(producerSlot.getExprId());
tupleDescriptor = slotRef.getDesc().getParent();
context.addExprIdSlotRefPair((SlotReference) consumerSlot, slotRef);
context.addExprIdSlotRefPair(consumerSlot.getExprId(), slotRef);
}
CTEScanNode cteScanNode = new CTEScanNode(tupleDescriptor);
context.getRuntimeTranslator().ifPresent(runtimeFilterTranslator ->
Expand Down Expand Up @@ -1646,7 +1645,7 @@ public PlanFragment visitPhysicalProject(PhysicalProject<? extends Plan> project
inputPlanNode.setOutputTupleDesc(projectionTuple);
} else {
for (int i = 0; i < slots.size(); ++i) {
context.addExprIdSlotRefPair((SlotReference) slots.get(i),
context.addExprIdSlotRefPair(slots.get(i).getExprId(),
(SlotRef) projectionExprs.get(i));
slotIdsByOrder.add(((SlotRef) projectionExprs.get(i)).getSlotId());
}
Expand Down
14 changes: 4 additions & 10 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ public class PlanTranslatorContext {
* index from Nereids' slot to legacy slot.
*/
private final Map<ExprId, SlotRef> exprIdToSlotRef = Maps.newHashMap();
private final Map<SlotReference, SlotRef> slotReferenceToSlotRef = Maps.newHashMap();

/**
* Inverted index from legacy slot to Nereids' slot.
Expand Down Expand Up @@ -200,10 +199,9 @@ public void addPlanFragment(PlanFragment planFragment) {
this.planFragments.add(planFragment);
}

public void addExprIdSlotRefPair(SlotReference slotReference, SlotRef slotRef) {
exprIdToSlotRef.put(slotReference.getExprId(), slotRef);
slotIdToExprId.put(slotRef.getDesc().getId(), slotReference.getExprId());
slotReferenceToSlotRef.put(slotReference, slotRef);
public void addExprIdSlotRefPair(ExprId exprId, SlotRef slotRef) {
exprIdToSlotRef.put(exprId, slotRef);
slotIdToExprId.put(slotRef.getDesc().getId(), exprId);
}

public void addExprIdColumnRefPair(ExprId exprId, ColumnRefExpr columnRefExpr) {
Expand All @@ -220,10 +218,6 @@ public SlotRef findSlotRef(ExprId exprId) {
return exprIdToSlotRef.get(exprId);
}

public SlotRef findSlotRef(SlotReference slotReference) {
return slotReferenceToSlotRef.get(slotReference);
}

public ColumnRefExpr findColumnRef(ExprId exprId) {
return exprIdToColumnRef.get(exprId);
}
Expand Down Expand Up @@ -279,7 +273,7 @@ public SlotDescriptor createSlotDesc(TupleDescriptor tupleDesc, SlotReference sl
}
slotRef.setTable(table);
slotRef.setLabel(slotReference.getName());
this.addExprIdSlotRefPair(slotReference, slotRef);
this.addExprIdSlotRefPair(slotReference.getExprId(), slotRef);
slotDescriptor.setIsNullable(slotReference.nullable());
return slotDescriptor;
}
Expand Down
5 changes: 5 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@

/**
* HyperGraph Node.
* Jc
* \
* F
* \
* JC
*/
public class Node {
private final int index;
Expand Down
12 changes: 12 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.doris.nereids.cost.Cost;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.rewrite.mv.StructInfo;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
Expand Down Expand Up @@ -74,6 +75,8 @@ public class Group {

private int chosenGroupExpressionId = -1;

private Optional<StructInfo> structInfo = Optional.empty();

/**
* Constructor for Group.
*
Expand Down Expand Up @@ -152,6 +155,7 @@ public GroupExpression logicalExpressionsAt(int index) {
* @return the first logical group expression in this group
*/
public GroupExpression getLogicalExpression() {
// poc tmp
Preconditions.checkArgument(logicalExpressions.size() == 1,
"There should be only one Logical Expression in Group");
return logicalExpressions.get(0);
Expand Down Expand Up @@ -532,4 +536,12 @@ public String treeString() {

return TreeStringUtils.treeString(this, toString, getChildren, getExtraPlans, displayExtraPlan);
}

public Optional<StructInfo> getStructInfo() {
return structInfo;
}

public void setStructInfo(StructInfo structInfo) {
this.structInfo = Optional.ofNullable(structInfo);
}
}
17 changes: 17 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ private Group init(Plan plan) {
plan = replaceChildrenToGroupPlan(plan, childrenGroups);
GroupExpression newGroupExpression = new GroupExpression(plan, childrenGroups);
Group group = new Group(groupIdGenerator.getNextId(), newGroupExpression, plan.getLogicalProperties());
// PoC add struct info to group

groups.put(group.getGroupId(), group);
if (groupExpressions.containsKey(newGroupExpression)) {
Expand All @@ -323,6 +324,22 @@ private Group init(Plan plan) {
return group;
}

/** initPoC */
public Group initPoC(Plan plan) {
Preconditions.checkArgument(!(plan instanceof GroupPlan), "Cannot init memo by a GroupPlan");

/* initialize children recursively */
List<Group> childrenGroups = new ArrayList<>(plan.arity());
for (Plan child : plan.children()) {
childrenGroups.add(initPoC(child));
}

plan = replaceChildrenToGroupPlan(plan, childrenGroups);
GroupExpression newGroupExpression = new GroupExpression(plan, childrenGroups);
Group group = new Group(groupIdGenerator.getNextId(), newGroupExpression, plan.getLogicalProperties());
return group;
}

/**
* add or replace the plan into the target group.
* <p>
Expand Down
2 changes: 1 addition & 1 deletion ...core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ private Plan deferMaterialize(LogicalResultSink<? extends Plan> logicalResultSin
LogicalTopN<? extends Plan> logicalTopN, Optional<LogicalFilter<? extends Plan>> logicalFilter,
LogicalOlapScan logicalOlapScan) {
Column rowId = new Column(Column.ROWID_COL, Type.STRING, false, null, false, "", "rowid column");
SlotReference columnId = SlotReference.fromColumn(rowId, logicalOlapScan.getQualifier());
SlotReference columnId = SlotReference.fromColumn(rowId, logicalOlapScan.getQualifier(), null);
Set<ExprId> deferredMaterializedExprIds = Sets.newHashSet(logicalOlapScan.getOutputExprIdSet());
logicalFilter.ifPresent(filter -> filter.getConjuncts()
.forEach(e -> deferredMaterializedExprIds.removeAll(e.getInputSlotExprIds())));
Expand Down
74 changes: 63 additions & 11 deletions ...main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewJoinRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,27 @@
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitors;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitors.SlotReferenceReplacer.ExprReplacer;

import com.google.common.collect.BiMap;

import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;

/**
Expand All @@ -45,26 +53,70 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali
protected Plan rewriteView(MatchMode matchMode,
StructInfo queryStructInfo,
StructInfo viewStructInfo,
BiMap<Long, Long> queryToViewTableMappings,
Plan temporaryRewrite) {
BiMap<RelationId, RelationId> queryToViewTableMappings,
Plan tempRewritedPlan) {

List<Expression> expressions = rewriteExpression(queryStructInfo.getTopExpressions(),
queryStructInfo,
viewStructInfo,
queryToViewTableMappings,
temporaryRewrite
tempRewritedPlan
);
if (expressions == null) {
return queryStructInfo.getPlan();
}
// TODO add rewrited project correctly
Map<Slot, Slot> mvOutputSet = temporaryRewrite.getLogicalProperties().getOutputMap();
// Simplify implement
List<NamedExpression> namedExpressions = queryStructInfo.getPlan().getLogicalProperties().getOutput()
.stream()
.map(slot -> (NamedExpression) mvOutputSet.get(slot))
.collect(Collectors.toList());
return new LogicalProject<>(namedExpressions, temporaryRewrite);
// PoC Generate mapping from query slot reference to mv slot reference, note: clone
// if any slot can not map then bail out
// simplfy implement
Set<SlotReference> querySlotSet = new HashSet<>();
queryStructInfo.getPlan().accept(PlanVisitors.SLOT_REFERENCE_COLLECTOR, querySlotSet);

Set<SlotReference> viewSlotSet = new HashSet<>();
viewStructInfo.getPlan().accept(PlanVisitors.SLOT_REFERENCE_COLLECTOR, viewSlotSet);

Map<SlotReference, SlotReference> queryToViewSlotMapping = new HashMap<>();
for (SlotReference querySlot : querySlotSet) {
for (SlotReference viewSlot : viewSlotSet) {
if (Objects.equals(querySlot.getName(), viewSlot.getName())
&& Objects.equals(querySlot.getQualifier(), viewSlot.getQualifier())) {
queryToViewSlotMapping.put(querySlot, viewSlot);
}
}
}
// PoC Generate mapping from mv sql output to mv scan out put
Map<SlotReference, SlotReference> mvToMvScanMapping = new HashMap<>();
List<Slot> mvScanSlotList = tempRewritedPlan.getOutput();
List<Slot> mvSlotList = viewStructInfo.getPlan().getOutput();
for (int i = 0; i < mvSlotList.size(); i++) {
mvToMvScanMapping.put((SlotReference) mvSlotList.get(i), (SlotReference) mvScanSlotList.get(i));
}

// TODO check if the query expr can derive from the view
// PoC If the query expression can get from mv sql, so replace the mv scan slot reference
// PoC according to the mapping above. Simplify implement
Map<Slot, Slot> mvScanToQueryMapping = new HashMap<>();
List<Slot> output = queryStructInfo.getPlan().getOutput();
for (Slot querySlot : output) {
Slot mvSlot = queryToViewSlotMapping.get(querySlot);
if (mvSlot == null) {
return null;
}
SlotReference mvScanSlot = mvToMvScanMapping.get(mvSlot);
if (mvScanSlot == null) {
return null;
}
mvScanToQueryMapping.put(mvScanSlot, querySlot);
}
// Replace the mv scan output with query slot, lazy before add filter and other project

// tempRewritedPlan.accept(SlotReferenceReplacer.INSTANCE, mvScanToQueryMapping);

tempRewritedPlan.getOutput().stream()
.forEach(slot -> slot.accept(ExprReplacer.INSTANCE, mvScanToQueryMapping));
LogicalProject<Plan> planLogicalProject = new LogicalProject<>(
output.stream().map(NamedExpression.class::cast).collect(Collectors.toList()),
tempRewritedPlan);
return planLogicalProject;
}

protected boolean isPatternSupport(LogicalProject topProject, Plan plan) {
Expand Down
Loading

0 comments on commit 6f0b29b

Please sign in to comment.