Skip to content

Commit

Permalink
tmp
Browse files Browse the repository at this point in the history
  • Loading branch information
@seawinde
seawinde committed Nov 28, 2023
1 parent fc4acab commit fc48ab4
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 32 deletions.
43 changes: 43 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.common;

/**MaterializedViewException*/
public class MaterializedViewException extends UserException{

public MaterializedViewException(String msg, Throwable cause) {
super(msg, cause);
}

public MaterializedViewException(Throwable cause) {
super(cause);
}

public MaterializedViewException(String msg, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(msg, cause, enableSuppression, writableStackTrace);
}

public MaterializedViewException(String msg) {
super(msg);
}

public MaterializedViewException(InternalErrorCode errCode, String msg) {
super(errCode, msg);
}
}
100 changes: 80 additions & 20 deletions ...main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,26 @@
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionIndexMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.util.ExpressionUtils;

import com.clearspring.analytics.util.Lists;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

/**
Expand Down Expand Up @@ -72,7 +78,7 @@ protected List<Plan> rewrite(Plan queryPlan, CascadesContext cascadesContext) {
continue;
}
MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations());
if (MatchMode.NOT_MATCH == matchMode) {
if (MatchMode.COMPLETE != matchMode) {
continue;
}
List<RelationMapping> queryToViewTableMappings =
Expand All @@ -82,7 +88,7 @@ protected List<Plan> rewrite(Plan queryPlan, CascadesContext cascadesContext) {
SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo,
queryToViewSlotMapping);
// Can not compensate, bail out
if (compensatePredicates == null || compensatePredicates.isEmpty()) {
if (compensatePredicates.isEmpty()) {
continue;
}
Plan rewritedPlan;
Expand Down Expand Up @@ -170,36 +176,90 @@ protected SplitPredicate predicatesCompensate(
EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass();
EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass();
// viewEquivalenceClass to query based
EquivalenceClass viewEquivalenceClassQueryBased =
viewEquivalenceClass.permute(queryToViewSlotMapping.inverse().toSlotReferenceMap());
SplitPredicate splitPredicate = null;
if (queryEquivalenceClass.isEmpty()
&& viewEquivalenceClass.isEmpty()) {
BooleanLiteral trueLiteral = BooleanLiteral.of(true);
splitPredicate = SplitPredicate.of(trueLiteral, null, null);
Map<SlotReference, SlotReference> viewToQuerySlotMapping = queryToViewSlotMapping.inverse().toSlotReferenceMap();
EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMapping);
final List<Expression> equalCompensateConjunctions = new ArrayList<>();
if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) {
equalCompensateConjunctions.add(BooleanLiteral.of(true));
}
if (queryEquivalenceClass.isEmpty()
&& !viewEquivalenceClass.isEmpty()) {
return SplitPredicate.empty();
}
EquivalenceClassSetMapping equivalenceClassSetMapping =
EquivalenceClassSetMapping queryToViewEquivalenceMapping =
EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased);
// can not map all can not compensate
if (equivalenceClassSetMapping.getEquivalenceClassSetMap().size() !=
queryEquivalenceClass.getEquivalenceSetList().size()) {
// can not map all target equivalence class, can not compensate
if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size() <
viewEquivalenceClass.getEquivalenceSetList().size()) {
return SplitPredicate.empty();
}
// do compensate
Expression compensatePredicate;
equivalenceClassSetMapping.getEquivalenceClassSetMap().forEach(
equivalenceMapping -> {

// do equal compensate
Set<Set<SlotReference>> mappedQueryEquivalenceSet =
queryToViewEquivalenceMapping.getEquivalenceClassSetMap().keySet();
queryEquivalenceClass.getEquivalenceSetList().forEach(
queryEquivalenceSet -> {
// compensate the equivalence in query but not in view
if (!mappedQueryEquivalenceSet.contains(queryEquivalenceSet)) {
Iterator<SlotReference> iterator = queryEquivalenceSet.iterator();
SlotReference first = iterator.next();
while (iterator.hasNext()) {
Expression equals = new EqualTo(first, iterator.next());
equalCompensateConjunctions.add(equals);
}
} else {
// compensate the equivalence both in query and view, but query has more equivalence
Set<SlotReference> viewEquivalenceSet =
queryToViewEquivalenceMapping.getEquivalenceClassSetMap().get(queryEquivalenceSet);
Set<SlotReference> copiedQueryEquivalenceSet = new HashSet<>(queryEquivalenceSet);
copiedQueryEquivalenceSet.removeAll(viewEquivalenceSet);
SlotReference first = viewEquivalenceSet.iterator().next();
for (SlotReference slotReference : copiedQueryEquivalenceSet) {
Expression equals = new EqualTo(first, slotReference);
equalCompensateConjunctions.add(equals);
}
}
}
);
// TODO range predicates and residual predicates compensate, Simplify implementation.
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();

// range compensate
List<Expression> rangeCompensate = new ArrayList<>();
Expression queryRangePredicate = querySplitPredicate.getRangePredicate();
Expression viewRangePredicate = viewSplitPredicate.getRangePredicate();
Expression viewRangePredicateQueryBased =
ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping);

Set<Expression> queryRangeSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate));
Set<Expression> viewRangeQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased));
if (!queryRangeSet.containsAll(viewRangeQueryBasedSet)) {
return SplitPredicate.empty();
}
queryRangeSet.removeAll(viewRangeQueryBasedSet);
rangeCompensate.addAll(queryRangeSet);

// residual compensate
List<Expression> residualCompensate = new ArrayList<>();
Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate();
Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate();
Expression viewResidualPredicateQueryBased =
ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping);
Set<Expression> queryResidualSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate));
Set<Expression> viewResidualQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased));
if (!queryResidualSet.containsAll(viewResidualQueryBasedSet)) {
return SplitPredicate.empty();
}
queryResidualSet.removeAll(viewResidualQueryBasedSet);
residualCompensate.addAll(queryResidualSet);

// TODO range predicates and residual predicates compensate
return splitPredicate;
return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions),
rangeCompensate.isEmpty()? BooleanLiteral.of(true) : ExpressionUtils.and(rangeCompensate),
residualCompensate.isEmpty()? BooleanLiteral.of(true) : ExpressionUtils.and(residualCompensate));
}

/**
Expand Down
25 changes: 14 additions & 11 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,29 +35,28 @@
* StructInfo
*/
public class StructInfo {
private final List<CatalogRelation> relations;
private final Predicates predicates;
// Used by predicate compensation
private final EquivalenceClass equivalenceClass;
// source data
private final Plan originalPlan;
private final HyperGraph hyperGraph;
// derived data
private EquivalenceClass equivalenceClass;
private SplitPredicate splitPredicate;
private List<CatalogRelation> relations;
private Predicates predicates;

private StructInfo(List<CatalogRelation> relations,
Predicates predicates,
Plan originalPlan,
HyperGraph hyperGraph) {
private StructInfo(Plan originalPlan, HyperGraph hyperGraph) {
this.relations = relations;
this.predicates = predicates;
this.originalPlan = originalPlan;
this.hyperGraph = hyperGraph;
// construct equivalenceClass according to equals predicates
this.equivalenceClass = new EquivalenceClass();

List<Expression> shuttledExpression = ExpressionUtils.shuttleExpressionWithLineage(
predicates.getPredicates(), originalPlan).stream()
.map(Expression.class::cast)
.collect(Collectors.toList());
SplitPredicate splitPredicate = Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression));
this.splitPredicate = splitPredicate;
for (Expression expression : ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicate())) {
EqualTo equalTo = (EqualTo) expression;
equivalenceClass.addEquivalenceClass(
Expand All @@ -68,12 +67,12 @@ private StructInfo(List<CatalogRelation> relations,

public static StructInfo of(Plan originalPlan) {
// TODO build graph from original plan and get relations and predicates from graph
return new StructInfo(null, null, originalPlan, null);
return new StructInfo( originalPlan, null);
}

public static StructInfo of(Group group) {
// TODO build graph from original plan and get relations and predicates from graph
return new StructInfo(null, null, group.getLogicalExpression().getPlan(), null);
return new StructInfo(group.getLogicalExpression().getPlan(), null);
}

public List<CatalogRelation> getRelations() {
Expand All @@ -96,6 +95,10 @@ public HyperGraph getHyperGraph() {
return hyperGraph;
}

public SplitPredicate getSplitPredicate() {
return splitPredicate;
}

public List<? extends Expression> getExpressions() {
return originalPlan instanceof LogicalProject
? ((LogicalProject<Plan>) originalPlan).getProjects() : originalPlan.getOutput();
Expand Down
8 changes: 7 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
package org.apache.doris.nereids.util;

import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.MaterializedViewException;
import org.apache.doris.common.NereidsException;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
Expand Down Expand Up @@ -226,7 +228,11 @@ public static List<? extends Expression> shuttleExpressionWithLineage(List<? ext

plan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext);
// Replace expressions by expression map
return replaceContext.getReplacedExpressions();
List<Expression> replacedExpressions = replaceContext.getReplacedExpressions();
if (expressions.size() != replacedExpressions.size()) {
throw new NereidsException("shuttle expression fail", new MaterializedViewException("shuttle expression fail"));
}
return replacedExpressions;
}

/**
Expand Down

0 comments on commit fc48ab4

Please sign in to comment.