From fc48ab49e9eba25b1c06c0d62c961303e09ef435 Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 28 Nov 2023 15:05:12 +0800 Subject: [PATCH] tmp --- .../common/MaterializedViewException.java | 43 ++++++++ .../mv/AbstractMaterializedViewRule.java | 100 ++++++++++++++---- .../rules/exploration/mv/StructInfo.java | 25 +++-- .../doris/nereids/util/ExpressionUtils.java | 8 +- 4 files changed, 144 insertions(+), 32 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java b/fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java new file mode 100644 index 000000000000000..40c6a73537f82d2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +/**MaterializedViewException*/ +public class MaterializedViewException extends UserException{ + + public MaterializedViewException(String msg, Throwable cause) { + super(msg, cause); + } + + public MaterializedViewException(Throwable cause) { + super(cause); + } + + public MaterializedViewException(String msg, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(msg, cause, enableSuppression, writableStackTrace); + } + + public MaterializedViewException(String msg) { + super(msg); + } + + public MaterializedViewException(InternalErrorCode errCode, String msg) { + super(errCode, msg); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 4ad9e07ef8fdd5c..ea5e86f38f2af72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -25,20 +25,26 @@ import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionIndexMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.util.ExpressionUtils; +import com.clearspring.analytics.util.Lists; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -72,7 +78,7 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { continue; } MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations()); - if (MatchMode.NOT_MATCH == matchMode) { + if (MatchMode.COMPLETE != matchMode) { continue; } List queryToViewTableMappings = @@ -82,7 +88,7 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo, queryToViewSlotMapping); // Can not compensate, bail out - if (compensatePredicates == null || compensatePredicates.isEmpty()) { + if (compensatePredicates.isEmpty()) { continue; } Plan rewritedPlan; @@ -170,36 +176,90 @@ protected SplitPredicate predicatesCompensate( EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass(); EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass(); // viewEquivalenceClass to query based - EquivalenceClass viewEquivalenceClassQueryBased = - viewEquivalenceClass.permute(queryToViewSlotMapping.inverse().toSlotReferenceMap()); - SplitPredicate splitPredicate = null; - if (queryEquivalenceClass.isEmpty() - && viewEquivalenceClass.isEmpty()) { - BooleanLiteral trueLiteral = BooleanLiteral.of(true); - splitPredicate = SplitPredicate.of(trueLiteral, null, null); + Map viewToQuerySlotMapping = queryToViewSlotMapping.inverse().toSlotReferenceMap(); + EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMapping); + final List equalCompensateConjunctions = new ArrayList<>(); + if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) { + equalCompensateConjunctions.add(BooleanLiteral.of(true)); } if (queryEquivalenceClass.isEmpty() && !viewEquivalenceClass.isEmpty()) { return SplitPredicate.empty(); } - EquivalenceClassSetMapping equivalenceClassSetMapping = + EquivalenceClassSetMapping queryToViewEquivalenceMapping = EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased); - // can not map all can not compensate - if (equivalenceClassSetMapping.getEquivalenceClassSetMap().size() != - queryEquivalenceClass.getEquivalenceSetList().size()) { + // can not map all target equivalence class, can not compensate + if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size() < + viewEquivalenceClass.getEquivalenceSetList().size()) { return SplitPredicate.empty(); } - // do compensate - Expression compensatePredicate; - equivalenceClassSetMapping.getEquivalenceClassSetMap().forEach( - equivalenceMapping -> { - + // do equal compensate + Set> mappedQueryEquivalenceSet = + queryToViewEquivalenceMapping.getEquivalenceClassSetMap().keySet(); + queryEquivalenceClass.getEquivalenceSetList().forEach( + queryEquivalenceSet -> { + // compensate the equivalence in query but not in view + if (!mappedQueryEquivalenceSet.contains(queryEquivalenceSet)) { + Iterator iterator = queryEquivalenceSet.iterator(); + SlotReference first = iterator.next(); + while (iterator.hasNext()) { + Expression equals = new EqualTo(first, iterator.next()); + equalCompensateConjunctions.add(equals); + } + } else { + // compensate the equivalence both in query and view, but query has more equivalence + Set viewEquivalenceSet = + queryToViewEquivalenceMapping.getEquivalenceClassSetMap().get(queryEquivalenceSet); + Set copiedQueryEquivalenceSet = new HashSet<>(queryEquivalenceSet); + copiedQueryEquivalenceSet.removeAll(viewEquivalenceSet); + SlotReference first = viewEquivalenceSet.iterator().next(); + for (SlotReference slotReference : copiedQueryEquivalenceSet) { + Expression equals = new EqualTo(first, slotReference); + equalCompensateConjunctions.add(equals); + } + } } ); + // TODO range predicates and residual predicates compensate, Simplify implementation. + SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate(); + SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate(); + // range compensate + List rangeCompensate = new ArrayList<>(); + Expression queryRangePredicate = querySplitPredicate.getRangePredicate(); + Expression viewRangePredicate = viewSplitPredicate.getRangePredicate(); + Expression viewRangePredicateQueryBased = + ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping); + + Set queryRangeSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate)); + Set viewRangeQueryBasedSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased)); + if (!queryRangeSet.containsAll(viewRangeQueryBasedSet)) { + return SplitPredicate.empty(); + } + queryRangeSet.removeAll(viewRangeQueryBasedSet); + rangeCompensate.addAll(queryRangeSet); + + // residual compensate + List residualCompensate = new ArrayList<>(); + Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate(); + Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate(); + Expression viewResidualPredicateQueryBased = + ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping); + Set queryResidualSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate)); + Set viewResidualQueryBasedSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased)); + if (!queryResidualSet.containsAll(viewResidualQueryBasedSet)) { + return SplitPredicate.empty(); + } + queryResidualSet.removeAll(viewResidualQueryBasedSet); + residualCompensate.addAll(queryResidualSet); - // TODO range predicates and residual predicates compensate - return splitPredicate; + return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions), + rangeCompensate.isEmpty()? BooleanLiteral.of(true) : ExpressionUtils.and(rangeCompensate), + residualCompensate.isEmpty()? BooleanLiteral.of(true) : ExpressionUtils.and(residualCompensate)); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 926545a86dbb427..05a20ff276707dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -35,29 +35,28 @@ * StructInfo */ public class StructInfo { - private final List relations; - private final Predicates predicates; - // Used by predicate compensation - private final EquivalenceClass equivalenceClass; + // source data private final Plan originalPlan; private final HyperGraph hyperGraph; + // derived data + private EquivalenceClass equivalenceClass; + private SplitPredicate splitPredicate; + private List relations; + private Predicates predicates; - private StructInfo(List relations, - Predicates predicates, - Plan originalPlan, - HyperGraph hyperGraph) { + private StructInfo(Plan originalPlan, HyperGraph hyperGraph) { this.relations = relations; this.predicates = predicates; this.originalPlan = originalPlan; this.hyperGraph = hyperGraph; // construct equivalenceClass according to equals predicates this.equivalenceClass = new EquivalenceClass(); - List shuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( predicates.getPredicates(), originalPlan).stream() .map(Expression.class::cast) .collect(Collectors.toList()); SplitPredicate splitPredicate = Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression)); + this.splitPredicate = splitPredicate; for (Expression expression : ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicate())) { EqualTo equalTo = (EqualTo) expression; equivalenceClass.addEquivalenceClass( @@ -68,12 +67,12 @@ private StructInfo(List relations, public static StructInfo of(Plan originalPlan) { // TODO build graph from original plan and get relations and predicates from graph - return new StructInfo(null, null, originalPlan, null); + return new StructInfo( originalPlan, null); } public static StructInfo of(Group group) { // TODO build graph from original plan and get relations and predicates from graph - return new StructInfo(null, null, group.getLogicalExpression().getPlan(), null); + return new StructInfo(group.getLogicalExpression().getPlan(), null); } public List getRelations() { @@ -96,6 +95,10 @@ public HyperGraph getHyperGraph() { return hyperGraph; } + public SplitPredicate getSplitPredicate() { + return splitPredicate; + } + public List getExpressions() { return originalPlan instanceof LogicalProject ? ((LogicalProject) originalPlan).getProjects() : originalPlan.getOutput(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 3d2d1f5490c6745..b24f782b5da93a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -18,6 +18,8 @@ package org.apache.doris.nereids.util; import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.MaterializedViewException; +import org.apache.doris.common.NereidsException; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule; @@ -226,7 +228,11 @@ public static List shuttleExpressionWithLineage(List replacedExpressions = replaceContext.getReplacedExpressions(); + if (expressions.size() != replacedExpressions.size()) { + throw new NereidsException("shuttle expression fail", new MaterializedViewException("shuttle expression fail")); + } + return replacedExpressions; } /**