From 1b3c437613538638872ce1b2359c8c71611b8fd3 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 27 Nov 2024 10:54:07 +0800 Subject: [PATCH] [fix](mtmv) Fix rewritten by mv fail when filter expression in query has alias but actual same --- .../mv/AbstractMaterializedViewRule.java | 4 +- .../exploration/mv/HyperGraphComparator.java | 19 ++- .../mv/LogicalCompatibilityContext.java | 96 +++++++------- .../rules/exploration/mv/StructInfo.java | 20 +-- .../mv/mapping/ExpressionMapping.java | 121 ++++++++++++++++++ .../rules/exploration/mv/mapping/Mapping.java | 69 ++++++++++ .../mv/mapping/RelationMapping.java | 77 +---------- 7 files changed, 269 insertions(+), 137 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 8e9ef1eaa97b7a..3167432048b428 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -180,8 +180,8 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca () -> String.format("matchMode is %s", matchMode)); return rewriteResults; } - List queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(), - viewStructInfo.getRelations()); + List queryToViewTableMappings = RelationMapping.EMPTY_INSTANCE.generate( + queryStructInfo.getRelations(), viewStructInfo.getRelations()); // if any relation in query and view can not map, bail out. if (queryToViewTableMappings == null) { materializationContext.recordFailReason(queryStructInfo, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index 868f97949c0705..9b5c957cd4bdbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.JoinUtils; +import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -75,9 +76,9 @@ public class HyperGraphComparator { private final Map> pullUpViewExprWithEdge = new HashMap<>(); private final LogicalCompatibilityContext logicalCompatibilityContext; // this records the slots which needs to reject null - // the key is the target join which should reject null, the value is a pair, the first value of the pair is the - // join type, the second value is also a pair which left represents the slots in the left of join that should - // reject null, right represents the slots in the right of join that should reject null. + // the key is the view join edge which should reject null, the value is a pair, the first value of the pair is the + // query join type, the second value is also a pair which left represents the slots in the left of view join that + // should reject null, right represents the slots in the right of view join that should reject null. private final Map, Set>>> inferredViewEdgeWithCond = new HashMap<>(); private List viewJoinEdgesAfterInferring; private List viewFilterEdgesAfterInferring; @@ -104,6 +105,18 @@ public HyperGraphComparator(HyperGraph queryHyperGraph, HyperGraph viewHyperGrap */ public static ComparisonResult isLogicCompatible(HyperGraph queryHG, HyperGraph viewHG, LogicalCompatibilityContext ctx) { + ComparisonResult logicCompatible; + if (ctx.getQueryToViewFilterEdgeExpressionMappingList().size() > 1) { + for (BiMap expressionBiMap : ctx.getQueryToViewFilterEdgeExpressionMappingList()) { + LogicalCompatibilityContext eachCtx = ctx.withQueryToViewFilterEdgeExpressionMapping(expressionBiMap); + logicCompatible = + new HyperGraphComparator(queryHG, viewHG, eachCtx).isLogicCompatible(); + if (logicCompatible.isInvalid()) { + continue; + } + return logicCompatible; + } + } return new HyperGraphComparator(queryHG, viewHG, ctx).isLogicCompatible(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index ca13c9701dabc2..3bf6fc6a1ef788 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -20,24 +20,21 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition; +import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.Mapping.MappedRelation; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; -import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.RelationId; -import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Suppliers; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -51,8 +48,27 @@ public class LogicalCompatibilityContext { private final Supplier> queryToViewJoinEdgeExpressionMappingSupplier; private final Supplier> queryToViewNodeExpressionMappingSupplier; private final Supplier> queryToViewFilterEdgeExpressionMappingSupplier; - @Deprecated - private BiMap queryToViewAllExpressionMapping; + private final List> queryToViewFilterEdgeExpressionMappingList; + + private LogicalCompatibilityContext( + BiMap queryToViewNodeMapping, + BiMap queryToViewNodeIDMapping, + ObjectId planNodeId, + BiMap queryToViewJoinEdgeExpressionMapping, + BiMap queryToViewNodeExpressionMapping, + BiMap queryToViewFilterEdgeExpressionMapping, + List> queryToViewFilterEdgeExpressionMappingList) { + this.queryToViewNodeMapping = queryToViewNodeMapping; + this.queryToViewNodeIDMapping = queryToViewNodeIDMapping; + this.planNodeId = planNodeId; + this.queryToViewJoinEdgeExpressionMappingSupplier = + Suppliers.memoize(() -> queryToViewJoinEdgeExpressionMapping); + this.queryToViewNodeExpressionMappingSupplier = + Suppliers.memoize(() -> queryToViewNodeExpressionMapping); + this.queryToViewFilterEdgeExpressionMappingSupplier = + Suppliers.memoize(() -> queryToViewFilterEdgeExpressionMapping); + this.queryToViewFilterEdgeExpressionMappingList = queryToViewFilterEdgeExpressionMappingList; + } /** * LogicalCompatibilityContext @@ -62,17 +78,17 @@ private LogicalCompatibilityContext(BiMap queryT StructInfo viewStructInfo) { this.queryToViewJoinEdgeExpressionMappingSupplier = - Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping, + Suppliers.memoize(() -> ExpressionMapping.generateExpressionMappingFirst(viewToQuerySlotMapping, queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE), viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE))); this.queryToViewNodeExpressionMappingSupplier = - Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping, + Suppliers.memoize(() -> ExpressionMapping.generateExpressionMappingFirst(viewToQuerySlotMapping, queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE), viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE))); this.queryToViewFilterEdgeExpressionMappingSupplier = - Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping, + Suppliers.memoize(() -> ExpressionMapping.generateExpressionMappingFirst(viewToQuerySlotMapping, queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE), viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE))); @@ -82,6 +98,11 @@ private LogicalCompatibilityContext(BiMap queryT this.planNodeId = queryStructInfo.getTopPlan().getGroupExpression() .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); + + this.queryToViewFilterEdgeExpressionMappingList = ExpressionMapping.EMPTY_INSTANCE + .generateExpressionMappingCombine(viewToQuerySlotMapping, + queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE), + viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE)); } public BiMap getQueryToViewNodeMapping() { @@ -134,50 +155,21 @@ public static LogicalCompatibilityContext from(RelationMapping relationMapping, viewStructInfo); } - private static BiMap generateExpressionMapping( - Map viewToQuerySlotMapping, - Map queryShuttledExprToExprMap, - Map viewShuttledExprToExprMap) { - final Map viewEdgeToConjunctsMapQueryBased = new HashMap<>(); - BiMap queryToViewEdgeMapping = HashBiMap.create(); - if (queryShuttledExprToExprMap == null || viewShuttledExprToExprMap == null - || queryShuttledExprToExprMap.isEmpty() || viewShuttledExprToExprMap.isEmpty()) { - return queryToViewEdgeMapping; - } - viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> { - viewEdgeToConjunctsMapQueryBased.put( - orderSlotAsc(ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping)), expr); - }); - queryShuttledExprToExprMap.forEach((exprSet, edge) -> { - Expression viewExpr = viewEdgeToConjunctsMapQueryBased.get(orderSlotAsc(exprSet)); - if (viewExpr != null) { - queryToViewEdgeMapping.put(edge, viewExpr); - } - }); - return queryToViewEdgeMapping; - } - - private static Expression orderSlotAsc(Expression expression) { - return expression.accept(ExpressionSlotOrder.INSTANCE, null); + /** + * Construct LogicalCompatibilityContext with queryToViewFilterEdgeExpressionMapping + */ + public LogicalCompatibilityContext withQueryToViewFilterEdgeExpressionMapping( + BiMap queryToViewFilterEdgeExpressionMapping) { + return new LogicalCompatibilityContext(this.queryToViewNodeMapping, + this.queryToViewNodeIDMapping, + this.planNodeId, + this.queryToViewJoinEdgeExpressionMappingSupplier.get(), + this.queryToViewNodeExpressionMappingSupplier.get(), queryToViewFilterEdgeExpressionMapping, + this.queryToViewFilterEdgeExpressionMappingList); } - private static final class ExpressionSlotOrder extends DefaultExpressionRewriter { - public static final ExpressionSlotOrder INSTANCE = new ExpressionSlotOrder(); - - @Override - public Expression visitEqualTo(EqualTo equalTo, Void context) { - if (!(equalTo.getArgument(0) instanceof NamedExpression) - || !(equalTo.getArgument(1) instanceof NamedExpression)) { - return equalTo; - } - NamedExpression left = (NamedExpression) equalTo.getArgument(0); - NamedExpression right = (NamedExpression) equalTo.getArgument(1); - if (right.getExprId().asInt() < left.getExprId().asInt()) { - return new EqualTo(right, left); - } else { - return equalTo; - } - } + public List> getQueryToViewFilterEdgeExpressionMappingList() { + return queryToViewFilterEdgeExpressionMappingList; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 526ec7030d2db5..fd05d94e0baf4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -61,8 +61,10 @@ import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer; import org.apache.doris.nereids.util.ExpressionUtils; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import java.util.ArrayList; @@ -110,8 +112,10 @@ public class StructInfo { private SplitPredicate splitPredicate; private EquivalenceClass equivalenceClass; // Key is the expression shuttled and the value is the origin expression + // Sometimes origin expressions are different and shuttled expression is same + // Such as origin expressions are l_partkey#0 > 1 and l_partkey#10 > 1 and shuttled expression is l_partkey#10 > 1 // this is for building LogicalCompatibilityContext later. - private final Map> shuttledExpressionsToExpressionsMap; + private final Map> shuttledExpressionsToExpressionsMap; // Record the exprId and the corresponding expr map, this is used by expression shuttled private final Map namedExprIdAndExprMapping; private final List planOutputShuttledExpressions; @@ -123,7 +127,7 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG Plan bottomPlan, List relations, Map relationIdStructInfoNodeMap, @Nullable Predicates predicates, - Map> shuttledExpressionsToExpressionsMap, + Map> shuttledExpressionsToExpressionsMap, Map namedExprIdAndExprMapping, BitSet tableIdSet, SplitPredicate splitPredicate, @@ -168,7 +172,7 @@ public StructInfo withTableBitSet(BitSet tableBitSet) { private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, Plan topPlan, - Map> shuttledExpressionsToExpressionsMap, + Map> shuttledExpressionsToExpressionsMap, Map namedExprIdAndExprMapping, List relations, Map relationIdStructInfoNodeMap, @@ -311,7 +315,7 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable // collect struct info fromGraph List relationList = new ArrayList<>(); Map relationIdStructInfoNodeMap = new LinkedHashMap<>(); - Map> shuttledHashConjunctsToConjunctsMap = + Map> shuttledHashConjunctsToConjunctsMap = new LinkedHashMap<>(); Map namedExprIdAndExprMapping = new LinkedHashMap<>(); BitSet tableBitSet = new BitSet(); @@ -401,18 +405,18 @@ public Map getRelationIdStructInfoNodeMap() { return relationIdStructInfoNodeMap; } - public Map> getShuttledExpressionsToExpressionsMap() { + public Map> getShuttledExpressionsToExpressionsMap() { return shuttledExpressionsToExpressionsMap; } private static void putShuttledExpressionsToExpressionsMap( - Map> shuttledExpressionsToExpressionsMap, + Map> shuttledExpressionsToExpressionsMap, ExpressionPosition expressionPosition, Expression key, Expression value) { - Map expressionExpressionMap = shuttledExpressionsToExpressionsMap.get( + Multimap expressionExpressionMap = shuttledExpressionsToExpressionsMap.get( expressionPosition); if (expressionExpressionMap == null) { - expressionExpressionMap = new LinkedHashMap<>(); + expressionExpressionMap = HashMultimap.create(); shuttledExpressionsToExpressionsMap.put(expressionPosition, expressionExpressionMap); } expressionExpressionMap.put(key, value); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java index 8c77eacfaf0945..2c8f7f4b62a6a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java @@ -18,11 +18,19 @@ package org.apache.doris.nereids.rules.exploration.mv.mapping; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; @@ -31,11 +39,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; /** * Expression mapping, maybe one expression map to multi expression */ public class ExpressionMapping extends Mapping { + + public static final ExpressionMapping EMPTY_INSTANCE = + ExpressionMapping.generate(ImmutableList.of(), ImmutableList.of()); private final Multimap expressionMapping; public ExpressionMapping(Multimap expressionMapping) { @@ -102,6 +114,115 @@ public static ExpressionMapping generate( return new ExpressionMapping(expressionMultiMap); } + /** + * Generate expression mapping, just map the first element in queryShuttledExprToExprMap and + * viewShuttledExprToExprMap + * Such as + * queryShuttledExprToExprMap is {c1#0 : [c1#1, c1#2]} + * viewShuttledExprToExprMap is {c1#3 : [c1#4, c1#4]} + * if viewToQuerySlotMapping is {c1#0 : c1#3} + * the mapping result is {c1#1 : c1#4} + * */ + public static BiMap generateExpressionMappingFirst( + Map viewToQuerySlotMapping, + Multimap queryShuttledExprToExprMap, + Multimap viewShuttledExprToExprMap) { + final Map viewEdgeToConjunctsMapQueryBased = new HashMap<>(); + BiMap queryToViewEdgeMapping = HashBiMap.create(); + if (queryShuttledExprToExprMap == null || viewShuttledExprToExprMap == null + || queryShuttledExprToExprMap.isEmpty() || viewShuttledExprToExprMap.isEmpty()) { + return queryToViewEdgeMapping; + } + viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> { + viewEdgeToConjunctsMapQueryBased.put( + orderSlotAsc(ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping)), expr); + }); + queryShuttledExprToExprMap.forEach((exprSet, edge) -> { + Expression viewExpr = viewEdgeToConjunctsMapQueryBased.get(orderSlotAsc(exprSet)); + if (viewExpr != null) { + queryToViewEdgeMapping.putIfAbsent(edge, viewExpr); + } + }); + return queryToViewEdgeMapping; + } + + /** + * Generate expression mapping, map the unique combine element in queryShuttledExprToExprMap and + * viewShuttledExprToExprMap + * Such as + * queryShuttledExprToExprMap is {c1#0 : [c1#1, c1#2]} + * viewShuttledExprToExprMap is {c1#3 : [c1#4, c1#5]} + * if viewToQuerySlotMapping is {c1#0 : c1#3} + * the mapping result is {c1#1 : c1#4, c1#2 : c1#5} + * */ + public List> generateExpressionMappingCombine( + Map viewToQuerySlotMapping, + Multimap queryShuttledExprToExprMap, + Multimap viewShuttledExprToExprMap) { + List> result = new ArrayList<>(); + if (queryShuttledExprToExprMap == null || viewShuttledExprToExprMap == null + || queryShuttledExprToExprMap.isEmpty() || viewShuttledExprToExprMap.isEmpty()) { + return result; + } + + final Multimap viewEdgeToConjunctsMapQueryBased = HashMultimap.create(); + viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> { + viewEdgeToConjunctsMapQueryBased.put( + orderSlotAsc(ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping)), expr); + }); + + List>> expressionMappingList = new ArrayList<>(); + for (Entry> queryEntry : queryShuttledExprToExprMap.asMap().entrySet()) { + Collection viewExpressions = viewEdgeToConjunctsMapQueryBased.get( + orderSlotAsc(queryEntry.getKey())); + if (queryEntry.getValue().size() == 1 && viewExpressions.size() == 1) { + HashBiMap expressionMapping = HashBiMap.create(); + expressionMapping.put(queryEntry.getValue().iterator().next(), viewExpressions.iterator().next()); + expressionMappingList.add(ImmutableList.of(expressionMapping)); + continue; + } + List> permutationMappings = getUniquePermutation( + queryEntry.getValue().toArray(new Expression[0]), + viewExpressions.toArray(new Expression[0]), Expression.class); + expressionMappingList.add(permutationMappings); + } + return Lists.cartesianProduct(expressionMappingList).stream() + .map(listMapping -> { + if (listMapping.size() == 1) { + return listMapping.get(0); + } + HashBiMap expressionMapping = HashBiMap.create(); + for (BiMap expressionBiMap : listMapping) { + expressionMapping.putAll(expressionBiMap); + } + return expressionMapping; + }) + .collect(ImmutableList.toImmutableList()); + } + + private static Expression orderSlotAsc(Expression expression) { + return expression.accept(ExpressionSlotOrder.INSTANCE, null); + } + + private static final class ExpressionSlotOrder extends DefaultExpressionRewriter { + public static final ExpressionSlotOrder INSTANCE = new ExpressionSlotOrder(); + + @Override + public Expression visitEqualTo(EqualTo equalTo, Void context) { + if (!(equalTo.getArgument(0) instanceof NamedExpression) + || !(equalTo.getArgument(1) instanceof NamedExpression)) { + return equalTo; + } + NamedExpression left = (NamedExpression) equalTo.getArgument(0); + NamedExpression right = (NamedExpression) equalTo.getArgument(1); + if (right.getExprId().asInt() < left.getExprId().asInt()) { + return new EqualTo(right, left); + } else { + return equalTo; + } + } + } + @Override public String toString() { return Utils.toSqlString("ExpressionMapping", "expressionMapping", expressionMapping); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java index d9192569a0cbca..7957fc0beb6c40 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java @@ -17,15 +17,20 @@ package org.apache.doris.nereids.rules.exploration.mv.mapping; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableList; +import java.lang.reflect.Array; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,6 +43,70 @@ */ public abstract class Mapping { + /** + * Permutation and remove duplicated element + * For example: + * Given [1, 4, 5] and [191, 194, 195] + * This would return + * [ + * [(1, 191) (4, 194) (5, 195)], + * [(1, 191) (4, 195) (5, 194)], + * [(1, 194) (4, 191) (5, 195)], + * [(1, 194) (4, 195) (5, 191)], + * [(1, 195) (4, 191) (5, 194)], + * [(1, 195) (4, 194) (5, 191)] + * ] + * */ + protected List> getUniquePermutation(M[] left, M[] right, Class type) { + boolean needSwap = left.length > right.length; + if (needSwap) { + M[] temp = left; + left = right; + right = temp; + } + + boolean[] used = new boolean[right.length]; + M[] current = (M[]) Array.newInstance(type, left.length); + List> results = new ArrayList<>(); + backtrack(left, right, 0, used, current, results); + if (needSwap) { + List> tmpResults = results; + results = new ArrayList<>(); + for (Pair relation : tmpResults) { + results.add(Pair.of(relation.value(), relation.key())); + } + } + List> mappingPowerList = new ArrayList<>(); + for (Pair combination : results) { + BiMap combinationBiMap = HashBiMap.create(); + M[] key = combination.key(); + M[] value = combination.value(); + int length = Math.min(key.length, value.length); + for (int i = 0; i < length; i++) { + combinationBiMap.put(key[i], value[i]); + } + mappingPowerList.add(combinationBiMap); + } + return mappingPowerList; + } + + protected void backtrack(M[] left, M[] right, int index, + boolean[] used, M[] current, List> results) { + if (index == left.length) { + results.add(Pair.of(Arrays.copyOf(left, left.length), Arrays.copyOf(current, current.length))); + return; + } + + for (int i = 0; i < right.length; i++) { + if (!used[i]) { + used[i] = true; + current[index] = right[i]; + backtrack(left, right, index + 1, used, current, results); + used[i] = false; + } + } + } + /** * The relation for mapping */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java index d71e128e9f166d..0d363aebda3a69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java @@ -19,11 +19,9 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.constraint.TableIdentifier; -import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableBiMap.Builder; @@ -31,7 +29,6 @@ import com.google.common.collect.Lists; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Set; @@ -44,6 +41,7 @@ */ public class RelationMapping extends Mapping { + public static final RelationMapping EMPTY_INSTANCE = RelationMapping.of(ImmutableBiMap.of()); private final ImmutableBiMap mappedRelationMap; public RelationMapping(ImmutableBiMap mappedRelationMap) { @@ -61,7 +59,7 @@ public static RelationMapping of(ImmutableBiMap /** * Generate mapping according to source and target relation */ - public static List generate(List sources, List targets) { + public List generate(List sources, List targets) { // Construct tmp map, key is the table qualifier, value is the corresponding catalog relations HashMultimap sourceTableRelationIdMap = HashMultimap.create(); for (CatalogRelation relation : sources) { @@ -98,20 +96,9 @@ public static List generate(List sources, List // {tableA0 -> tableA3, tableA1 -> tableA2} // ] // query is select * from tableA0, tableA1, tableA4 - List> relationMappingPowerList = new ArrayList<>(); - List> combinations = getUniquePermutation( - sourceMappedRelations.toArray(sourceMappedRelations.toArray(new MappedRelation[0])), - targetMappedRelations.toArray(new MappedRelation[0])); - for (Pair combination : combinations) { - BiMap combinationBiMap = HashBiMap.create(); - MappedRelation[] key = combination.key(); - MappedRelation[] value = combination.value(); - int length = Math.min(key.length, value.length); - for (int i = 0; i < length; i++) { - combinationBiMap.put(key[i], value[i]); - } - relationMappingPowerList.add(combinationBiMap); - } + List> relationMappingPowerList = getUniquePermutation( + sourceMappedRelations.toArray(new MappedRelation[0]), + targetMappedRelations.toArray(new MappedRelation[0]), MappedRelation.class); mappedRelations.add(relationMappingPowerList); } // mappedRelations product and merge into each relationMapping @@ -153,58 +140,4 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(mappedRelationMap); } - - /** - * Permutation and remove duplicated element - * For example: - * Given [1, 4, 5] and [191, 194, 195] - * This would return - * [ - * [(1, 191) (4, 194) (5, 195)], - * [(1, 191) (4, 195) (5, 194)], - * [(1, 194) (4, 191) (5, 195)], - * [(1, 194) (4, 195) (5, 191)], - * [(1, 195) (4, 191) (5, 194)], - * [(1, 195) (4, 194) (5, 191)] - * ] - * */ - private static List> getUniquePermutation( - MappedRelation[] left, MappedRelation[] right) { - boolean needSwap = left.length > right.length; - if (needSwap) { - MappedRelation[] temp = left; - left = right; - right = temp; - } - - boolean[] used = new boolean[right.length]; - MappedRelation[] current = new MappedRelation[left.length]; - List> results = new ArrayList<>(); - backtrack(left, right, 0, used, current, results); - if (needSwap) { - List> tmpResults = results; - results = new ArrayList<>(); - for (Pair relation : tmpResults) { - results.add(Pair.of(relation.value(), relation.key())); - } - } - return results; - } - - private static void backtrack(MappedRelation[] left, MappedRelation[] right, int index, - boolean[] used, MappedRelation[] current, List> results) { - if (index == left.length) { - results.add(Pair.of(Arrays.copyOf(left, left.length), Arrays.copyOf(current, current.length))); - return; - } - - for (int i = 0; i < right.length; i++) { - if (!used[i]) { - used[i] = true; - current[index] = right[i]; - backtrack(left, right, index + 1, used, current, results); - used[i] = false; - } - } - } }