From ecdd70db8904bc623e034a590a5d32b5caba73bd Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Thu, 21 Sep 2023 17:20:25 +0800 Subject: [PATCH] [prune](partiiton)support prune partition when is auto partition with function call --- be/src/vec/sink/vtablet_sink.cpp | 4 +- fe/fe-core/src/main/cup/sql_parser.cup | 9 ++ .../doris/analysis/BinaryPredicate.java | 55 +++++++++++- .../org/apache/doris/analysis/IntLiteral.java | 18 ++++ .../apache/doris/analysis/PartitionDesc.java | 4 +- .../doris/analysis/PartitionExprUtil.java | 27 ++++-- .../org/apache/doris/analysis/SlotRef.java | 1 + .../apache/doris/catalog/PartitionInfo.java | 1 + .../apache/doris/planner/OlapScanNode.java | 14 ++- .../apache/doris/planner/OlapTableSink.java | 7 +- .../org/apache/doris/planner/ScanNode.java | 51 ++++++----- .../doris/planner/SingleNodePlanner.java | 87 ------------------- .../doris/planner/StreamLoadPlanner.java | 2 +- 13 files changed, 157 insertions(+), 123 deletions(-) diff --git a/be/src/vec/sink/vtablet_sink.cpp b/be/src/vec/sink/vtablet_sink.cpp index c6b9dbe673fc3a9..4caaca7c01a41e8 100644 --- a/be/src/vec/sink/vtablet_sink.cpp +++ b/be/src/vec/sink/vtablet_sink.cpp @@ -1578,10 +1578,10 @@ Status VOlapTableSink::send(RuntimeState* state, vectorized::Block* input_block, } else { // if there's projection of partition calc, we need to calc it first. auto [part_ctx, part_func] = _get_partition_function(); - int result_idx; + int result_idx = -1; if (_vpartition->is_projection_partition()) { // calc the start value of missing partition ranges. - part_func->execute(part_ctx.get(), block.get(), &result_idx); + RETURN_IF_ERROR(part_func->execute(part_ctx.get(), block.get(), &result_idx)); VLOG_DEBUG << "Partition-calculated block:" << block->dump_data(); // change the column to compare to transformed. _vpartition->set_transformed_slots({(uint16_t)result_idx}); diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index baec3cd18d00b43..fccebbe0b619b09 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -3248,6 +3248,15 @@ opt_partition ::= exprs.add(fnExpr); RESULT = RangePartitionDesc.createRangePartitionDesc(exprs, list); :} + | KW_AUTO KW_PARTITION KW_BY KW_RANGE function_name:functionName LPAREN expr_list:l COMMA + KW_INTERVAL expr:v ident:u RPAREN LPAREN opt_all_partition_desc_list:list RPAREN + {: + Expr fnExpr = FunctionCallExpr.functionWithIntervalConvert(functionName.getFunction().toLowerCase(), l.get(0), v, u); + ArrayList exprs = new ArrayList(); + exprs.add(fnExpr); + RESULT = RangePartitionDesc.createRangePartitionDesc(exprs, list); + :} + /* expr list partition */ | KW_AUTO KW_PARTITION KW_BY KW_LIST LPAREN expr_list:exprs RPAREN LPAREN opt_all_partition_desc_list:list RPAREN diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java index dd9253109e89d6e..047926f2854a7ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java @@ -46,6 +46,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.ArrayList; import java.util.Objects; /** @@ -535,12 +536,33 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException { // vectorizedAnalyze(analyzer); } + public Expr invokeFunctionExpr(ArrayList partitionExprs, Expr paramExpr) { + for (Expr partExpr : partitionExprs) { + if (partExpr instanceof FunctionCallExpr) { + FunctionCallExpr function = (FunctionCallExpr) partExpr.clone(); + ArrayList children = function.getChildren(); + for (int i = 0; i < children.size(); ++i) { + if (children.get(i) instanceof SlotRef) { + // when create partition have check only support one slotRef + function.setChild(i, paramExpr); + return ExpressionFunctions.INSTANCE.evalExpr(function); + } + } + } + } + return null; + } + /** * If predicate is of the form " ", returns expr, * otherwise returns null. Slotref may be wrapped in a CastExpr. + * now, when support auto create partition by function(column), so need check the " " + * because import data use function result to create partition, + * so when have a SQL of query, prune partition also need use this function */ - public Expr getSlotBinding(SlotId id) { + public Expr getSlotBinding(SlotId id, ArrayList partitionExprs) { SlotRef slotRef = null; + boolean isFunctionCallExpr = false; // check left operand if (getChild(0) instanceof SlotRef) { slotRef = (SlotRef) getChild(0); @@ -548,9 +570,25 @@ public Expr getSlotBinding(SlotId id) { if (((CastExpr) getChild(0)).canHashPartition()) { slotRef = (SlotRef) getChild(0).getChild(0); } + } else if (getChild(0) instanceof FunctionCallExpr) { + FunctionCallExpr left = (FunctionCallExpr) getChild(0); + if (partitionExprs != null && left.findEqual(partitionExprs) != null) { + ArrayList children = left.getChildren(); + for (int i = 0; i < children.size(); ++i) { + if (children.get(i) instanceof SlotRef) { + slotRef = (SlotRef) children.get(i); + isFunctionCallExpr = true; + break; + } + } + } } + if (slotRef != null && slotRef.getSlotId() == id) { slotIsleft = true; + if (isFunctionCallExpr) { + return invokeFunctionExpr(partitionExprs, getChild(1)); + } return getChild(1); } @@ -561,10 +599,25 @@ public Expr getSlotBinding(SlotId id) { if (((CastExpr) getChild(1)).canHashPartition()) { slotRef = (SlotRef) getChild(1).getChild(0); } + } else if (getChild(1) instanceof FunctionCallExpr) { + FunctionCallExpr left = (FunctionCallExpr) getChild(1); + if (partitionExprs != null && left.findEqual(partitionExprs) != null) { + ArrayList children = left.getChildren(); + for (int i = 0; i < children.size(); ++i) { + if (children.get(i) instanceof SlotRef) { + slotRef = (SlotRef) children.get(i); + isFunctionCallExpr = true; + break; + } + } + } } if (slotRef != null && slotRef.getSlotId() == id) { slotIsleft = false; + if (isFunctionCallExpr) { + return invokeFunctionExpr(partitionExprs, getChild(0)); + } return getChild(0); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java index 6e7c5d6f21a3b3b..c8356b1f165ade7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java @@ -194,6 +194,24 @@ public static IntLiteral createMaxValue(Type type) { return new IntLiteral(value); } + @Override + protected void analyzeImpl(Analyzer analyzer) throws AnalysisException { + //it's so strange, now in write/read function, not write type info + if (this.type.getPrimitiveType() == Type.INVALID.getPrimitiveType()) { + if (this.value <= TINY_INT_MAX && this.value >= TINY_INT_MIN) { + type = Type.TINYINT; + } else if (this.value <= SMALL_INT_MAX && this.value >= SMALL_INT_MIN) { + type = Type.SMALLINT; + } else if (this.value <= INT_MAX && this.value >= INT_MIN) { + type = Type.INT; + } else if (this.value <= BIG_INT_MAX && this.value >= BIG_INT_MIN) { + type = Type.BIGINT; + } else { + Preconditions.checkState(false, value); + } + } + } + @Override public boolean isMinValue() { switch (type.getPrimitiveType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java index ea66d4a0f246b69..0d365d9c2b70782 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java @@ -46,7 +46,9 @@ public class PartitionDesc { protected boolean isAutoCreatePartitions; protected PartitionType type; public static final ImmutableSet RANGE_PARTITION_FUNCTIONS = new ImmutableSortedSet.Builder( - String.CASE_INSENSITIVE_ORDER).add("date_trunc").add("date_ceil").add("date_floor") + String.CASE_INSENSITIVE_ORDER).add("date_trunc").add("date_ceil").add("date_floor").add("second_floor") + .add("minute_floor").add("hour_floor").add("day_floor").add("month_floor").add("year_floor") + .add("second_ceil").add("minute_ceil").add("hour_ceil").add("day_ceil").add("month_ceil").add("year_ceil") .build(); public PartitionDesc() {} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java index 750386162f67672..811e1db30de84de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java @@ -57,9 +57,9 @@ public static FunctionIntervalInfo getFunctionIntervalInfo(ArrayList parti throw new AnalysisException("now range partition only support FunctionCallExpr"); } FunctionCallExpr functionCallExpr = (FunctionCallExpr) e; - String fnName = functionCallExpr.getFnName().getFunction(); + String fnName = functionCallExpr.getFnName().getFunction().toLowerCase(); String timeUnit; - int interval; + long interval; if ("date_trunc".equalsIgnoreCase(fnName)) { List paramsExprs = functionCallExpr.getParams().exprs(); if (paramsExprs.size() != 2) { @@ -70,9 +70,22 @@ public static FunctionIntervalInfo getFunctionIntervalInfo(ArrayList parti throw new AnalysisException("date_trunc param of time unit is not string literal."); } timeUnit = ((StringLiteral) param).getStringValue().toLowerCase(); - interval = 1; + interval = 1L; + } else if (PartitionDesc.RANGE_PARTITION_FUNCTIONS.contains(fnName)) { + List paramsExprs = functionCallExpr.getParams().exprs(); + if (paramsExprs.size() != 3) { + throw new AnalysisException("date_floor/date_ceil params exprs size should be 3."); + } + Expr param = paramsExprs.get(1); + if (!(param instanceof IntLiteral)) { + throw new AnalysisException("date_floor/date_ceil param of interval must be int literal."); + } + //date_floor(event_day,interval 5 day) ---> day_floor(`event_day`, 5, '0001-01-01 00:00:00') + String[] splits = fnName.split("_"); + timeUnit = splits[0]; //day + interval = ((IntLiteral) param).getLongValue(); //5 } else { - throw new AnalysisException("now range partition only support date_trunc."); + throw new AnalysisException("now range partition only support date_trunc/date_floor/date_ceil."); } return partitionExprUtil.new FunctionIntervalInfo(timeUnit, interval); } @@ -80,7 +93,7 @@ public static FunctionIntervalInfo getFunctionIntervalInfo(ArrayList parti public static DateLiteral getRangeEnd(DateLiteral beginTime, FunctionIntervalInfo intervalInfo) throws AnalysisException { String timeUnit = intervalInfo.timeUnit; - int interval = intervalInfo.interval; + long interval = intervalInfo.interval; switch (timeUnit) { case "year": return beginTime.plusYears(interval); @@ -183,9 +196,9 @@ public static PartitionKeyDesc createPartitionKeyDescWithRange(DateLiteral begin public class FunctionIntervalInfo { public String timeUnit; - public int interval; + public long interval; - public FunctionIntervalInfo(String timeUnit, int interval) { + public FunctionIntervalInfo(String timeUnit, long interval) { this.timeUnit = timeUnit; this.interval = interval; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java index ae8eeabe63a5334..ef46f1f70dd1eca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -225,6 +225,7 @@ public String debugString() { MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); helper.add("slotDesc", desc != null ? desc.debugString() : "null"); helper.add("col", col); + helper.add("type1", type.toSql()); helper.add("label", label); helper.add("tblName", tblName != null ? tblName.toSql() : "null"); return helper.toString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index b7ca3c622cc7b04..24db05fc0eaa9a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -394,6 +394,7 @@ public void write(DataOutput out) throws IOException { out.writeInt(size); for (int i = 0; i < size; ++i) { Expr e = this.partitionExprs.get(i); + LOG.info("this.partitionExprs.get(i): " + e.debugString()); Expr.writeTo(e, out); } out.writeBoolean(isAutoCreatePartitions); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 806000c63731504..899a168a50e3238 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -536,6 +536,10 @@ public void init(Analyzer analyzer) throws UserException { // lazy evaluation, since stmt is a prepared statment isFromPrepareStmt = analyzer.getPrepareStmt() != null; if (!isFromPrepareStmt) { + if (olapTable.getPartitionInfo().enableAutomaticPartition()) { + partitionsInfo = olapTable.getPartitionInfo(); + analyzerPartitionExpr(analyzer, partitionsInfo); + } computeColumnsFilter(); computePartitionInfo(); } @@ -1116,7 +1120,7 @@ public List lazyEvaluateRangeLocations() throws UserExcepti // Lazy evaluation selectedIndexId = olapTable.getBaseIndexId(); // Only key columns - computeColumnsFilter(olapTable.getBaseSchemaKeyColumns()); + computeColumnsFilter(olapTable.getBaseSchemaKeyColumns(), olapTable.getPartitionInfo()); computePartitionInfo(); scanBackendIds.clear(); scanTabletIds.clear(); @@ -1388,6 +1392,13 @@ public void collectColumns(Analyzer analyzer, Set equivalenceColumns, Se } } + private void analyzerPartitionExpr(Analyzer analyzer, PartitionInfo partitionInfo) throws AnalysisException { + ArrayList exprs = partitionInfo.getPartitionExprs(); + for (Expr e : exprs) { + e.analyze(analyzer); + } + } + public TupleId getTupleId() { Preconditions.checkNotNull(desc); return desc.getId(); @@ -1585,3 +1596,4 @@ public boolean pushDownAggNoGroupingCheckCol(FunctionCallExpr aggExpr, Column co } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index a91fdb28f144233..38bc0272abf7944 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -354,14 +354,15 @@ private TOlapTablePartitionParam createPartition(long dbId, OlapTable table, Ana } } } + boolean enableAutomaticPartition = partitionInfo.enableAutomaticPartition(); // for auto create partition by function expr, there is no any partition firstly, // But this is required in thrift struct. - if (partitionIds.isEmpty()) { + if (enableAutomaticPartition && partitionIds.isEmpty()) { partitionParam.setDistributedColumns(getDistColumns(table.getDefaultDistributionInfo())); partitionParam.setPartitions(new ArrayList()); } ArrayList exprs = partitionInfo.getPartitionExprs(); - if (exprs != null && analyzer != null) { + if (enableAutomaticPartition && exprs != null && analyzer != null) { Analyzer funcAnalyzer = new Analyzer(analyzer.getEnv(), analyzer.getContext()); tupleDescriptor.setTable(table); funcAnalyzer.registerTupleDescriptor(tupleDescriptor); @@ -370,7 +371,7 @@ private TOlapTablePartitionParam createPartition(long dbId, OlapTable table, Ana } partitionParam.setPartitionFunctionExprs(Expr.treesToThrift(exprs)); } - partitionParam.setEnableAutomaticPartition(partitionInfo.enableAutomaticPartition()); + partitionParam.setEnableAutomaticPartition(enableAutomaticPartition); break; } case UNPARTITIONED: { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index ef42ae1e4eb8312..e63b67a3495d2a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -36,6 +36,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.NotImplementedException; @@ -61,6 +62,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -79,6 +81,7 @@ public abstract class ScanNode extends PlanNode { protected String sortColumn = null; protected Analyzer analyzer; protected List scanRangeLocations = Lists.newArrayList(); + protected PartitionInfo partitionsInfo = null; public ScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType) { super(id, desc.getId().asList(), planNodeName, statisticalType); @@ -154,23 +157,23 @@ public void updateRequiredSlots(PlanTranslatorContext context, Set requiredByProjectSlotIdSet) throws UserException { } - private void computeColumnFilter(Column column, SlotDescriptor slotDesc) { + private void computeColumnFilter(Column column, SlotDescriptor slotDesc, PartitionInfo partitionsInfo) { // Set `columnFilters` all the time because `DistributionPruner` also use this. // Maybe we could use `columnNameToRange` for `DistributionPruner` and // only create `columnFilters` when `partition_prune_algorithm_version` is 1. - PartitionColumnFilter keyFilter = createPartitionFilter(slotDesc, conjuncts); + PartitionColumnFilter keyFilter = createPartitionFilter(slotDesc, conjuncts, partitionsInfo); if (null != keyFilter) { columnFilters.put(column.getName(), keyFilter); } - ColumnRange columnRange = createColumnRange(slotDesc, conjuncts); + ColumnRange columnRange = createColumnRange(slotDesc, conjuncts, partitionsInfo); if (columnRange != null) { columnNameToRange.put(column.getName(), columnRange); } } // TODO(ML): move it into PrunerOptimizer - public void computeColumnsFilter(List columns) { + public void computeColumnsFilter(List columns, PartitionInfo partitionsInfo) { if (columns.size() > conjuncts.size()) { Set slotRefs = Sets.newHashSet(); for (Expr conjunct : conjuncts) { @@ -185,7 +188,7 @@ public void computeColumnsFilter(List columns) { if (column == null) { continue; } - computeColumnFilter(column, slotDesc); + computeColumnFilter(column, slotDesc, partitionsInfo); } } else { for (Column column : columns) { @@ -193,20 +196,21 @@ public void computeColumnsFilter(List columns) { if (null == slotDesc) { continue; } - computeColumnFilter(column, slotDesc); + computeColumnFilter(column, slotDesc, partitionsInfo); } } } public void computeColumnsFilter() { // for load scan node, table is null + // partitionsInfo maybe null for other scan node, eg: ExternalScanNode... if (desc.getTable() != null) { - computeColumnsFilter(desc.getTable().getBaseSchema()); + computeColumnsFilter(desc.getTable().getBaseSchema(), partitionsInfo); } } public static ColumnRange createColumnRange(SlotDescriptor desc, - List conjuncts) { + List conjuncts, PartitionInfo partitionsInfo) { ColumnRange result = ColumnRange.create(); for (Expr expr : conjuncts) { if (!expr.isBound(desc.getId())) { @@ -224,7 +228,7 @@ public static ColumnRange createColumnRange(SlotDescriptor desc, List> disjunctiveRanges = Lists.newArrayList(); Set hasIsNull = Sets.newHashSet(); boolean allMatch = disjunctivePredicates.stream().allMatch(e -> { - ColumnRanges ranges = expressionToRanges(e, desc); + ColumnRanges ranges = expressionToRanges(e, desc, partitionsInfo); switch (ranges.type) { case IS_NULL: hasIsNull.add(true); @@ -244,7 +248,7 @@ public static ColumnRange createColumnRange(SlotDescriptor desc, } } else { // Try to get column filter from conjunctive predicates. - ColumnRanges ranges = expressionToRanges(expr, desc); + ColumnRanges ranges = expressionToRanges(expr, desc, partitionsInfo); switch (ranges.type) { case IS_NULL: result.setHasConjunctiveIsNull(true); @@ -262,7 +266,7 @@ public static ColumnRange createColumnRange(SlotDescriptor desc, } public static ColumnRanges expressionToRanges(Expr expr, - SlotDescriptor desc) { + SlotDescriptor desc, PartitionInfo partitionsInfo) { if (expr instanceof IsNullPredicate) { IsNullPredicate isNullPredicate = (IsNullPredicate) expr; if (isNullPredicate.isSlotRefChildren() && !isNullPredicate.isNotNull()) { @@ -273,8 +277,10 @@ public static ColumnRanges expressionToRanges(Expr expr, List> result = Lists.newArrayList(); if (expr instanceof BinaryPredicate) { BinaryPredicate binPred = (BinaryPredicate) expr; - Expr slotBinding = binPred.getSlotBinding(desc.getId()); - + ArrayList partitionExprs = (partitionsInfo != null && partitionsInfo.enableAutomaticPartition()) + ? partitionsInfo.getPartitionExprs() + : null; + Expr slotBinding = binPred.getSlotBinding(desc.getId(), partitionExprs); if (slotBinding == null || !slotBinding.isConstant() || !(slotBinding instanceof LiteralExpr)) { return ColumnRanges.createFailure(); } @@ -327,15 +333,15 @@ public static ColumnRanges expressionToRanges(Expr expr, ColumnRanges rightChildRange = null; switch (compoundPredicate.getOp()) { case AND: - leftChildRange = expressionToRanges(compoundPredicate.getChild(0), desc); - rightChildRange = expressionToRanges(compoundPredicate.getChild(1), desc); + leftChildRange = expressionToRanges(compoundPredicate.getChild(0), desc, partitionsInfo); + rightChildRange = expressionToRanges(compoundPredicate.getChild(1), desc, partitionsInfo); return leftChildRange.intersectRanges(rightChildRange); case OR: - leftChildRange = expressionToRanges(compoundPredicate.getChild(0), desc); - rightChildRange = expressionToRanges(compoundPredicate.getChild(1), desc); + leftChildRange = expressionToRanges(compoundPredicate.getChild(0), desc, partitionsInfo); + rightChildRange = expressionToRanges(compoundPredicate.getChild(1), desc, partitionsInfo); return leftChildRange.unionRanges(rightChildRange); case NOT: - leftChildRange = expressionToRanges(compoundPredicate.getChild(0), desc); + leftChildRange = expressionToRanges(compoundPredicate.getChild(0), desc, partitionsInfo); return leftChildRange.complementOfRanges(); default: throw new RuntimeException("unknown OP in compound predicate: " @@ -350,7 +356,8 @@ public static ColumnRanges expressionToRanges(Expr expr, } } - private PartitionColumnFilter createPartitionFilter(SlotDescriptor desc, List conjuncts) { + private PartitionColumnFilter createPartitionFilter(SlotDescriptor desc, List conjuncts, + PartitionInfo partitionsInfo) { PartitionColumnFilter partitionColumnFilter = null; for (Expr expr : conjuncts) { if (!expr.isBound(desc.getId())) { @@ -363,7 +370,11 @@ private PartitionColumnFilter createPartitionFilter(SlotDescriptor desc, List partitionExprs = (partitionsInfo != null && partitionsInfo.enableAutomaticPartition()) + ? partitionsInfo.getPartitionExprs() + : null; + Expr slotBinding = binPredicate.getSlotBinding(desc.getId(), partitionExprs); + if (slotBinding == null || !slotBinding.isConstant() || !(slotBinding instanceof LiteralExpr)) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index e0bdfdce4de734d..e4edb3517a5acf9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -34,13 +34,9 @@ import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.GroupByClause; import org.apache.doris.analysis.GroupingInfo; -import org.apache.doris.analysis.InPredicate; import org.apache.doris.analysis.InlineViewRef; -import org.apache.doris.analysis.IsNullPredicate; import org.apache.doris.analysis.JoinOperator; import org.apache.doris.analysis.LateralViewRef; -import org.apache.doris.analysis.LiteralExpr; -import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.QueryStmt; import org.apache.doris.analysis.SelectStmt; import org.apache.doris.analysis.SetOperationStmt; @@ -1513,89 +1509,6 @@ private TupleDescriptor createResultTupleDescriptor(SelectStmt selectStmt, return tupleDesc; } - // no need to remove? - public static PartitionColumnFilter createPartitionFilter(SlotDescriptor desc, List conjuncts) { - PartitionColumnFilter partitionColumnFilter = null; - for (Expr expr : conjuncts) { - if (!expr.isBound(desc.getId())) { - continue; - } - if (expr instanceof BinaryPredicate) { - BinaryPredicate binPredicate = (BinaryPredicate) expr; - Expr slotBinding = binPredicate.getSlotBinding(desc.getId()); - if (slotBinding == null || !slotBinding.isConstant()) { - continue; - } - if (binPredicate.getOp() == BinaryPredicate.Operator.NE - || !(slotBinding instanceof LiteralExpr)) { - continue; - } - if (null == partitionColumnFilter) { - partitionColumnFilter = new PartitionColumnFilter(); - } - LiteralExpr literal = (LiteralExpr) slotBinding; - BinaryPredicate.Operator op = binPredicate.getOp(); - if (!binPredicate.slotIsLeft()) { - op = op.commutative(); - } - switch (op) { - case EQ: - partitionColumnFilter.setLowerBound(literal, true); - partitionColumnFilter.setUpperBound(literal, true); - break; - case LE: - partitionColumnFilter.setUpperBound(literal, true); - if (null == partitionColumnFilter.lowerBound) { - partitionColumnFilter.lowerBoundInclusive = true; - } - break; - case LT: - partitionColumnFilter.setUpperBound(literal, false); - if (null == partitionColumnFilter.lowerBound) { - partitionColumnFilter.lowerBoundInclusive = true; - } - break; - case GE: - partitionColumnFilter.setLowerBound(literal, true); - break; - case GT: - partitionColumnFilter.setLowerBound(literal, false); - break; - default: - break; - } - } else if (expr instanceof InPredicate) { - InPredicate inPredicate = (InPredicate) expr; - if (!inPredicate.isLiteralChildren() || inPredicate.isNotIn()) { - continue; - } - if (!(inPredicate.getChild(0).unwrapExpr(false) instanceof SlotRef)) { - // If child(0) of the in predicate is not a SlotRef, - // then other children of in predicate should not be used as a condition for partition prune. - continue; - } - if (null == partitionColumnFilter) { - partitionColumnFilter = new PartitionColumnFilter(); - } - partitionColumnFilter.setInPredicate(inPredicate); - } else if (expr instanceof IsNullPredicate) { - IsNullPredicate isNullPredicate = (IsNullPredicate) expr; - if (!isNullPredicate.isSlotRefChildren() || isNullPredicate.isNotNull()) { - continue; - } - - // If we meet a IsNull predicate on partition column, then other predicates are useless - // eg: (xxxx) and (col is null), only the IsNull predicate has an effect on partition pruning. - partitionColumnFilter = new PartitionColumnFilter(); - NullLiteral nullLiteral = new NullLiteral(); - partitionColumnFilter.setLowerBound(nullLiteral, true); - partitionColumnFilter.setUpperBound(nullLiteral, true); - break; - } - } - LOG.debug("partitionColumnFilter: {}", partitionColumnFilter); - return partitionColumnFilter; - } /** * Returns plan tree for an inline view ref: diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index 77fe4697f1918fe..9fea7d6ea268f60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -555,7 +555,7 @@ private List getAllPartitionIds() throws DdlException, AnalysisException { if (null == slotDesc) { continue; } - ColumnRange columnRange = ScanNode.createColumnRange(slotDesc, conjuncts); + ColumnRange columnRange = ScanNode.createColumnRange(slotDesc, conjuncts, partitionInfo); if (columnRange != null) { columnNameToRange.put(column.getName(), columnRange); }