From 4a7e2911c7f3790c49d493d5ed9a54d026d73067 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 10:04:33 +0800 Subject: [PATCH] branch-2.1: [fix](catelog) Unifies partition items string #45669 (#45700) Cherry-picked from #45669 --------- Co-authored-by: walter --- .../java/org/apache/doris/catalog/Env.java | 6 +----- .../doris/catalog/ListPartitionItem.java | 19 +++++++++++++++---- .../apache/doris/catalog/PartitionInfo.java | 10 ++++++++++ .../apache/doris/catalog/PartitionItem.java | 3 +++ .../apache/doris/catalog/PartitionKey.java | 5 +++++ .../doris/catalog/RangePartitionItem.java | 12 +++++++++++- .../common/proc/EsPartitionsProcDir.java | 2 +- .../doris/common/proc/PartitionsProcDir.java | 2 +- .../transaction/DatabaseTransactionMgr.java | 16 ++++------------ 9 files changed, 51 insertions(+), 24 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 423092d6169baa..0bbe662ec6e8e6 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -6168,11 +6168,7 @@ private static void getTableMeta(OlapTable olapTable, TGetMetaDBMeta dbMeta) { long partitionId = partition.getId(); partitionMeta.setId(partitionId); partitionMeta.setName(partition.getName()); - String partitionRange = ""; - if (tblPartitionInfo.getType() == PartitionType.RANGE - || tblPartitionInfo.getType() == PartitionType.LIST) { - partitionRange = tblPartitionInfo.getItem(partitionId).getItems().toString(); - } + String partitionRange = tblPartitionInfo.getPartitionRangeString(partitionId); partitionMeta.setRange(partitionRange); partitionMeta.setVisibleVersion(partition.getVisibleVersion()); // partitionMeta.setTemp（partition.isTemp()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java index 2db1531de9b37f..d4e781322e186a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java @@ -59,7 +59,12 @@ public List getItems() { } public String getItemsString() { - return toString(); + // ATTN: DO NOT EDIT unless unless you explicitly guarantee compatibility + // between different versions. + // + // the ccr syncer depends on this string to identify partitions between two + // clusters (cluster versions may be different). + return getItems().toString(); } public String getItemsSql() { @@ -119,9 +124,10 @@ public boolean isGreaterThanSpecifiedTime(int pos, Optional dateFormatOp partitionKey.toString(), pos)); } - if (!isDefaultPartition() && MTMVUtil.getExprTimeSec(partitionKey.getKeys().get(pos), dateFormatOptional) - >= nowTruncSubSec) { - // As long as one of the partitionKeys meets the requirements, this partition needs to be retained + if (!isDefaultPartition() + && MTMVUtil.getExprTimeSec(partitionKey.getKeys().get(pos), dateFormatOptional) >= nowTruncSubSec) { + // As long as one of the partitionKeys meets the requirements, this partition + // needs to be retained return true; } } @@ -178,6 +184,11 @@ public int hashCode() { @Override public String toString() { + // ATTN: DO NOT EDIT unless unless you explicitly guarantee compatibility + // between different versions. + // + // the ccr syncer depends on this string to identify partitions between two + // clusters (cluster versions may be different). StringBuilder builder = new StringBuilder(); builder.append("partitionKeys: ["); for (PartitionKey partitionKey : partitionKeys) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index 8f148188a5a882..19be4783336948 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -161,6 +161,16 @@ public PartitionItem getItem(long partitionId) { return item; } + // Get the unique string of the partition range. + public String getPartitionRangeString(long partitionId) { + String partitionRange = ""; + if (getType() == PartitionType.RANGE || getType() == PartitionType.LIST) { + PartitionItem item = getItem(partitionId); + partitionRange = item.getItemsString(); + } + return partitionRange; + } + public PartitionItem getItemOrAnalysisException(long partitionId) throws AnalysisException { PartitionItem item = idToItem.get(partitionId); if (item == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java index 386f7537b03d08..e562e13cc82874 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionItem.java @@ -29,6 +29,9 @@ public abstract class PartitionItem implements Comparable, Writab public static final Comparator> ITEM_MAP_ENTRY_COMPARATOR = Comparator.comparing(o -> ((ListPartitionItem) o.getValue()).getItems().iterator().next()); + // get the unique string of the partition item. + public abstract String getItemsString(); + public abstract T getItems(); public abstract PartitionItem getIntersect(PartitionItem newItem); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index 401111af66a0be..100f41022452c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -324,6 +324,11 @@ public String toSql() { @Override public String toString() { + // ATTN: DO NOT EDIT unless unless you explicitly guarantee compatibility + // between different versions. + // + // the ccr syncer depends on this string to identify partitions between two + // clusters (cluster versions may be different). StringBuilder builder = new StringBuilder(); builder.append("types: ["); builder.append(Joiner.on(", ").join(types)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java index e7f2a9cab5dbf9..09ca1a3519f05d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java @@ -47,7 +47,12 @@ public Range getItems() { } public String getItemsString() { - return toString(); + // ATTN: DO NOT EDIT unless unless you explicitly guarantee compatibility + // between different versions. + // + // the ccr syncer depends on this string to identify partitions between two + // clusters (cluster versions may be different). + return partitionKeyRange.toString(); } public String getItemsSql() { @@ -126,6 +131,11 @@ public boolean equals(Object obj) { @Override public String toString() { + // ATTN: DO NOT EDIT unless unless you explicitly guarantee compatibility + // between different versions. + // + // the ccr syncer depends on this string to identify partitions between two + // clusters (cluster versions may be different). return partitionKeyRange.toString(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java index 87e7fa449f13a2..ed5cfc18d13e48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java @@ -93,7 +93,7 @@ public ProcResult fetchResult() throws AnalysisException { } partitionInfo.add(joiner.join(colNames)); // partition key partitionInfo.add( - rangePartitionInfo.getItem(esShardPartitions.getPartitionId()).getItems().toString()); // range + rangePartitionInfo.getItem(esShardPartitions.getPartitionId()).getItemsString()); // range partitionInfo.add("-"); // dis partitionInfo.add(esShardPartitions.getShardRoutings().size()); // shards partitionInfo.add(1); // replica num diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index 6b5a63a49e4005..3bd9fffaf493ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -323,7 +323,7 @@ private List, TRow>> getPartitionInfosInrernal() throws An String colNamesStr = joiner.join(colNames); partitionInfo.add(colNamesStr); trow.addToColumnValue(new TCell().setStringVal(colNamesStr)); - String itemStr = tblPartitionInfo.getItem(partitionId).getItems().toString(); + String itemStr = tblPartitionInfo.getPartitionRangeString(partitionId); partitionInfo.add(itemStr); trow.addToColumnValue(new TCell().setStringVal(itemStr)); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index 61721e7d89258f..9db515300aed1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -27,7 +27,6 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Partition.PartitionState; import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; @@ -1433,12 +1432,9 @@ protected void unprotectedPreCommitTransaction2PC(TransactionState transactionSt TableCommitInfo tableCommitInfo = new TableCommitInfo(tableId); PartitionInfo tblPartitionInfo = table.getPartitionInfo(); for (long partitionId : tableToPartition.get(tableId)) { - String partitionRange = ""; - if (tblPartitionInfo.getType() == PartitionType.RANGE - || tblPartitionInfo.getType() == PartitionType.LIST) { - partitionRange = tblPartitionInfo.getItem(partitionId).getItems().toString(); - } - PartitionCommitInfo partitionCommitInfo = new PartitionCommitInfo(partitionId, partitionRange, -1, -1, + String partitionRange = tblPartitionInfo.getPartitionRangeString(partitionId); + PartitionCommitInfo partitionCommitInfo = new PartitionCommitInfo( + partitionId, partitionRange, -1, -1, table.isTemporaryPartition(partitionId)); tableCommitInfo.addPartitionCommitInfo(partitionCommitInfo); } @@ -1475,11 +1471,7 @@ protected void unprotectedCommitTransaction(TransactionState transactionState, S PartitionInfo tblPartitionInfo = table.getPartitionInfo(); for (long partitionId : tableToPartition.get(tableId)) { Partition partition = table.getPartition(partitionId); - String partitionRange = ""; - if (tblPartitionInfo.getType() == PartitionType.RANGE - || tblPartitionInfo.getType() == PartitionType.LIST) { - partitionRange = tblPartitionInfo.getItem(partitionId).getItems().toString(); - } + String partitionRange = tblPartitionInfo.getPartitionRangeString(partitionId); PartitionCommitInfo partitionCommitInfo = new PartitionCommitInfo(partitionId, partitionRange, partition.getNextVersion(), System.currentTimeMillis() /* use as partition visible time */,