diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html index fd788301eea9..9bbed03bd948 100644 --- a/docs/layouts/shortcodes/generated/core_configuration.html +++ b/docs/layouts/shortcodes/generated/core_configuration.html @@ -575,6 +575,12 @@ Duration The expiration interval of a partition. A partition will be expired if it‘s lifetime is over this value. Partition time is extracted from the partition value. + +
partition.legacy-name
+ true + Boolean + The legacy partition name is using `toString` fpr all types. If false, using cast to string for all types. +
partition.mark-done-action
"success-file" diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java index d965fd142deb..18dae0d38416 100644 --- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java @@ -250,6 +250,14 @@ public class CoreOptions implements Serializable { "The default partition name in case the dynamic partition" + " column value is null/empty string."); + public static final ConfigOption PARTITION_GENERATE_LEGCY_NAME = + key("partition.legacy-name") + .booleanType() + .defaultValue(true) + .withDescription( + "The legacy partition name is using `toString` fpr all types. If false, using " + + "cast to string for all types."); + public static final ConfigOption SNAPSHOT_NUM_RETAINED_MIN = key("snapshot.num-retained.min") .intType() @@ -1539,6 +1547,10 @@ public String partitionDefaultName() { return options.get(PARTITION_DEFAULT_NAME); } + public boolean legacyPartitionName() { + return options.get(PARTITION_GENERATE_LEGCY_NAME); + } + public boolean sortBySize() { return options.get(SORT_RANG_STRATEGY) == RangeStrategy.SIZE; } diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/AbstractCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/AbstractCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/AbstractCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/AbstractCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastExecutor.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastExecutor.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastExecutor.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastExecutor.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastExecutors.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastExecutors.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastExecutors.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastExecutors.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastFieldGetter.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastFieldGetter.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastFieldGetter.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastFieldGetter.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastRulePredicate.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastRulePredicate.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastRulePredicate.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastRulePredicate.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastedRow.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastedRow.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastedRow.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastedRow.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/DefaultValueRow.java b/paimon-common/src/main/java/org/apache/paimon/casting/DefaultValueRow.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/DefaultValueRow.java rename to paimon-common/src/main/java/org/apache/paimon/casting/DefaultValueRow.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java b/paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java rename to paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java index 881f0f4d1053..1bc8eecc88d1 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java @@ -20,6 +20,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; import java.util.Arrays; @@ -35,11 +36,18 @@ public class InternalRowPartitionComputer { protected final String defaultPartValue; protected final String[] partitionColumns; protected final InternalRow.FieldGetter[] partitionFieldGetters; + protected final List types; + protected final boolean legacyPartitionName; public InternalRowPartitionComputer( - String defaultPartValue, RowType rowType, String[] partitionColumns) { + String defaultPartValue, + RowType rowType, + String[] partitionColumns, + boolean legacyPartitionName) { this.defaultPartValue = defaultPartValue; this.partitionColumns = partitionColumns; + this.types = rowType.getFieldTypes(); + this.legacyPartitionName = legacyPartitionName; List columnList = rowType.getFieldNames(); this.partitionFieldGetters = Arrays.stream(partitionColumns) @@ -56,7 +64,8 @@ public LinkedHashMap generatePartValues(InternalRow in) { for (int i = 0; i < partitionFieldGetters.length; i++) { Object field = partitionFieldGetters[i].getFieldOrNull(in); - String partitionValue = field != null ? field.toString() : null; + String partitionValue = + TypeUtils.castPartitionValueToString(field, types.get(i), legacyPartitionName); if (StringUtils.isNullOrWhitespaceOnly(partitionValue)) { partitionValue = defaultPartValue; } diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java index 8d8d84953da7..046e3d4b3e67 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java @@ -18,6 +18,8 @@ package org.apache.paimon.utils; +import org.apache.paimon.casting.CastExecutor; +import org.apache.paimon.casting.CastExecutors; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Decimal; import org.apache.paimon.data.GenericArray; @@ -88,6 +90,24 @@ public static RowType project(RowType inputType, List names) { .collect(Collectors.toList())); } + public static String castPartitionValueToString( + Object value, DataType type, boolean legacyPartitionName) { + if (value == null) { + return null; + } + if (legacyPartitionName) { + return value.toString(); + } + + CastExecutor castExecutor = + (CastExecutor) CastExecutors.resolve(type, VarCharType.STRING_TYPE); + if (castExecutor == null) { + throw new UnsupportedOperationException(type + " is not supported"); + } + Object result = castExecutor.cast(value); + return result == null ? null : result.toString(); + } + public static Object castFromString(String s, DataType type) { return castFromStringInternal(s, type, false); } diff --git a/paimon-core/src/test/java/org/apache/paimon/casting/CastExecutorTest.java b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java similarity index 100% rename from paimon-core/src/test/java/org/apache/paimon/casting/CastExecutorTest.java rename to paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java index 53bd1291f1f0..8118484539ec 100644 --- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java @@ -105,7 +105,8 @@ public FileStorePathFactory pathFactory() { options.partitionDefaultName(), options.fileFormat().getFormatIdentifier(), options.dataFilePrefix(), - options.changelogFilePrefix()); + options.changelogFilePrefix(), + options.legacyPartitionName()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java b/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java index cc4579898401..2e3aa965f91a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java @@ -206,7 +206,8 @@ private Map format2PathFactory() { options.partitionDefaultName(), format, options.dataFilePrefix(), - options.changelogFilePrefix()))); + options.changelogFilePrefix(), + options.legacyPartitionName()))); return pathFactoryMap; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java index dab20d642cb9..d63887030090 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java @@ -88,6 +88,7 @@ public abstract class AbstractFileStoreWrite implements FileStoreWrite { protected CompactionMetrics compactionMetrics = null; protected final String tableName; private boolean isInsertOnly; + private boolean legacyPartitionName; protected AbstractFileStoreWrite( SnapshotManager snapshotManager, @@ -97,7 +98,8 @@ protected AbstractFileStoreWrite( String tableName, int totalBuckets, RowType partitionType, - int writerNumberMax) { + int writerNumberMax, + boolean legacyPartitionName) { this.snapshotManager = snapshotManager; this.scan = scan; this.indexFactory = indexFactory; @@ -107,6 +109,7 @@ protected AbstractFileStoreWrite( this.writers = new HashMap<>(); this.tableName = tableName; this.writerNumberMax = writerNumberMax; + this.legacyPartitionName = legacyPartitionName; } @Override @@ -469,7 +472,8 @@ private List scanExistingFileMetas( ? "partition " + getPartitionComputer( partitionType, - PARTITION_DEFAULT_NAME.defaultValue()) + PARTITION_DEFAULT_NAME.defaultValue(), + legacyPartitionName) .generatePartValues(partition) : "table"; throw new RuntimeException( diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java index b7feeead4bbb..bbde3fd48580 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java @@ -75,7 +75,8 @@ public MemoryFileStoreWrite( tableName, options.bucket(), partitionType, - options.writeMaxWritersToSpill()); + options.writeMaxWritersToSpill(), + options.legacyPartitionName()); this.options = options; this.cacheManager = new CacheManager(options.lookupCacheMaxMemory()); } diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java index df442383a3fa..612565c728cf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java @@ -56,11 +56,13 @@ public FileStorePathFactory( String defaultPartValue, String formatIdentifier, String dataFilePrefix, - String changelogFilePrefix) { + String changelogFilePrefix, + boolean legacyPartitionName) { this.root = root; this.uuid = UUID.randomUUID().toString(); - this.partitionComputer = getPartitionComputer(partitionType, defaultPartValue); + this.partitionComputer = + getPartitionComputer(partitionType, defaultPartValue, legacyPartitionName); this.formatIdentifier = formatIdentifier; this.dataFilePrefix = dataFilePrefix; this.changelogFilePrefix = changelogFilePrefix; @@ -78,9 +80,10 @@ public Path root() { @VisibleForTesting public static InternalRowPartitionComputer getPartitionComputer( - RowType partitionType, String defaultPartValue) { + RowType partitionType, String defaultPartValue, boolean legacyPartitionName) { String[] partitionColumns = partitionType.getFieldNames().toArray(new String[0]); - return new InternalRowPartitionComputer(defaultPartValue, partitionType, partitionColumns); + return new InternalRowPartitionComputer( + defaultPartValue, partitionType, partitionColumns, legacyPartitionName); } public Path newManifestFile() { diff --git a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java index 3c89552310f6..de593e80bb54 100644 --- a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java @@ -228,7 +228,8 @@ protected KeyValueFileWriterFactory createWriterFactory(String pathStr, String f CoreOptions.PARTITION_DEFAULT_NAME.defaultValue(), format, CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue()); int suggestedFileSize = ThreadLocalRandom.current().nextInt(8192) + 1024; FileIO fileIO = FileIOFinder.find(path); Options options = new Options(); @@ -244,7 +245,8 @@ protected KeyValueFileWriterFactory createWriterFactory(String pathStr, String f CoreOptions.PARTITION_DEFAULT_NAME.defaultValue(), CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue())); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue())); return KeyValueFileWriterFactory.builder( fileIO, diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java index 31be9346c49b..87a6f2e45fc1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java @@ -146,7 +146,8 @@ protected ManifestFile createManifestFile(String pathStr) { "default", CoreOptions.FILE_FORMAT.defaultValue(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()), + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue()), Long.MAX_VALUE, null) .create(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index a11f3e485053..30e8e7b20f8e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -102,7 +102,8 @@ private ManifestFile createManifestFile(String pathStr) { "default", CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue()); int suggestedFileSize = ThreadLocalRandom.current().nextInt(8192) + 1024; FileIO fileIO = FileIOFinder.find(path); return new ManifestFile.Factory( diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java index faf1004e1619..f4703c1ff9df 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java @@ -106,7 +106,8 @@ private ManifestList createManifestList(String pathStr) { "default", CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue()); return new ManifestList.Factory(FileIOFinder.find(path), avro, "zstd", pathFactory, null) .create(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java index 994367d0cd06..26bbd28e42ed 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java @@ -87,7 +87,8 @@ public void testCreateDataFilePathFactoryWithPartition() { "default", CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue()); assertPartition("20211224", 16, pathFactory, "/dt=20211224/hr=16"); assertPartition("20211224", null, pathFactory, "/dt=20211224/hr=default"); @@ -124,6 +125,7 @@ public static FileStorePathFactory createNonPartFactory(Path root) { PARTITION_DEFAULT_NAME.defaultValue(), CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue()); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java index 633fa9c4bde1..c1e7db6b2633 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java @@ -1075,9 +1075,12 @@ private List getPartitionSpecs( getPartitionEntries(table, tablePath, partitionSpec); org.apache.paimon.types.RowType partitionRowType = table.schema().logicalPartitionType(); + CoreOptions options = new CoreOptions(table.options()); InternalRowPartitionComputer partitionComputer = FileStorePathFactory.getPartitionComputer( - partitionRowType, new CoreOptions(table.options()).partitionDefaultName()); + partitionRowType, + options.partitionDefaultName(), + options.legacyPartitionName()); return partitionEntries.stream() .map( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java index 2c5d55f203dc..d0825bcdb752 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java @@ -71,7 +71,8 @@ public static PartitionMarkDone create( new InternalRowPartitionComputer( coreOptions.partitionDefaultName(), table.schema().logicalPartitionType(), - table.partitionKeys().toArray(new String[0])); + table.partitionKeys().toArray(new String[0]), + coreOptions.legacyPartitionName()); PartitionMarkDoneTrigger trigger = PartitionMarkDoneTrigger.create(coreOptions, isRestored, stateStore); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java index aab8666fe998..762a14ea1516 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java @@ -107,7 +107,8 @@ public TestChangelogDataReadWrite(String root) { "default", CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue()); this.snapshotManager = new SnapshotManager(LocalFileIO.create(), new Path(root)); this.commitUser = UUID.randomUUID().toString(); } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java index b006f09c99a4..6ec3b086db4d 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java @@ -56,11 +56,13 @@ public class HiveMetastoreClient implements MetastoreClient { ClientPool clients) throws TException, InterruptedException { this.identifier = identifier; + CoreOptions options = new CoreOptions(schema.options()); this.partitionComputer = new InternalRowPartitionComputer( - new CoreOptions(schema.options()).partitionDefaultName(), + options.partitionDefaultName(), schema.logicalPartitionType(), - schema.partitionKeys().toArray(new String[0])); + schema.partitionKeys().toArray(new String[0]), + options.legacyPartitionName()); this.clients = clients; this.sd = diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala index cbdbd67a6de0..54970bfe3cb2 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala @@ -18,6 +18,7 @@ package org.apache.paimon.spark +import org.apache.paimon.CoreOptions import org.apache.paimon.metastore.MetastoreClient import org.apache.paimon.operation.FileStoreCommit import org.apache.paimon.table.FileStoreTable @@ -31,7 +32,6 @@ import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.connector.catalog.SupportsAtomicPartitionManagement import org.apache.spark.sql.types.StructType -import java.util import java.util.{Map => JMap, Objects, UUID} import scala.collection.JavaConverters._ @@ -52,7 +52,8 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement { val rowDataPartitionComputer = new InternalRowPartitionComputer( fileStoreTable.coreOptions().partitionDefaultName(), partitionRowType, - table.partitionKeys().asScala.toArray) + table.partitionKeys().asScala.toArray, + CoreOptions.fromMap(table.options()).legacyPartitionName) rows.map { r => diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala index 0087ffb740f2..bff69aa6fb92 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala @@ -83,7 +83,8 @@ case class DeleteFromPaimonTableCommand( val rowDataPartitionComputer = new InternalRowPartitionComputer( table.coreOptions().partitionDefaultName(), table.schema().logicalPartitionType(), - table.partitionKeys.asScala.toArray + table.partitionKeys.asScala.toArray, + table.coreOptions().legacyPartitionName() ) val dropPartitions = matchedPartitions.map { partition => rowDataPartitionComputer.generatePartValues(partition).asScala.asJava diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala index 91df04e6dc47..b44a66fce3ff 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala @@ -49,12 +49,15 @@ private[spark] trait StreamHelper { private lazy val partitionSchema: StructType = SparkTypeUtils.fromPaimonRowType(TypeUtils.project(table.rowType(), table.partitionKeys())) - private lazy val partitionComputer: InternalRowPartitionComputer = + private lazy val partitionComputer: InternalRowPartitionComputer = { + val options = new CoreOptions(table.options) new InternalRowPartitionComputer( - new CoreOptions(table.options).partitionDefaultName, + options.partitionDefaultName, TypeUtils.project(table.rowType(), table.partitionKeys()), - table.partitionKeys().asScala.toArray + table.partitionKeys().asScala.toArray, + options.legacyPartitionName() ) + } // Used to get the initial offset. lazy val streamScanStartingContext: StartingContext = streamScan.startingContext() diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java index 7af83186ef6e..d82f92af55bb 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java @@ -130,7 +130,8 @@ protected void foreachIndexReader(Consumer consumer) new CoreOptions(new Options()).partitionDefaultName(), CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue()); Table table = fileSystemCatalog.getTable(Identifier.create("db", "T")); ReadBuilder readBuilder = table.newReadBuilder(); diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala index f1215ede4eeb..0006e45ec4e5 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala +++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala @@ -27,7 +27,7 @@ import org.apache.spark.SparkException import org.apache.spark.sql.Row import org.junit.jupiter.api.Assertions -import java.sql.Timestamp +import java.sql.{Date, Timestamp} import java.time.LocalDateTime abstract class DDLTestBase extends PaimonSparkTestBase { @@ -495,4 +495,55 @@ abstract class DDLTestBase extends PaimonSparkTestBase { }.getMessage assert(error.contains("Unsupported partition transform")) } + + test("Fix partition column generate wrong partition spec") { + Seq(true, false).foreach { + legacyPartName => + withTable("p_t") { + spark.sql(s""" + |CREATE TABLE p_t ( + | id BIGINT, + | c1 STRING + |) using paimon + |PARTITIONED BY (day binary) + |tblproperties('partition.legacy-name'='$legacyPartName'); + |""".stripMargin) + + if (legacyPartName) { + spark.sql("insert into table p_t values(1, 'a', cast('2021' as binary))") + intercept[Exception] { + spark.sql("SELECT * FROM p_t").collect() + } + } else { + spark.sql("insert into table p_t values(1, 'a', cast('2021' as binary))") + checkAnswer(spark.sql("SELECT * FROM p_t"), Row(1, "a", "2021".getBytes)) + val path = spark.sql("SELECT __paimon_file_path FROM p_t").collect() + assert(path.length == 1) + assert(path.head.getString(0).contains("/day=2021/")) + } + } + + withTable("p_t") { + spark.sql(s""" + |CREATE TABLE p_t ( + | id BIGINT, + | c1 STRING + |) using paimon + |PARTITIONED BY (day date) + |tblproperties('partition.legacy-name'='$legacyPartName'); + |""".stripMargin) + + spark.sql("insert into table p_t values(1, 'a', cast('2021-01-01' as date))") + checkAnswer(spark.sql("SELECT * FROM p_t"), Row(1, "a", Date.valueOf("2021-01-01"))) + + val path = spark.sql("SELECT __paimon_file_path FROM p_t").collect() + assert(path.length == 1) + if (legacyPartName) { + assert(path.head.getString(0).contains("/day=18628/")) + } else { + assert(path.head.getString(0).contains("/day=2021-01-01/")) + } + } + } + } }