diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java index 74fc73ab52fb..1544e879c79d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java @@ -46,7 +46,7 @@ public class ManifestCommittableSerializerCompatibilityTest { @Test - public void testProduction() throws IOException { + public void testCompatibilityToV3CommitV6() throws IOException { SimpleStats keyStats = new SimpleStats( singleColumn("min_key"), @@ -105,10 +105,19 @@ public void testProduction() throws IOException { byte[] bytes = serializer.serialize(manifestCommittable); ManifestCommittable deserialized = serializer.deserialize(3, bytes); assertThat(deserialized).isEqualTo(manifestCommittable); + + byte[] oldBytes = + IOUtils.readFully( + ManifestCommittableSerializerCompatibilityTest.class + .getClassLoader() + .getResourceAsStream("compatibility/manifest-committable-v6"), + true); + deserialized = serializer.deserialize(3, oldBytes); + assertThat(deserialized).isEqualTo(manifestCommittable); } @Test - public void testCompatibilityToVersion5() throws IOException { + public void testCompatibilityToV3CommitV5() throws IOException { SimpleStats keyStats = new SimpleStats( singleColumn("min_key"), @@ -167,18 +176,18 @@ public void testCompatibilityToVersion5() throws IOException { byte[] bytes = serializer.serialize(manifestCommittable); ManifestCommittable deserialized = serializer.deserialize(3, bytes); assertThat(deserialized).isEqualTo(manifestCommittable); - byte[] v2Bytes = + byte[] oldBytes = IOUtils.readFully( ManifestCommittableSerializerCompatibilityTest.class .getClassLoader() .getResourceAsStream("compatibility/manifest-committable-v5"), true); - deserialized = serializer.deserialize(3, v2Bytes); + deserialized = serializer.deserialize(3, oldBytes); assertThat(deserialized).isEqualTo(manifestCommittable); } @Test - public void testCompatibilityToVersion4() throws IOException { + public void testCompatibilityToV3CommitV4() throws IOException { SimpleStats keyStats = new SimpleStats( singleColumn("min_key"), @@ -244,12 +253,12 @@ public void testCompatibilityToVersion4() throws IOException { .getClassLoader() .getResourceAsStream("compatibility/manifest-committable-v4"), true); - deserialized = serializer.deserialize(2, v2Bytes); + deserialized = serializer.deserialize(3, v2Bytes); assertThat(deserialized).isEqualTo(manifestCommittable); } @Test - public void testCompatibilityToVersion3() throws IOException { + public void testCompatibilityToV3CommitV3() throws IOException { SimpleStats keyStats = new SimpleStats( singleColumn("min_key"), @@ -309,18 +318,18 @@ public void testCompatibilityToVersion3() throws IOException { ManifestCommittable deserialized = serializer.deserialize(3, bytes); assertThat(deserialized).isEqualTo(manifestCommittable); - byte[] v2Bytes = + byte[] oldBytes = IOUtils.readFully( ManifestCommittableSerializerCompatibilityTest.class .getClassLoader() .getResourceAsStream("compatibility/manifest-committable-v3"), true); - deserialized = serializer.deserialize(2, v2Bytes); + deserialized = serializer.deserialize(3, oldBytes); assertThat(deserialized).isEqualTo(manifestCommittable); } @Test - public void testCompatibilityToVersion2() throws IOException { + public void testCompatibilityToV2CommitV2() throws IOException { SimpleStats keyStats = new SimpleStats( singleColumn("min_key"), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/SplitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/SplitTest.java index 9f895548dae3..a4e581b701ea 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/SplitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/SplitTest.java @@ -109,7 +109,7 @@ public void testSerializer() throws IOException { } @Test - public void testSerializerNormal() throws Exception { + public void testSerializerCompatibleV1() throws Exception { SimpleStats keyStats = new SimpleStats( singleColumn("min_key"), @@ -138,14 +138,11 @@ public void testSerializerNormal() throws Exception { Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")), 11L, new byte[] {1, 2, 4}, - FileSource.COMPACT, - Arrays.asList("field1", "field2", "field3"), - "hdfs:///path/to/warehouse"); + null, + null, + null); List dataFiles = Collections.singletonList(dataFile); - DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, 33L); - List deletionFiles = Collections.singletonList(deletionFile); - BinaryRow partition = new BinaryRow(1); BinaryRowWriter binaryRowWriter = new BinaryRowWriter(partition); binaryRowWriter.writeString(0, BinaryString.fromString("aaaaa")); @@ -157,15 +154,25 @@ public void testSerializerNormal() throws Exception { .withPartition(partition) .withBucket(20) .withDataFiles(dataFiles) - .withDataDeletionFiles(deletionFiles) .withBucketPath("my path") .build(); assertThat(InstantiationUtil.clone(split)).isEqualTo(split); + + byte[] v2Bytes = + IOUtils.readFully( + SplitTest.class + .getClassLoader() + .getResourceAsStream("compatibility/datasplit-v1"), + true); + + DataSplit actual = + InstantiationUtil.deserializeObject(v2Bytes, DataSplit.class.getClassLoader()); + assertThat(actual).isEqualTo(split); } @Test - public void testSerializerCompatibleV1() throws Exception { + public void testSerializerCompatibleV2() throws Exception { SimpleStats keyStats = new SimpleStats( singleColumn("min_key"), @@ -194,7 +201,7 @@ public void testSerializerCompatibleV1() throws Exception { Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")), 11L, new byte[] {1, 2, 4}, - null, + FileSource.COMPACT, null, null); List dataFiles = Collections.singletonList(dataFile); @@ -213,11 +220,13 @@ public void testSerializerCompatibleV1() throws Exception { .withBucketPath("my path") .build(); + assertThat(InstantiationUtil.clone(split)).isEqualTo(split); + byte[] v2Bytes = IOUtils.readFully( SplitTest.class .getClassLoader() - .getResourceAsStream("compatibility/datasplit-v1"), + .getResourceAsStream("compatibility/datasplit-v2"), true); DataSplit actual = @@ -226,7 +235,7 @@ public void testSerializerCompatibleV1() throws Exception { } @Test - public void testSerializerCompatibleV2() throws Exception { + public void testSerializerCompatibleV3() throws Exception { SimpleStats keyStats = new SimpleStats( singleColumn("min_key"), @@ -256,10 +265,13 @@ public void testSerializerCompatibleV2() throws Exception { 11L, new byte[] {1, 2, 4}, FileSource.COMPACT, - null, + Arrays.asList("field1", "field2", "field3"), null); List dataFiles = Collections.singletonList(dataFile); + DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, null); + List deletionFiles = Collections.singletonList(deletionFile); + BinaryRow partition = new BinaryRow(1); BinaryRowWriter binaryRowWriter = new BinaryRowWriter(partition); binaryRowWriter.writeString(0, BinaryString.fromString("aaaaa")); @@ -271,14 +283,17 @@ public void testSerializerCompatibleV2() throws Exception { .withPartition(partition) .withBucket(20) .withDataFiles(dataFiles) + .withDataDeletionFiles(deletionFiles) .withBucketPath("my path") .build(); + assertThat(InstantiationUtil.clone(split)).isEqualTo(split); + byte[] v2Bytes = IOUtils.readFully( SplitTest.class .getClassLoader() - .getResourceAsStream("compatibility/datasplit-v2"), + .getResourceAsStream("compatibility/datasplit-v3"), true); DataSplit actual = @@ -287,7 +302,7 @@ public void testSerializerCompatibleV2() throws Exception { } @Test - public void testSerializerCompatibleV3() throws Exception { + public void testSerializerCompatibleV4() throws Exception { SimpleStats keyStats = new SimpleStats( singleColumn("min_key"), @@ -321,7 +336,7 @@ public void testSerializerCompatibleV3() throws Exception { null); List dataFiles = Collections.singletonList(dataFile); - DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, null); + DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, 33L); List deletionFiles = Collections.singletonList(deletionFile); BinaryRow partition = new BinaryRow(1); @@ -339,20 +354,22 @@ public void testSerializerCompatibleV3() throws Exception { .withBucketPath("my path") .build(); - byte[] v2Bytes = + assertThat(InstantiationUtil.clone(split)).isEqualTo(split); + + byte[] v4Bytes = IOUtils.readFully( SplitTest.class .getClassLoader() - .getResourceAsStream("compatibility/datasplit-v3"), + .getResourceAsStream("compatibility/datasplit-v4"), true); DataSplit actual = - InstantiationUtil.deserializeObject(v2Bytes, DataSplit.class.getClassLoader()); + InstantiationUtil.deserializeObject(v4Bytes, DataSplit.class.getClassLoader()); assertThat(actual).isEqualTo(split); } @Test - public void testSerializerCompatibleV4() throws Exception { + public void testSerializerCompatibleV5() throws Exception { SimpleStats keyStats = new SimpleStats( singleColumn("min_key"), @@ -383,7 +400,7 @@ public void testSerializerCompatibleV4() throws Exception { new byte[] {1, 2, 4}, FileSource.COMPACT, Arrays.asList("field1", "field2", "field3"), - null); + "hdfs:///path/to/warehouse"); List dataFiles = Collections.singletonList(dataFile); DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, 33L); @@ -404,15 +421,17 @@ public void testSerializerCompatibleV4() throws Exception { .withBucketPath("my path") .build(); - byte[] v4Bytes = + assertThat(InstantiationUtil.clone(split)).isEqualTo(split); + + byte[] v5Bytes = IOUtils.readFully( SplitTest.class .getClassLoader() - .getResourceAsStream("compatibility/datasplit-v4"), + .getResourceAsStream("compatibility/datasplit-v5"), true); DataSplit actual = - InstantiationUtil.deserializeObject(v4Bytes, DataSplit.class.getClassLoader()); + InstantiationUtil.deserializeObject(v5Bytes, DataSplit.class.getClassLoader()); assertThat(actual).isEqualTo(split); } diff --git a/paimon-core/src/test/resources/compatibility/datasplit-v5 b/paimon-core/src/test/resources/compatibility/datasplit-v5 new file mode 100644 index 000000000000..74a7018c32c0 Binary files /dev/null and b/paimon-core/src/test/resources/compatibility/datasplit-v5 differ diff --git a/paimon-core/src/test/resources/compatibility/manifest-committable-v6 b/paimon-core/src/test/resources/compatibility/manifest-committable-v6 new file mode 100644 index 000000000000..2cf4c7dde6ff Binary files /dev/null and b/paimon-core/src/test/resources/compatibility/manifest-committable-v6 differ