Skip to content

Commit

Permalink
[test] Add current version compatibility tests for manifest committab…
Browse files Browse the repository at this point in the history 
…le and split
  • Loading branch information
@JingsongLi
JingsongLi committed Dec 23, 2024
1 parent 32f57dc commit 8ab24a1
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 34 deletions.
29 changes: 19 additions & 10 deletions .../test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
public class ManifestCommittableSerializerCompatibilityTest {

@Test
public void testProduction() throws IOException {
public void testCompatibilityToV3CommitV6() throws IOException {
SimpleStats keyStats =
new SimpleStats(
singleColumn("min_key"),
Expand Down Expand Up @@ -105,10 +105,19 @@ public void testProduction() throws IOException {
byte[] bytes = serializer.serialize(manifestCommittable);
ManifestCommittable deserialized = serializer.deserialize(3, bytes);
assertThat(deserialized).isEqualTo(manifestCommittable);

byte[] oldBytes =
IOUtils.readFully(
ManifestCommittableSerializerCompatibilityTest.class
.getClassLoader()
.getResourceAsStream("compatibility/manifest-committable-v6"),
true);
deserialized = serializer.deserialize(3, oldBytes);
assertThat(deserialized).isEqualTo(manifestCommittable);
}

@Test
public void testCompatibilityToVersion5() throws IOException {
public void testCompatibilityToV3CommitV5() throws IOException {
SimpleStats keyStats =
new SimpleStats(
singleColumn("min_key"),
Expand Down Expand Up @@ -167,18 +176,18 @@ public void testCompatibilityToVersion5() throws IOException {
byte[] bytes = serializer.serialize(manifestCommittable);
ManifestCommittable deserialized = serializer.deserialize(3, bytes);
assertThat(deserialized).isEqualTo(manifestCommittable);
byte[] v2Bytes =
byte[] oldBytes =
IOUtils.readFully(
ManifestCommittableSerializerCompatibilityTest.class
.getClassLoader()
.getResourceAsStream("compatibility/manifest-committable-v5"),
true);
deserialized = serializer.deserialize(3, v2Bytes);
deserialized = serializer.deserialize(3, oldBytes);
assertThat(deserialized).isEqualTo(manifestCommittable);
}

@Test
public void testCompatibilityToVersion4() throws IOException {
public void testCompatibilityToV3CommitV4() throws IOException {
SimpleStats keyStats =
new SimpleStats(
singleColumn("min_key"),
Expand Down Expand Up @@ -244,12 +253,12 @@ public void testCompatibilityToVersion4() throws IOException {
.getClassLoader()
.getResourceAsStream("compatibility/manifest-committable-v4"),
true);
deserialized = serializer.deserialize(2, v2Bytes);
deserialized = serializer.deserialize(3, v2Bytes);
assertThat(deserialized).isEqualTo(manifestCommittable);
}

@Test
public void testCompatibilityToVersion3() throws IOException {
public void testCompatibilityToV3CommitV3() throws IOException {
SimpleStats keyStats =
new SimpleStats(
singleColumn("min_key"),
Expand Down Expand Up @@ -309,18 +318,18 @@ public void testCompatibilityToVersion3() throws IOException {
ManifestCommittable deserialized = serializer.deserialize(3, bytes);
assertThat(deserialized).isEqualTo(manifestCommittable);

byte[] v2Bytes =
byte[] oldBytes =
IOUtils.readFully(
ManifestCommittableSerializerCompatibilityTest.class
.getClassLoader()
.getResourceAsStream("compatibility/manifest-committable-v3"),
true);
deserialized = serializer.deserialize(2, v2Bytes);
deserialized = serializer.deserialize(3, oldBytes);
assertThat(deserialized).isEqualTo(manifestCommittable);
}

@Test
public void testCompatibilityToVersion2() throws IOException {
public void testCompatibilityToV2CommitV2() throws IOException {
SimpleStats keyStats =
new SimpleStats(
singleColumn("min_key"),
Expand Down
67 changes: 43 additions & 24 deletions paimon-core/src/test/java/org/apache/paimon/table/source/SplitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ public void testSerializer() throws IOException {
}

@Test
public void testSerializerNormal() throws Exception {
public void testSerializerCompatibleV1() throws Exception {
SimpleStats keyStats =
new SimpleStats(
singleColumn("min_key"),
Expand Down Expand Up @@ -138,14 +138,11 @@ public void testSerializerNormal() throws Exception {
Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")),
11L,
new byte[] {1, 2, 4},
FileSource.COMPACT,
Arrays.asList("field1", "field2", "field3"),
"hdfs:///path/to/warehouse");
null,
null,
null);
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);

DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, 33L);
List<DeletionFile> deletionFiles = Collections.singletonList(deletionFile);

BinaryRow partition = new BinaryRow(1);
BinaryRowWriter binaryRowWriter = new BinaryRowWriter(partition);
binaryRowWriter.writeString(0, BinaryString.fromString("aaaaa"));
Expand All @@ -157,15 +154,25 @@ public void testSerializerNormal() throws Exception {
.withPartition(partition)
.withBucket(20)
.withDataFiles(dataFiles)
.withDataDeletionFiles(deletionFiles)
.withBucketPath("my path")
.build();

assertThat(InstantiationUtil.clone(split)).isEqualTo(split);

byte[] v2Bytes =
IOUtils.readFully(
SplitTest.class
.getClassLoader()
.getResourceAsStream("compatibility/datasplit-v1"),
true);

DataSplit actual =
InstantiationUtil.deserializeObject(v2Bytes, DataSplit.class.getClassLoader());
assertThat(actual).isEqualTo(split);
}

@Test
public void testSerializerCompatibleV1() throws Exception {
public void testSerializerCompatibleV2() throws Exception {
SimpleStats keyStats =
new SimpleStats(
singleColumn("min_key"),
Expand Down Expand Up @@ -194,7 +201,7 @@ public void testSerializerCompatibleV1() throws Exception {
Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")),
11L,
new byte[] {1, 2, 4},
null,
FileSource.COMPACT,
null,
null);
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);
Expand All @@ -213,11 +220,13 @@ public void testSerializerCompatibleV1() throws Exception {
.withBucketPath("my path")
.build();

assertThat(InstantiationUtil.clone(split)).isEqualTo(split);

byte[] v2Bytes =
IOUtils.readFully(
SplitTest.class
.getClassLoader()
.getResourceAsStream("compatibility/datasplit-v1"),
.getResourceAsStream("compatibility/datasplit-v2"),
true);

DataSplit actual =
Expand All @@ -226,7 +235,7 @@ public void testSerializerCompatibleV1() throws Exception {
}

@Test
public void testSerializerCompatibleV2() throws Exception {
public void testSerializerCompatibleV3() throws Exception {
SimpleStats keyStats =
new SimpleStats(
singleColumn("min_key"),
Expand Down Expand Up @@ -256,10 +265,13 @@ public void testSerializerCompatibleV2() throws Exception {
11L,
new byte[] {1, 2, 4},
FileSource.COMPACT,
null,
Arrays.asList("field1", "field2", "field3"),
null);
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);

DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, null);
List<DeletionFile> deletionFiles = Collections.singletonList(deletionFile);

BinaryRow partition = new BinaryRow(1);
BinaryRowWriter binaryRowWriter = new BinaryRowWriter(partition);
binaryRowWriter.writeString(0, BinaryString.fromString("aaaaa"));
Expand All @@ -271,14 +283,17 @@ public void testSerializerCompatibleV2() throws Exception {
.withPartition(partition)
.withBucket(20)
.withDataFiles(dataFiles)
.withDataDeletionFiles(deletionFiles)
.withBucketPath("my path")
.build();

assertThat(InstantiationUtil.clone(split)).isEqualTo(split);

byte[] v2Bytes =
IOUtils.readFully(
SplitTest.class
.getClassLoader()
.getResourceAsStream("compatibility/datasplit-v2"),
.getResourceAsStream("compatibility/datasplit-v3"),
true);

DataSplit actual =
Expand All @@ -287,7 +302,7 @@ public void testSerializerCompatibleV2() throws Exception {
}

@Test
public void testSerializerCompatibleV3() throws Exception {
public void testSerializerCompatibleV4() throws Exception {
SimpleStats keyStats =
new SimpleStats(
singleColumn("min_key"),
Expand Down Expand Up @@ -321,7 +336,7 @@ public void testSerializerCompatibleV3() throws Exception {
null);
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);

DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, null);
DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, 33L);
List<DeletionFile> deletionFiles = Collections.singletonList(deletionFile);

BinaryRow partition = new BinaryRow(1);
Expand All @@ -339,20 +354,22 @@ public void testSerializerCompatibleV3() throws Exception {
.withBucketPath("my path")
.build();

byte[] v2Bytes =
assertThat(InstantiationUtil.clone(split)).isEqualTo(split);

byte[] v4Bytes =
IOUtils.readFully(
SplitTest.class
.getClassLoader()
.getResourceAsStream("compatibility/datasplit-v3"),
.getResourceAsStream("compatibility/datasplit-v4"),
true);

DataSplit actual =
InstantiationUtil.deserializeObject(v2Bytes, DataSplit.class.getClassLoader());
InstantiationUtil.deserializeObject(v4Bytes, DataSplit.class.getClassLoader());
assertThat(actual).isEqualTo(split);
}

@Test
public void testSerializerCompatibleV4() throws Exception {
public void testSerializerCompatibleV5() throws Exception {
SimpleStats keyStats =
new SimpleStats(
singleColumn("min_key"),
Expand Down Expand Up @@ -383,7 +400,7 @@ public void testSerializerCompatibleV4() throws Exception {
new byte[] {1, 2, 4},
FileSource.COMPACT,
Arrays.asList("field1", "field2", "field3"),
null);
"hdfs:///path/to/warehouse");
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);

DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, 33L);
Expand All @@ -404,15 +421,17 @@ public void testSerializerCompatibleV4() throws Exception {
.withBucketPath("my path")
.build();

byte[] v4Bytes =
assertThat(InstantiationUtil.clone(split)).isEqualTo(split);

byte[] v5Bytes =
IOUtils.readFully(
SplitTest.class
.getClassLoader()
.getResourceAsStream("compatibility/datasplit-v4"),
.getResourceAsStream("compatibility/datasplit-v5"),
true);

DataSplit actual =
InstantiationUtil.deserializeObject(v4Bytes, DataSplit.class.getClassLoader());
InstantiationUtil.deserializeObject(v5Bytes, DataSplit.class.getClassLoader());
assertThat(actual).isEqualTo(split);
}

Expand Down
Binary file added paimon-core/src/test/resources/compatibility/datasplit-v5
View file
Binary file not shown.
Binary file added paimon-core/src/test/resources/compatibility/manifest-committable-v6
View file
Binary file not shown.

0 comments on commit 8ab24a1

Please sign in to comment.