From fbfde859b4d9c2e1801cc714564fab2db3797aed Mon Sep 17 00:00:00 2001 From: Jingsong Lee Date: Thu, 30 May 2024 14:47:14 +0800 Subject: [PATCH] [core] Fix raw convert fail for old storage ro table (#3438) --- .../paimon/table/source/MergeTreeSplitGenerator.java | 3 ++- .../test/java/org/apache/paimon/io/DataFileTestUtils.java | 2 +- .../org/apache/paimon/table/source/SplitGeneratorTest.java | 7 +++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/MergeTreeSplitGenerator.java b/paimon-core/src/main/java/org/apache/paimon/table/source/MergeTreeSplitGenerator.java index 4cf4a18031f5..c532f85b515b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/MergeTreeSplitGenerator.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/MergeTreeSplitGenerator.java @@ -144,6 +144,7 @@ private List flatFiles(List> section) { } private boolean withoutDeleteRow(DataFileMeta dataFileMeta) { - return dataFileMeta.deleteRowCount().map(count -> count == 0L).orElse(false); + // null to true to be compatible with old version + return dataFileMeta.deleteRowCount().map(count -> count == 0L).orElse(true); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestUtils.java b/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestUtils.java index d5f6ceda32a9..4124780b18dd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestUtils.java +++ b/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestUtils.java @@ -78,7 +78,7 @@ public static DataFileMeta newFile( } public static DataFileMeta newFile( - String name, int level, int minKey, int maxKey, long maxSequence, long deleteRowCount) { + String name, int level, int minKey, int maxKey, long maxSequence, Long deleteRowCount) { return new DataFileMeta( name, maxKey - minKey + 1, diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/SplitGeneratorTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/SplitGeneratorTest.java index 8223afa5f576..5035bd47c7c8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/SplitGeneratorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/SplitGeneratorTest.java @@ -184,6 +184,13 @@ public void testSplitRawConvertible() { .containsExactlyInAnyOrder( Pair.of(Arrays.asList("1", "2", "3", "4", "5"), false), Pair.of(Collections.singletonList("6"), true)); + + // test convertible for old version + List files6 = + Arrays.asList( + newFile("1", 1, 0, 10, 10L, null), newFile("2", 1, 10, 20, 20L, null)); + assertThat(toNamesAndRawConvertible(mergeTreeSplitGenerator.splitForBatch(files6))) + .containsExactlyInAnyOrder(Pair.of(Arrays.asList("1", "2"), true)); } @Test