[test] add random test case for manifest file full compaction (apache…

…#4326)
tongcheng-elong · Oct 15, 2024 · 5d444c5 · 5d444c5
1 parent a0eeee0
commit 5d444c5
Showing 1 changed file with 105 additions and 0 deletions.
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
@@ -39,8 +39,10 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import java.util.TreeSet;
 import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
@@ -446,6 +448,109 @@ public void testIdentifierAfterFullCompaction() throws Exception {
         containSameIdentifyEntryFile(fullCompacted, entryIdentifierExpected);
     }
 
+    @RepeatedTest(100)
+    public void testRandomFullCompaction() throws Exception {
+        List<ManifestFileMeta> input = new ArrayList<>();
+        Set<FileEntry.Identifier> manifestEntrySet = new HashSet<>();
+        Set<FileEntry.Identifier> deleteManifestEntrySet = new HashSet<>();
+        int inputSize = ThreadLocalRandom.current().nextInt(100) + 1;
+        int totalEntryNums = 0;
+        for (int i = 0; i < inputSize; i++) {
+            int entryNums = ThreadLocalRandom.current().nextInt(100) + 1;
+            input.add(
+                    generateRandomData(
+                            entryNums, totalEntryNums, manifestEntrySet, deleteManifestEntrySet));
+            totalEntryNums += entryNums;
+        }
+        int suggerstSize = ThreadLocalRandom.current().nextInt(3000) + 1;
+        int sizeTrigger = ThreadLocalRandom.current().nextInt(40000) + 1;
+        List<ManifestFileMeta> newMetas = new ArrayList<>();
+        Optional<List<ManifestFileMeta>> fullCompacted =
+                ManifestFileMerger.tryFullCompaction(
+                        input,
+                        newMetas,
+                        manifestFile,
+                        suggerstSize,
+                        sizeTrigger,
+                        getPartitionType(),
+                        null);
+
+        // *****verify result*****
+        List<ManifestFileMeta> mustMergedFiles =
+                input.stream()
+                        .filter(
+                                manifest ->
+                                        manifest.fileSize() < suggerstSize
+                                                || manifest.numDeletedFiles() > 0)
+                        .collect(Collectors.toList());
+        long mustMergeSize =
+                mustMergedFiles.stream().map(ManifestFileMeta::fileSize).reduce(0L, Long::sum);
+        // manifest files which were not written after full compaction
+        List<ManifestFileMeta> notMergedFiles =
+                input.stream()
+                        .filter(
+                                manifest ->
+                                        manifest.fileSize() >= suggerstSize
+                                                && manifest.numDeletedFiles() == 0)
+                        .filter(
+                                manifest ->
+                                        manifestFile.read(manifest.fileName(), manifest.fileSize())
+                                                .stream()
+                                                .map(ManifestEntry::identifier)
+                                                .noneMatch(deleteManifestEntrySet::contains))
+                        .collect(Collectors.toList());
+
+        if (mustMergeSize < sizeTrigger) {
+            assertThat(fullCompacted).isEmpty();
+            assertThat(newMetas).isEmpty();
+        } else if (mustMergedFiles.size() <= 1) {
+            assertThat(fullCompacted).isEmpty();
+            assertThat(newMetas).isEmpty();
+        } else {
+            assertThat(fullCompacted.get().size()).isEqualTo(notMergedFiles.size() + 1);
+            assertThat(newMetas).size().isEqualTo(1);
+        }
+    }
+
+    private ManifestFileMeta generateRandomData(
+            int entryNums,
+            int totalEntryNums,
+            Set<FileEntry.Identifier> manifestEntrySet,
+            Set<FileEntry.Identifier> deleteManifestEntrySet) {
+        List<ManifestEntry> entries = new ArrayList<>();
+        for (int i = 0; i < entryNums; i++) {
+            // 70% add, 30% delete
+            boolean isAdd = ThreadLocalRandom.current().nextInt(10) < 7;
+            if (manifestEntrySet.isEmpty() || isAdd) {
+                String fileName = String.format("file-%d", totalEntryNums + i);
+                Integer partition = ThreadLocalRandom.current().nextInt(10);
+                int level = ThreadLocalRandom.current().nextInt(6);
+                List<String> extraFiles = Lists.newArrayList(String.format("index-%s", fileName));
+                byte[] embeddedIndex = new byte[] {1, 2, 3};
+                entries.add(makeEntry(true, fileName, partition, level, extraFiles, embeddedIndex));
+            } else {
+                FileEntry.Identifier identifier =
+                        manifestEntrySet.stream()
+                                .skip(ThreadLocalRandom.current().nextInt(manifestEntrySet.size()))
+                                .findFirst()
+                                .get();
+                entries.add(
+                        makeEntry(
+                                false,
+                                identifier.fileName,
+                                identifier.partition.getInt(0),
+                                identifier.level,
+                                identifier.extraFiles,
+                                new byte[] {1, 2, 3}));
+                manifestEntrySet.remove(identifier);
+                deleteManifestEntrySet.add(identifier);
+            }
+        }
+        manifestEntrySet.addAll(
+                entries.stream().map(ManifestEntry::identifier).collect(Collectors.toSet()));
+        return makeManifest(entries.toArray(new ManifestEntry[0]));
+    }
+
     private void createData(
             int numLastBits, List<ManifestFileMeta> input, List<ManifestFileMeta> expected) {
         // suggested size 500 and suggested count 3