From e58197986a8d5a1fc3ca21b176be121b62dd9b57 Mon Sep 17 00:00:00 2001 From: Aitozi Date: Tue, 30 Jul 2024 20:02:37 +0800 Subject: [PATCH 1/3] [core] Manage the lookup file disk cache in task granularity --- .../apache/paimon/mergetree/LookupLevels.java | 40 +++++++++---------- .../LookupMergeTreeCompactRewriter.java | 4 +- .../operation/KeyValueFileStoreWrite.java | 23 +++++++++-- .../paimon/table/query/LocalTableQuery.java | 24 ++++++----- .../paimon/mergetree/ContainsLevelsTest.java | 14 +------ .../paimon/mergetree/LookupLevelsTest.java | 14 +------ 6 files changed, 60 insertions(+), 59 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java index d6024ebb88cf..33db0c1f57f6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java @@ -39,6 +39,7 @@ import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.RemovalCause; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Weigher; import org.apache.paimon.shade.guava30.com.google.common.util.concurrent.MoreExecutors; import javax.annotation.Nullable; @@ -61,7 +62,7 @@ import static org.apache.paimon.utils.VarLengthIntUtils.encodeLong; /** Provide lookup by key. */ -public class LookupLevels implements Levels.DropFileCallback, Closeable { +public class LookupLevels implements Levels.DropFileCallback { private final Levels levels; private final Comparator keyComparator; @@ -81,9 +82,8 @@ public LookupLevels( IOFunction> fileReaderFactory, Supplier localFileFactory, LookupStoreFactory lookupStoreFactory, - Duration fileRetention, - MemorySize maxDiskSize, - Function bfGenerator) { + Function bfGenerator, + Cache lookupFiles) { this.levels = levels; this.keyComparator = keyComparator; this.keySerializer = new RowCompactedSerializer(keyType); @@ -91,18 +91,22 @@ public LookupLevels( this.fileReaderFactory = fileReaderFactory; this.localFileFactory = localFileFactory; this.lookupStoreFactory = lookupStoreFactory; - this.lookupFiles = - Caffeine.newBuilder() - .expireAfterAccess(fileRetention) - .maximumWeight(maxDiskSize.getKibiBytes()) - .weigher(this::fileWeigh) - .removalListener(this::removalCallback) - .executor(MoreExecutors.directExecutor()) - .build(); this.bfGenerator = bfGenerator; + this.lookupFiles = lookupFiles; levels.addDropFileCallback(this); } + public static Cache createCache( + Duration fileRetention, MemorySize maxDiskSize) { + return Caffeine.newBuilder() + .expireAfterAccess(fileRetention) + .maximumWeight(maxDiskSize.getKibiBytes()) + .weigher((Weigher) LookupLevels::fileWeigh) + .removalListener(LookupLevels::removalCallback) + .executor(MoreExecutors.directExecutor()) + .build(); + } + public Levels getLevels() { return levels; } @@ -151,11 +155,11 @@ private T lookup(InternalRow key, DataFileMeta file) throws IOException { key, lookupFile.remoteFile().level(), valueBytes, file.fileName()); } - private int fileWeigh(String file, LookupFile lookupFile) { + private static int fileWeigh(String file, LookupFile lookupFile) { return fileKibiBytes(lookupFile.localFile); } - private void removalCallback(String key, LookupFile file, RemovalCause cause) { + private static void removalCallback(String key, LookupFile file, RemovalCause cause) { if (file != null) { try { file.close(); @@ -207,12 +211,8 @@ private LookupFile createLookupFile(DataFileMeta file) throws IOException { return new LookupFile(localFile, file, lookupStoreFactory.createReader(localFile, context)); } - @Override - public void close() throws IOException { - lookupFiles.invalidateAll(); - } - - private static class LookupFile implements Closeable { + /** Lookup file. */ + public static class LookupFile implements Closeable { private final File localFile; private final DataFileMeta remoteFile; diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java index ca7069455323..02d393d6b71f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java @@ -130,9 +130,7 @@ protected MergeFunctionWrapper createMergeWrapper(int outputLev } @Override - public void close() throws IOException { - lookupLevels.close(); - } + public void close() throws IOException {} /** Factory to create {@link MergeFunctionWrapper}. */ public interface MergeFunctionWrapperFactory { diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java index df3242141e50..6a5b30aeaa7e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java @@ -71,6 +71,8 @@ import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.UserDefinedSeqComparator; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,6 +104,7 @@ public class KeyValueFileStoreWrite extends MemoryFileStoreWrite { private final RowType keyType; private final RowType valueType; @Nullable private final RecordLevelExpire recordLevelExpire; + private Cache lookupFileCache; public KeyValueFileStoreWrite( FileIO fileIO, @@ -356,6 +359,13 @@ private LookupLevels createLookupLevels( cacheManager, new RowCompactedSerializer(keyType).createSliceComparator()); Options options = this.options.toConfiguration(); + if (lookupFileCache == null) { + lookupFileCache = + LookupLevels.createCache( + options.get(CoreOptions.LOOKUP_CACHE_FILE_RETENTION), + options.get(CoreOptions.LOOKUP_CACHE_MAX_DISK_SIZE)); + } + return new LookupLevels<>( levels, keyComparatorSupplier.get(), @@ -364,8 +374,15 @@ private LookupLevels createLookupLevels( readerFactory::createRecordReader, () -> ioManager.createChannel().getPathFile(), lookupStoreFactory, - options.get(CoreOptions.LOOKUP_CACHE_FILE_RETENTION), - options.get(CoreOptions.LOOKUP_CACHE_MAX_DISK_SIZE), - bfGenerator(options)); + bfGenerator(options), + lookupFileCache); + } + + @Override + public void close() throws Exception { + super.close(); + if (lookupFileCache != null) { + lookupFileCache.invalidateAll(); + } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java b/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java index 9181fdb16f8d..34edfe2182e6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java @@ -41,6 +41,8 @@ import org.apache.paimon.utils.KeyComparatorSupplier; import org.apache.paimon.utils.Preconditions; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; + import javax.annotation.Nullable; import java.io.IOException; @@ -70,6 +72,8 @@ public class LocalTableQuery implements TableQuery { private IOManager ioManager; + private Cache lookupFileCache; + public LocalTableQuery(FileStoreTable table) { this.options = table.coreOptions(); this.tableView = new HashMap<>(); @@ -130,6 +134,13 @@ private void newLookupLevels(BinaryRow partition, int bucket, List KeyValueFileReaderFactory factory = readerFactoryBuilder.build(partition, bucket, DeletionVector.emptyFactory()); Options options = this.options.toConfiguration(); + if (lookupFileCache == null) { + lookupFileCache = + LookupLevels.createCache( + options.get(CoreOptions.LOOKUP_CACHE_FILE_RETENTION), + options.get(CoreOptions.LOOKUP_CACHE_MAX_DISK_SIZE)); + } + LookupLevels lookupLevels = new LookupLevels<>( levels, @@ -148,9 +159,8 @@ private void newLookupLevels(BinaryRow partition, int bucket, List .createChannel() .getPathFile(), lookupStoreFactory, - options.get(CoreOptions.LOOKUP_CACHE_FILE_RETENTION), - options.get(CoreOptions.LOOKUP_CACHE_MAX_DISK_SIZE), - bfGenerator(options)); + bfGenerator(options), + lookupFileCache); tableView.computeIfAbsent(partition, k -> new HashMap<>()).put(bucket, lookupLevels); } @@ -195,12 +205,8 @@ public InternalRowSerializer createValueSerializer() { @Override public void close() throws IOException { - for (Map.Entry>> buckets : - tableView.entrySet()) { - for (Map.Entry> bucket : - buckets.getValue().entrySet()) { - bucket.getValue().close(); - } + if (lookupFileCache != null) { + lookupFileCache.invalidateAll(); } tableView.clear(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java index 781bab0dfb1a..780d2205a2fc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java @@ -105,9 +105,6 @@ public void testMultiLevels() throws IOException { // no exists assertThat(containsLevels.lookup(row(4), 1)).isNull(); - - containsLevels.close(); - assertThat(containsLevels.lookupFiles().estimatedSize()).isEqualTo(0); } @Test @@ -145,9 +142,6 @@ public void testMultiFiles() throws IOException { for (int key : notContains) { assertThat(containsLevels.lookup(row(key), 1)).isNull(); } - - containsLevels.close(); - assertThat(containsLevels.lookupFiles().estimatedSize()).isEqualTo(0); } @Test @@ -177,9 +171,6 @@ public void testMaxDiskSize() throws IOException { tempDir.toFile().list((dir, name) -> name.startsWith(LOOKUP_FILE_PREFIX)); assertThat(lookupFiles).isNotNull(); assertThat(fileNumber).isNotEqualTo(fileNum).isEqualTo(lookupFiles.length); - - lookupLevels.close(); - assertThat(lookupLevels.lookupFiles().estimatedSize()).isEqualTo(0); } private LookupLevels createContainsLevels(Levels levels, MemorySize maxDiskSize) { @@ -195,9 +186,8 @@ private LookupLevels createContainsLevels(Levels levels, MemorySize max () -> new File(tempDir.toFile(), LOOKUP_FILE_PREFIX + UUID.randomUUID()), new HashLookupStoreFactory( new CacheManager(MemorySize.ofMebiBytes(1)), 2048, 0.75, "none"), - Duration.ofHours(1), - maxDiskSize, - rowCount -> BloomFilter.builder(rowCount, 0.01)); + rowCount -> BloomFilter.builder(rowCount, 0.01), + LookupLevels.createCache(Duration.ofHours(1), maxDiskSize)); } private KeyValue kv(int key, int value) { diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java index 7b89f409dc8c..56fb9fbdfdc1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java @@ -119,9 +119,6 @@ public void testMultiLevels() throws IOException { // no exists kv = lookupLevels.lookup(row(4), 1); assertThat(kv).isNull(); - - lookupLevels.close(); - assertThat(lookupLevels.lookupFiles().estimatedSize()).isEqualTo(0); } @Test @@ -164,9 +161,6 @@ public void testMultiFiles() throws IOException { KeyValue kv = lookupLevels.lookup(row(key), 1); assertThat(kv).isNull(); } - - lookupLevels.close(); - assertThat(lookupLevels.lookupFiles().estimatedSize()).isEqualTo(0); } @Test @@ -200,9 +194,6 @@ public void testMaxDiskSize() throws IOException { tempDir.toFile().list((dir, name) -> name.startsWith(LOOKUP_FILE_PREFIX)); assertThat(lookupFiles).isNotNull(); assertThat(fileNumber).isNotEqualTo(fileNum).isEqualTo(lookupFiles.length); - - lookupLevels.close(); - assertThat(lookupLevels.lookupFiles().estimatedSize()).isEqualTo(0); } @Test @@ -271,9 +262,8 @@ private LookupLevels createLookupLevels(Levels levels, MemorySize maxD () -> new File(tempDir.toFile(), LOOKUP_FILE_PREFIX + UUID.randomUUID()), new HashLookupStoreFactory( new CacheManager(MemorySize.ofMebiBytes(1)), 2048, 0.75, "none"), - Duration.ofHours(1), - maxDiskSize, - rowCount -> BloomFilter.builder(rowCount, 0.05)); + rowCount -> BloomFilter.builder(rowCount, 0.05), + LookupLevels.createCache(Duration.ofHours(1), maxDiskSize)); } private KeyValue kv(int key, int value) { From febbd7fe38f9500dc90fc59014c5d06292ef96dd Mon Sep 17 00:00:00 2001 From: Aitozi Date: Wed, 31 Jul 2024 10:12:03 +0800 Subject: [PATCH 2/3] fix comments --- .../org/apache/paimon/mergetree/LookupLevels.java | 15 ++++++++++++++- .../compact/LookupMergeTreeCompactRewriter.java | 4 +++- .../paimon/table/query/LocalTableQuery.java | 7 +++++++ .../paimon/mergetree/ContainsLevelsTest.java | 9 +++++++++ .../apache/paimon/mergetree/LookupLevelsTest.java | 9 +++++++++ 5 files changed, 42 insertions(+), 2 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java index 33db0c1f57f6..3a6a37b41494 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java @@ -51,6 +51,8 @@ import java.time.Duration; import java.util.Arrays; import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; import java.util.TreeSet; import java.util.function.Function; import java.util.function.Supplier; @@ -62,7 +64,7 @@ import static org.apache.paimon.utils.VarLengthIntUtils.encodeLong; /** Provide lookup by key. */ -public class LookupLevels implements Levels.DropFileCallback { +public class LookupLevels implements Levels.DropFileCallback, Closeable { private final Levels levels; private final Comparator keyComparator; @@ -73,6 +75,7 @@ public class LookupLevels implements Levels.DropFileCallback { private final LookupStoreFactory lookupStoreFactory; private final Cache lookupFiles; private final Function bfGenerator; + private final Set cachedFiles; public LookupLevels( Levels levels, @@ -93,6 +96,7 @@ public LookupLevels( this.lookupStoreFactory = lookupStoreFactory; this.bfGenerator = bfGenerator; this.lookupFiles = lookupFiles; + this.cachedFiles = new HashSet<>(); levels.addDropFileCallback(this); } @@ -119,6 +123,7 @@ Cache lookupFiles() { @Override public void notifyDropFile(String file) { lookupFiles.invalidate(file); + cachedFiles.remove(file); } @Nullable @@ -142,6 +147,7 @@ private T lookup(InternalRow key, DataFileMeta file) throws IOException { while (lookupFile == null || lookupFile.isClosed) { lookupFile = createLookupFile(file); + cachedFiles.add(file.fileName()); lookupFiles.put(file.fileName(), lookupFile); } @@ -211,6 +217,13 @@ private LookupFile createLookupFile(DataFileMeta file) throws IOException { return new LookupFile(localFile, file, lookupStoreFactory.createReader(localFile, context)); } + @Override + public void close() throws IOException { + for (String cachedFile : cachedFiles) { + lookupFiles.invalidate(cachedFile); + } + } + /** Lookup file. */ public static class LookupFile implements Closeable { diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java index 02d393d6b71f..ca7069455323 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java @@ -130,7 +130,9 @@ protected MergeFunctionWrapper createMergeWrapper(int outputLev } @Override - public void close() throws IOException {} + public void close() throws IOException { + lookupLevels.close(); + } /** Factory to create {@link MergeFunctionWrapper}. */ public interface MergeFunctionWrapperFactory { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java b/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java index 34edfe2182e6..7475d6e8e1a2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java @@ -205,6 +205,13 @@ public InternalRowSerializer createValueSerializer() { @Override public void close() throws IOException { + for (Map.Entry>> buckets : + tableView.entrySet()) { + for (Map.Entry> bucket : + buckets.getValue().entrySet()) { + bucket.getValue().close(); + } + } if (lookupFileCache != null) { lookupFileCache.invalidateAll(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java index 780d2205a2fc..07972b28a2f4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java @@ -105,6 +105,9 @@ public void testMultiLevels() throws IOException { // no exists assertThat(containsLevels.lookup(row(4), 1)).isNull(); + + containsLevels.close(); + assertThat(containsLevels.lookupFiles().estimatedSize()).isEqualTo(0); } @Test @@ -142,6 +145,9 @@ public void testMultiFiles() throws IOException { for (int key : notContains) { assertThat(containsLevels.lookup(row(key), 1)).isNull(); } + + containsLevels.close(); + assertThat(containsLevels.lookupFiles().estimatedSize()).isEqualTo(0); } @Test @@ -171,6 +177,9 @@ public void testMaxDiskSize() throws IOException { tempDir.toFile().list((dir, name) -> name.startsWith(LOOKUP_FILE_PREFIX)); assertThat(lookupFiles).isNotNull(); assertThat(fileNumber).isNotEqualTo(fileNum).isEqualTo(lookupFiles.length); + + lookupLevels.close(); + assertThat(lookupLevels.lookupFiles().estimatedSize()).isEqualTo(0); } private LookupLevels createContainsLevels(Levels levels, MemorySize maxDiskSize) { diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java index 56fb9fbdfdc1..cc9ee4fcb3e6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java @@ -119,6 +119,9 @@ public void testMultiLevels() throws IOException { // no exists kv = lookupLevels.lookup(row(4), 1); assertThat(kv).isNull(); + + lookupLevels.close(); + assertThat(lookupLevels.lookupFiles().estimatedSize()).isEqualTo(0); } @Test @@ -161,6 +164,9 @@ public void testMultiFiles() throws IOException { KeyValue kv = lookupLevels.lookup(row(key), 1); assertThat(kv).isNull(); } + + lookupLevels.close(); + assertThat(lookupLevels.lookupFiles().estimatedSize()).isEqualTo(0); } @Test @@ -194,6 +200,9 @@ public void testMaxDiskSize() throws IOException { tempDir.toFile().list((dir, name) -> name.startsWith(LOOKUP_FILE_PREFIX)); assertThat(lookupFiles).isNotNull(); assertThat(fileNumber).isNotEqualTo(fileNum).isEqualTo(lookupFiles.length); + + lookupLevels.close(); + assertThat(lookupLevels.lookupFiles().estimatedSize()).isEqualTo(0); } @Test From 45f9a67f369c85878a582b92d6a614b6b9302029 Mon Sep 17 00:00:00 2001 From: Aitozi Date: Wed, 31 Jul 2024 10:40:29 +0800 Subject: [PATCH 3/3] remove the local filename when evicted --- .../apache/paimon/mergetree/LookupLevels.java | 21 +++++++++++++++---- .../paimon/mergetree/LookupLevelsTest.java | 3 +++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java index 3a6a37b41494..1ba31816d3e7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java @@ -120,10 +120,14 @@ Cache lookupFiles() { return lookupFiles; } + @VisibleForTesting + Set cachedFiles() { + return cachedFiles; + } + @Override public void notifyDropFile(String file) { lookupFiles.invalidate(file); - cachedFiles.remove(file); } @Nullable @@ -214,12 +218,14 @@ private LookupFile createLookupFile(DataFileMeta file) throws IOException { context = kvWriter.close(); } - return new LookupFile(localFile, file, lookupStoreFactory.createReader(localFile, context)); + return new LookupFile( + localFile, file, lookupStoreFactory.createReader(localFile, context), cachedFiles); } @Override public void close() throws IOException { - for (String cachedFile : cachedFiles) { + Set toClean = new HashSet<>(cachedFiles); + for (String cachedFile : toClean) { lookupFiles.invalidate(cachedFile); } } @@ -230,13 +236,19 @@ public static class LookupFile implements Closeable { private final File localFile; private final DataFileMeta remoteFile; private final LookupStoreReader reader; + private final Set cachedFiles; private boolean isClosed = false; - public LookupFile(File localFile, DataFileMeta remoteFile, LookupStoreReader reader) { + public LookupFile( + File localFile, + DataFileMeta remoteFile, + LookupStoreReader reader, + Set cachedFiles) { this.localFile = localFile; this.remoteFile = remoteFile; this.reader = reader; + this.cachedFiles = cachedFiles; } @Nullable @@ -253,6 +265,7 @@ public DataFileMeta remoteFile() { public void close() throws IOException { reader.close(); isClosed = true; + cachedFiles.remove(remoteFile.fileName()); FileIOUtils.deleteFileOrDirectory(localFile); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java index cc9ee4fcb3e6..a5d58cf13cb1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java @@ -193,6 +193,8 @@ public void testMaxDiskSize() throws IOException { assertThat(kv.level()).isEqualTo(1); assertThat(kv.value().getInt(1)).isEqualTo(i); } + assertThat(lookupLevels.lookupFiles().asMap().keySet()) + .isEqualTo(lookupLevels.cachedFiles()); // some files are invalided long fileNumber = lookupLevels.lookupFiles().estimatedSize(); @@ -202,6 +204,7 @@ public void testMaxDiskSize() throws IOException { assertThat(fileNumber).isNotEqualTo(fileNum).isEqualTo(lookupFiles.length); lookupLevels.close(); + assertThat(lookupLevels.cachedFiles()).isEmpty(); assertThat(lookupLevels.lookupFiles().estimatedSize()).isEqualTo(0); }