fix

apache · Aug 1, 2024 · 15d5d0b · 15d5d0b
1 parent 6d75518
commit 15d5d0b
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 2 deletions.
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java
@@ -62,7 +62,6 @@ public class CachingCatalog extends DelegateCatalog {
     protected final Cache<String, Map<String, String>> databaseCache;
     protected final Cache<Identifier, Table> tableCache;
     @Nullable protected final SegmentsCache<Path> manifestCache;
-    protected final long manifestCacheThreshold;
 
     public CachingCatalog(Catalog wrapped) {
         this(
@@ -92,7 +91,6 @@ public CachingCatalog(
             long manifestCacheThreshold,
             Ticker ticker) {
         super(wrapped);
-        this.manifestCacheThreshold = manifestCacheThreshold;
         if (expirationInterval.isZero() || expirationInterval.isNegative()) {
             throw new IllegalArgumentException(
                     "When cache.expiration-interval is set to negative or 0, the catalog cache should be disabled.");

diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java b/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java
@@ -35,6 +35,7 @@ public class SegmentsCache<T> {
 
     private final int pageSize;
     private final Cache<T, Segments> cache;
+    private final MemorySize maxMemorySize;
     private final long maxElementSize;
 
     public SegmentsCache(int pageSize, MemorySize maxMemorySize, long maxElementSize) {
@@ -46,13 +47,18 @@ public SegmentsCache(int pageSize, MemorySize maxMemorySize, long maxElementSize
                         .maximumWeight(maxMemorySize.getBytes())
                         .executor(Runnable::run)
                         .build();
+        this.maxMemorySize = maxMemorySize;
         this.maxElementSize = maxElementSize;
     }
 
     public int pageSize() {
         return pageSize;
     }
 
+    public MemorySize maxMemorySize() {
+        return maxMemorySize;
+    }
+
     public long maxElementSize() {
         return maxElementSize;
     }

diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java
@@ -21,6 +21,7 @@
 import org.apache.paimon.data.GenericRow;
 import org.apache.paimon.fs.Path;
 import org.apache.paimon.options.MemorySize;
+import org.apache.paimon.options.Options;
 import org.apache.paimon.schema.SchemaChange;
 import org.apache.paimon.table.Table;
 import org.apache.paimon.table.sink.BatchTableCommit;
@@ -51,6 +52,9 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.apache.paimon.data.BinaryString.fromString;
+import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_MAX_MEMORY;
+import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SMALL_FILE_MEMORY;
+import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SMALL_FILE_THRESHOLD;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
@@ -340,4 +344,26 @@ private void innerTestManifestCache(long manifestCacheThreshold) throws Exceptio
             }
         }
     }
+
+    @Test
+    public void testManifestCacheOptions() {
+        Options options = new Options();
+
+        CachingCatalog caching = (CachingCatalog) CachingCatalog.tryToCreate(catalog, options);
+        assertThat(caching.manifestCache.maxMemorySize())
+                .isEqualTo(CACHE_MANIFEST_SMALL_FILE_MEMORY.defaultValue());
+        assertThat(caching.manifestCache.maxElementSize())
+                .isEqualTo(CACHE_MANIFEST_SMALL_FILE_THRESHOLD.defaultValue().getBytes());
+
+        options.set(CACHE_MANIFEST_SMALL_FILE_MEMORY, MemorySize.ofMebiBytes(100));
+        options.set(CACHE_MANIFEST_SMALL_FILE_THRESHOLD, MemorySize.ofBytes(100));
+        caching = (CachingCatalog) CachingCatalog.tryToCreate(catalog, options);
+        assertThat(caching.manifestCache.maxMemorySize()).isEqualTo(MemorySize.ofMebiBytes(100));
+        assertThat(caching.manifestCache.maxElementSize()).isEqualTo(100);
+
+        options.set(CACHE_MANIFEST_MAX_MEMORY, MemorySize.ofMebiBytes(256));
+        caching = (CachingCatalog) CachingCatalog.tryToCreate(catalog, options);
+        assertThat(caching.manifestCache.maxMemorySize()).isEqualTo(MemorySize.ofMebiBytes(256));
+        assertThat(caching.manifestCache.maxElementSize()).isEqualTo(Long.MAX_VALUE);
+    }
 }