diff --git a/docs/layouts/shortcodes/generated/catalog_configuration.html b/docs/layouts/shortcodes/generated/catalog_configuration.html index 63f7adda1e0d..a67b55dc0ba5 100644 --- a/docs/layouts/shortcodes/generated/catalog_configuration.html +++ b/docs/layouts/shortcodes/generated/catalog_configuration.html @@ -36,7 +36,7 @@
cache-enabled
true Boolean - Controls whether the catalog will cache databases, tables and manifests. + Controls whether the catalog will cache databases, tables, manifests and partitions.
cache.expiration-interval
@@ -74,6 +74,12 @@ Integer Controls the max number for snapshots per table in the catalog are cached. + +
cache.cache-stats-enabled
+ 20 + Integer + Controls whether the catalog cache stats are enabled. +
client-pool-size
2 diff --git a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java index bb8cfae68284..f240f5f286d9 100644 --- a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java +++ b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java @@ -87,7 +87,7 @@ public class CatalogOptions { .booleanType() .defaultValue(true) .withDescription( - "Controls whether the catalog will cache databases, tables and manifests."); + "Controls whether the catalog will cache databases, tables, manifests and partitions."); public static final ConfigOption CACHE_EXPIRATION_INTERVAL_MS = key("cache.expiration-interval") @@ -128,6 +128,12 @@ public class CatalogOptions { .withDescription( "Controls the max number for snapshots per table in the catalog are cached."); + public static final ConfigOption CACHE_STATS_ENABLED = + key("cache.cache-stats-enabled") + .booleanType() + .defaultValue(false) + .withDescription("Controls whether the catalog cache stats are enabled."); + public static final ConfigOption ALLOW_UPPER_CASE = ConfigOptions.key("allow-upper-case") .booleanType() diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java index 1caff252a654..c61ff9e14a26 100644 --- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java @@ -100,7 +100,7 @@ protected AbstractFileStore( this.catalogEnvironment = catalogEnvironment; this.writeManifestCache = SegmentsCache.create( - options.pageSize(), options.writeManifestCache(), Long.MAX_VALUE); + options.pageSize(), options.writeManifestCache(), Long.MAX_VALUE, false); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java index 82d503b7a272..73e98c829247 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java @@ -20,6 +20,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.operation.metrics.CatalogCacheStats; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.schema.SchemaChange; @@ -48,6 +49,7 @@ import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SMALL_FILE_THRESHOLD; import static org.apache.paimon.options.CatalogOptions.CACHE_PARTITION_MAX_NUM; import static org.apache.paimon.options.CatalogOptions.CACHE_SNAPSHOT_MAX_NUM_PER_TABLE; +import static org.apache.paimon.options.CatalogOptions.CACHE_STATS_ENABLED; import static org.apache.paimon.utils.Preconditions.checkNotNull; /** A {@link Catalog} to cache databases and tables and manifests. */ @@ -63,6 +65,8 @@ public class CachingCatalog extends DelegateCatalog { // partition cache will affect data latency @Nullable protected final Cache> partitionCache; + private final CatalogCacheStats catalogCacheStats; + public CachingCatalog(Catalog wrapped) { this( wrapped, @@ -70,7 +74,8 @@ public CachingCatalog(Catalog wrapped) { CACHE_MANIFEST_SMALL_FILE_MEMORY.defaultValue(), CACHE_MANIFEST_SMALL_FILE_THRESHOLD.defaultValue().getBytes(), CACHE_PARTITION_MAX_NUM.defaultValue(), - CACHE_SNAPSHOT_MAX_NUM_PER_TABLE.defaultValue()); + CACHE_SNAPSHOT_MAX_NUM_PER_TABLE.defaultValue(), + CACHE_STATS_ENABLED.defaultValue()); } public CachingCatalog( @@ -79,7 +84,8 @@ public CachingCatalog( MemorySize manifestMaxMemory, long manifestCacheThreshold, long cachedPartitionMaxNum, - int snapshotMaxNumPerTable) { + int snapshotMaxNumPerTable, + boolean cacheStatsEnabled) { this( wrapped, expirationInterval, @@ -87,6 +93,7 @@ public CachingCatalog( manifestCacheThreshold, cachedPartitionMaxNum, snapshotMaxNumPerTable, + cacheStatsEnabled, Ticker.systemTicker()); } @@ -97,6 +104,7 @@ public CachingCatalog( long manifestCacheThreshold, long cachedPartitionMaxNum, int snapshotMaxNumPerTable, + boolean cacheStatsEnabled, Ticker ticker) { super(wrapped); if (expirationInterval.isZero() || expirationInterval.isNegative()) { @@ -121,7 +129,8 @@ public CachingCatalog( .expireAfterAccess(expirationInterval) .ticker(ticker) .build(); - this.manifestCache = SegmentsCache.create(manifestMaxMemory, manifestCacheThreshold); + this.manifestCache = + SegmentsCache.create(manifestMaxMemory, manifestCacheThreshold, cacheStatsEnabled); this.partitionCache = cachedPartitionMaxNum == 0 ? null @@ -135,6 +144,7 @@ public CachingCatalog( .maximumWeight(cachedPartitionMaxNum) .ticker(ticker) .build(); + this.catalogCacheStats = cacheStatsEnabled ? new CatalogCacheStats() : null; } public static Catalog tryToCreate(Catalog catalog, Options options) { @@ -156,7 +166,8 @@ public static Catalog tryToCreate(Catalog catalog, Options options) { manifestMaxMemory, manifestThreshold, options.get(CACHE_PARTITION_MAX_NUM), - options.get(CACHE_SNAPSHOT_MAX_NUM_PER_TABLE)); + options.get(CACHE_SNAPSHOT_MAX_NUM_PER_TABLE), + options.get(CACHE_STATS_ENABLED)); } @Override @@ -305,6 +316,23 @@ public void invalidateTable(Identifier identifier) { } } + public CatalogCacheStats getCacheStats() { + catalogCacheStats.setDatabaseCacheSize(databaseCache.estimatedSize()); + catalogCacheStats.setTableCacheSize(tableCache.estimatedSize()); + if (manifestCache != null) { + catalogCacheStats.setManifestCacheSize(manifestCache.getSegmentCacheSize()); + } + if (partitionCache != null) { + int partitionSize = 0; + for (Map.Entry> entry : + partitionCache.asMap().entrySet()) { + partitionSize += entry.getValue().size(); + } + catalogCacheStats.setPartitionCacheSize(partitionSize); + } + return catalogCacheStats; + } + // ================================== refresh ================================================ // following caches will affect the latency of table, so refresh method is provided for engine diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java index c73a92062b80..dbf8d94f5701 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java @@ -393,6 +393,10 @@ private ManifestsReader.Result readManifests() { return manifestsReader.read(specifiedSnapshot, scanMode); } + public ScanMetrics getScanMetrics() { + return scanMetrics; + } + // ------------------------------------------------------------------------ // Start Thread Safe Methods: The following methods need to be thread safe because they will be // called by multiple threads @@ -425,19 +429,18 @@ private List readManifest( ManifestFileMeta manifest, @Nullable Filter additionalFilter, @Nullable Filter additionalTFilter) { + ManifestFile manifestFile = manifestFileFactory.create(); List entries = - manifestFileFactory - .create() - .read( - manifest.fileName(), - manifest.fileSize(), - createCacheRowFilter(), - createEntryRowFilter().and(additionalFilter), - entry -> - (additionalTFilter == null || additionalTFilter.test(entry)) - && (manifestEntryFilter == null - || manifestEntryFilter.test(entry)) - && filterByStats(entry)); + manifestFile.read( + manifest.fileName(), + manifest.fileSize(), + createCacheRowFilter(), + createEntryRowFilter().and(additionalFilter), + entry -> + (additionalTFilter == null || additionalTFilter.test(entry)) + && (manifestEntryFilter == null + || manifestEntryFilter.test(entry)) + && filterByStats(entry)); if (dropStats) { List copied = new ArrayList<>(entries.size()); for (ManifestEntry entry : entries) { @@ -445,6 +448,13 @@ private List readManifest( } entries = copied; } + if (scanMetrics != null && manifestFile.hitCache() != null) { + if (manifestFile.hitCache()) { + this.scanMetrics.getObjectCacheStats().increaseHitObject(); + } else { + this.scanMetrics.getObjectCacheStats().increaseMissedObject(); + } + } return entries; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CatalogCacheStats.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CatalogCacheStats.java new file mode 100644 index 000000000000..aa69a409c6f9 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CatalogCacheStats.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation.metrics; + +/** Statistics for cache of a caching catalog. */ +public class CatalogCacheStats { + private long databaseCacheSize; + private long tableCacheSize; + private long manifestCacheSize; + private long partitionCacheSize; + + public CatalogCacheStats() {} + + public long getDatabaseCacheSize() { + return databaseCacheSize; + } + + public void setDatabaseCacheSize(long databaseCacheSize) { + this.databaseCacheSize = databaseCacheSize; + } + + public long getTableCacheSize() { + return tableCacheSize; + } + + public void setTableCacheSize(long tableCacheSize) { + this.tableCacheSize = tableCacheSize; + } + + public long getManifestCacheSize() { + return manifestCacheSize; + } + + public void setManifestCacheSize(long manifestCacheSize) { + this.manifestCacheSize = manifestCacheSize; + } + + public long getPartitionCacheSize() { + return partitionCacheSize; + } + + public void setPartitionCacheSize(long partitionCacheSize) { + this.partitionCacheSize = partitionCacheSize; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ObjectCacheStats.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ObjectCacheStats.java new file mode 100644 index 000000000000..4aab9878d8d8 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ObjectCacheStats.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation.metrics; + +import java.util.concurrent.atomic.AtomicLong; + +/** Statistics for object cache of a caching catalog. */ +public class ObjectCacheStats { + private final AtomicLong hitObject; + private final AtomicLong missedObject; + + public ObjectCacheStats() { + this.hitObject = new AtomicLong(0); + this.missedObject = new AtomicLong(0); + } + + public AtomicLong getHitObject() { + return hitObject; + } + + public void increaseHitObject() { + this.hitObject.incrementAndGet(); + } + + public AtomicLong getMissedObject() { + return missedObject; + } + + public void increaseMissedObject() { + this.missedObject.incrementAndGet(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java index 96f0aec1c0b2..2ee6834c8ac1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java @@ -30,9 +30,14 @@ public class ScanMetrics { public static final String GROUP_NAME = "scan"; private final MetricGroup metricGroup; + private Histogram durationHistogram; + + private ScanStats latestScan; + private ObjectCacheStats objectCacheStats; public ScanMetrics(MetricRegistry registry, String tableName) { this.metricGroup = registry.tableMetricGroup(GROUP_NAME, tableName); + this.objectCacheStats = new ObjectCacheStats(); registerGenericScanMetrics(); } @@ -41,10 +46,6 @@ public MetricGroup getMetricGroup() { return metricGroup; } - private Histogram durationHistogram; - - private ScanStats latestScan; - public static final String LAST_SCAN_DURATION = "lastScanDuration"; public static final String SCAN_DURATION = "scanDuration"; public static final String LAST_SCANNED_MANIFESTS = "lastScannedManifests"; @@ -72,4 +73,16 @@ public void reportScan(ScanStats scanStats) { latestScan = scanStats; durationHistogram.update(scanStats.getDuration()); } + + public ScanStats getLatestScan() { + return latestScan; + } + + public ObjectCacheStats getObjectCacheStats() { + return objectCacheStats; + } + + public void setObjectCacheStats(ObjectCacheStats objectCacheStats) { + this.objectCacheStats = objectCacheStats; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanStats.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanStats.java index 700619c3680f..0ad9514e5d04 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanStats.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanStats.java @@ -38,22 +38,22 @@ public ScanStats( } @VisibleForTesting - protected long getScannedManifests() { + public long getScannedManifests() { return scannedManifests; } @VisibleForTesting - protected long getSkippedTableFiles() { + public long getSkippedTableFiles() { return skippedTableFiles; } @VisibleForTesting - protected long getResultedTableFiles() { + public long getResultedTableFiles() { return resultedTableFiles; } @VisibleForTesting - protected long getDuration() { + public long getDuration() { return duration; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java index 24c6943f546f..e542ff7ae20a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java @@ -26,6 +26,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.metrics.MetricRegistry; import org.apache.paimon.operation.FileStoreScan; +import org.apache.paimon.operation.metrics.ScanMetrics; import org.apache.paimon.table.source.snapshot.CompactedStartingScanner; import org.apache.paimon.table.source.snapshot.ContinuousCompactorStartingScanner; import org.apache.paimon.table.source.snapshot.ContinuousFromSnapshotFullStartingScanner; @@ -112,6 +113,10 @@ public CoreOptions options() { return options; } + public ScanMetrics getScanMetrics() { + return snapshotReader.getScanMetrics(); + } + protected StartingScanner createStartingScanner(boolean isStreaming) { SnapshotManager snapshotManager = snapshotReader.snapshotManager(); CoreOptions.StreamScanMode type = diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReader.java b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReader.java index f3e0a92b8fc7..82095dbefd41 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReader.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReader.java @@ -27,6 +27,7 @@ import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.metrics.MetricRegistry; import org.apache.paimon.operation.ManifestsReader; +import org.apache.paimon.operation.metrics.ScanMetrics; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.ScanMode; @@ -110,6 +111,8 @@ public interface SnapshotReader { Iterator readFileIterator(); + ScanMetrics getScanMetrics(); + /** Result plan of this scan. */ interface Plan extends TableScan.Plan { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java index ce01bdba9447..89ac82086f18 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java @@ -33,6 +33,7 @@ import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.metrics.MetricRegistry; +import org.apache.paimon.operation.AbstractFileStoreScan; import org.apache.paimon.operation.DefaultValueAssigner; import org.apache.paimon.operation.FileStoreScan; import org.apache.paimon.operation.ManifestsReader; @@ -276,6 +277,14 @@ public SnapshotReader dropStats() { return this; } + @Override + public ScanMetrics getScanMetrics() { + if (scan instanceof AbstractFileStoreScan) { + return ((AbstractFileStoreScan) scan).getScanMetrics(); + } + return null; + } + @Override public SnapshotReader withShard(int indexOfThisSubtask, int numberOfParallelSubtasks) { if (splitGenerator.alwaysRawConvertible()) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java index 1cb967f8d1e2..80d5bc187236 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java @@ -34,6 +34,7 @@ import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.metrics.MetricRegistry; import org.apache.paimon.operation.ManifestsReader; +import org.apache.paimon.operation.metrics.ScanMetrics; import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; @@ -406,6 +407,11 @@ public List bucketEntries() { public Iterator readFileIterator() { return wrapped.readFileIterator(); } + + @Override + public ScanMetrics getScanMetrics() { + return wrapped.getScanMetrics(); + } } private class AuditLogBatchScan implements DataTableScan { diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java index 1c9d9664f22f..b74e63f4179c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java @@ -47,6 +47,7 @@ public class ObjectsCache { private final ThreadLocal formatSerializer; private final FunctionWithIOException fileSizeFunction; private final BiFunctionWithIOE> reader; + private boolean hitCache; public ObjectsCache( SegmentsCache cache, @@ -71,8 +72,14 @@ public List read( throws IOException { Segments segments = cache.getIfPresents(key); if (segments != null) { + if (cacheStatsEnabled()) { + hitCache = true; + } return readFromSegments(segments, readFilter, readVFilter); } else { + if (cacheStatsEnabled()) { + hitCache = false; + } if (fileSize == null) { fileSize = fileSizeFunction.apply(key); } @@ -130,4 +137,12 @@ private Segments readSegments(K key, @Nullable Long fileSize, Filter List readFromIterator( throw new RuntimeException(e); } } + + public Boolean hitCache() { + if (cache != null && cache.cacheStatsEnabled()) { + return cache.hitCache(); + } + return null; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java b/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java index d5c2178c8b6a..2d1ad82f1aea 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java @@ -37,8 +37,13 @@ public class SegmentsCache { private final Cache cache; private final MemorySize maxMemorySize; private final long maxElementSize; + private final boolean cacheStatsEnabled; - public SegmentsCache(int pageSize, MemorySize maxMemorySize, long maxElementSize) { + public SegmentsCache( + int pageSize, + MemorySize maxMemorySize, + long maxElementSize, + boolean cacheStatsEnabled) { this.pageSize = pageSize; this.cache = Caffeine.newBuilder() @@ -49,6 +54,7 @@ public SegmentsCache(int pageSize, MemorySize maxMemorySize, long maxElementSize .build(); this.maxMemorySize = maxMemorySize; this.maxElementSize = maxElementSize; + this.cacheStatsEnabled = cacheStatsEnabled; } public int pageSize() { @@ -63,6 +69,10 @@ public long maxElementSize() { return maxElementSize; } + public boolean cacheStatsEnabled() { + return cacheStatsEnabled; + } + @Nullable public Segments getIfPresents(T key) { return cache.getIfPresent(key); @@ -77,17 +87,29 @@ private int weigh(T cacheKey, Segments segments) { } @Nullable - public static SegmentsCache create(MemorySize maxMemorySize, long maxElementSize) { - return create((int) PAGE_SIZE.defaultValue().getBytes(), maxMemorySize, maxElementSize); + public static SegmentsCache create( + MemorySize maxMemorySize, long maxElementSize, boolean cacheStatsEnabled) { + return create( + (int) PAGE_SIZE.defaultValue().getBytes(), + maxMemorySize, + maxElementSize, + cacheStatsEnabled); } @Nullable public static SegmentsCache create( - int pageSize, MemorySize maxMemorySize, long maxElementSize) { + int pageSize, + MemorySize maxMemorySize, + long maxElementSize, + boolean cacheStatsEnabled) { if (maxMemorySize.getBytes() == 0) { return null; } - return new SegmentsCache<>(pageSize, maxMemorySize, maxElementSize); + return new SegmentsCache<>(pageSize, maxMemorySize, maxElementSize, cacheStatsEnabled); + } + + public long getSegmentCacheSize() { + return cache.estimatedSize(); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java index 4792e33c932b..ad9c70191141 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java @@ -353,7 +353,8 @@ private void innerTestManifestCache(long manifestCacheThreshold) throws Exceptio MemorySize.ofMebiBytes(1), manifestCacheThreshold, 0L, - 10); + 10, + false); Identifier tableIdent = new Identifier("db", "tbl"); catalog.dropTable(tableIdent, true); catalog.createTable(tableIdent, DEFAULT_TABLE_SCHEMA, false); diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/TestableCachingCatalog.java b/paimon-core/src/test/java/org/apache/paimon/catalog/TestableCachingCatalog.java index 1d4a9b0e8a58..0c1a53536aa1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/TestableCachingCatalog.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/TestableCachingCatalog.java @@ -45,6 +45,7 @@ public TestableCachingCatalog(Catalog catalog, Duration expirationInterval, Tick Long.MAX_VALUE, Long.MAX_VALUE, Integer.MAX_VALUE, + false, ticker); this.cacheExpirationInterval = expirationInterval; } diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java index 9d3275e3ab48..9216c32e7e2e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java @@ -44,7 +44,7 @@ public void test() throws IOException { Map> map = new HashMap<>(); ObjectsCache cache = new ObjectsCache<>( - new SegmentsCache<>(1024, MemorySize.ofKibiBytes(5), Long.MAX_VALUE), + new SegmentsCache<>(1024, MemorySize.ofKibiBytes(5), Long.MAX_VALUE, true), new StringSerializer(), RowType.of(DataTypes.STRING()), k -> 1L, @@ -62,6 +62,7 @@ public void test() throws IOException { cache.read( "k1", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue()); assertThat(values).isEmpty(); + assertThat(cache.hitCache()).isFalse(); // test values List expect = Arrays.asList("v1", "v2", "v3"); @@ -70,12 +71,14 @@ public void test() throws IOException { cache.read( "k2", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue()); assertThat(values).containsExactlyElementsOf(expect); + assertThat(cache.hitCache()).isFalse(); // test cache values = cache.read( "k2", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue()); assertThat(values).containsExactlyElementsOf(expect); + assertThat(cache.hitCache()).isTrue(); // test filter values =