diff --git a/docs/layouts/shortcodes/generated/catalog_configuration.html b/docs/layouts/shortcodes/generated/catalog_configuration.html
index 6355c9558653..03efd178b8fc 100644
--- a/docs/layouts/shortcodes/generated/catalog_configuration.html
+++ b/docs/layouts/shortcodes/generated/catalog_configuration.html
@@ -30,7 +30,7 @@
cache-enabled |
true |
Boolean |
- Controls whether the catalog will cache databases, tables and manifests. |
+ Controls whether the catalog will cache databases, tables, manifests and partitions. |
cache.expiration-interval |
diff --git a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java
index b22274e011fb..d0cfbeaf39ed 100644
--- a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java
@@ -87,7 +87,7 @@ public class CatalogOptions {
.booleanType()
.defaultValue(true)
.withDescription(
- "Controls whether the catalog will cache databases, tables and manifests.");
+ "Controls whether the catalog will cache databases, tables, manifests and partitions.");
public static final ConfigOption CACHE_EXPIRATION_INTERVAL_MS =
key("cache.expiration-interval")
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java
index 82d503b7a272..68dd54ecb744 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java
@@ -305,6 +305,30 @@ public void invalidateTable(Identifier identifier) {
}
}
+ public CacheSizes estimatedCacheSizes() {
+ long databaseCacheSize = databaseCache.estimatedSize();
+ long tableCacheSize = tableCache.estimatedSize();
+ long manifestCacheSize = 0L;
+ long manifestCacheBytes = 0L;
+ if (manifestCache != null) {
+ manifestCacheSize = manifestCache.getSegmentCacheSize();
+ manifestCacheBytes = manifestCache.getSegmentCacheBytes();
+ }
+ long partitionCacheSize = 0L;
+ if (partitionCache != null) {
+ for (Map.Entry> entry :
+ partitionCache.asMap().entrySet()) {
+ partitionCacheSize += entry.getValue().size();
+ }
+ }
+ return new CacheSizes(
+ databaseCacheSize,
+ tableCacheSize,
+ manifestCacheSize,
+ manifestCacheBytes,
+ partitionCacheSize);
+ }
+
// ================================== refresh ================================================
// following caches will affect the latency of table, so refresh method is provided for engine
@@ -314,4 +338,46 @@ public void refreshPartitions(Identifier identifier) throws TableNotExistExcepti
partitionCache.put(identifier, result);
}
}
+
+ /** Cache sizes of a caching catalog. */
+ public static class CacheSizes {
+ private final long databaseCacheSize;
+ private final long tableCacheSize;
+ private final long manifestCacheSize;
+ private final long manifestCacheBytes;
+ private final long partitionCacheSize;
+
+ public CacheSizes(
+ long databaseCacheSize,
+ long tableCacheSize,
+ long manifestCacheSize,
+ long manifestCacheBytes,
+ long partitionCacheSize) {
+ this.databaseCacheSize = databaseCacheSize;
+ this.tableCacheSize = tableCacheSize;
+ this.manifestCacheSize = manifestCacheSize;
+ this.manifestCacheBytes = manifestCacheBytes;
+ this.partitionCacheSize = partitionCacheSize;
+ }
+
+ public long databaseCacheSize() {
+ return databaseCacheSize;
+ }
+
+ public long tableCacheSize() {
+ return tableCacheSize;
+ }
+
+ public long manifestCacheSize() {
+ return manifestCacheSize;
+ }
+
+ public long manifestCacheBytes() {
+ return manifestCacheBytes;
+ }
+
+ public long partitionCacheSize() {
+ return partitionCacheSize;
+ }
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java
index c73a92062b80..4f8a1f3264de 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java
@@ -428,6 +428,8 @@ private List readManifest(
List entries =
manifestFileFactory
.create()
+ .withCacheMetrics(
+ scanMetrics != null ? scanMetrics.getCacheMetrics() : null)
.read(
manifest.fileName(),
manifest.fileSize(),
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CacheMetrics.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CacheMetrics.java
new file mode 100644
index 000000000000..fdf9ce6921c1
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CacheMetrics.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.operation.metrics;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/** Metrics for manifest file cache of a caching catalog. */
+public class CacheMetrics {
+
+ private final AtomicLong hitObject;
+ private final AtomicLong missedObject;
+
+ public CacheMetrics() {
+ this.hitObject = new AtomicLong(0);
+ this.missedObject = new AtomicLong(0);
+ }
+
+ public AtomicLong getHitObject() {
+ return hitObject;
+ }
+
+ public void increaseHitObject() {
+ this.hitObject.incrementAndGet();
+ }
+
+ public AtomicLong getMissedObject() {
+ return missedObject;
+ }
+
+ public void increaseMissedObject() {
+ this.missedObject.incrementAndGet();
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java
index 96f0aec1c0b2..fdc41baf280c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java
@@ -30,9 +30,14 @@ public class ScanMetrics {
public static final String GROUP_NAME = "scan";
private final MetricGroup metricGroup;
+ private Histogram durationHistogram;
+
+ private ScanStats latestScan;
+ private CacheMetrics cacheMetrics;
public ScanMetrics(MetricRegistry registry, String tableName) {
this.metricGroup = registry.tableMetricGroup(GROUP_NAME, tableName);
+ this.cacheMetrics = new CacheMetrics();
registerGenericScanMetrics();
}
@@ -41,17 +46,13 @@ public MetricGroup getMetricGroup() {
return metricGroup;
}
- private Histogram durationHistogram;
-
- private ScanStats latestScan;
-
public static final String LAST_SCAN_DURATION = "lastScanDuration";
public static final String SCAN_DURATION = "scanDuration";
public static final String LAST_SCANNED_MANIFESTS = "lastScannedManifests";
-
public static final String LAST_SCAN_SKIPPED_TABLE_FILES = "lastScanSkippedTableFiles";
-
public static final String LAST_SCAN_RESULTED_TABLE_FILES = "lastScanResultedTableFiles";
+ public static final String MANIFEST_HIT_CACHE = "manifestHitCache";
+ public static final String MANIFEST_MISSED_CACHE = "manifestMissedCache";
private void registerGenericScanMetrics() {
metricGroup.gauge(
@@ -66,10 +67,20 @@ private void registerGenericScanMetrics() {
metricGroup.gauge(
LAST_SCAN_RESULTED_TABLE_FILES,
() -> latestScan == null ? 0L : latestScan.getResultedTableFiles());
+ metricGroup.gauge(
+ MANIFEST_HIT_CACHE,
+ () -> cacheMetrics == null ? 0L : cacheMetrics.getHitObject().get());
+ metricGroup.gauge(
+ MANIFEST_MISSED_CACHE,
+ () -> cacheMetrics == null ? 0L : cacheMetrics.getMissedObject().get());
}
public void reportScan(ScanStats scanStats) {
latestScan = scanStats;
durationHistogram.update(scanStats.getDuration());
}
+
+ public CacheMetrics getCacheMetrics() {
+ return cacheMetrics;
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java
index 1c9d9664f22f..6f14c78107a2 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java
@@ -26,6 +26,7 @@
import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.memory.MemorySegment;
import org.apache.paimon.memory.MemorySegmentSource;
+import org.apache.paimon.operation.metrics.CacheMetrics;
import org.apache.paimon.types.RowType;
import javax.annotation.Nullable;
@@ -47,6 +48,7 @@ public class ObjectsCache {
private final ThreadLocal formatSerializer;
private final FunctionWithIOException fileSizeFunction;
private final BiFunctionWithIOE> reader;
+ private CacheMetrics cacheMetrics;
public ObjectsCache(
SegmentsCache cache,
@@ -71,8 +73,14 @@ public List read(
throws IOException {
Segments segments = cache.getIfPresents(key);
if (segments != null) {
+ if (cacheMetrics != null) {
+ cacheMetrics.increaseHitObject();
+ }
return readFromSegments(segments, readFilter, readVFilter);
} else {
+ if (cacheMetrics != null) {
+ cacheMetrics.increaseMissedObject();
+ }
if (fileSize == null) {
fileSize = fileSizeFunction.apply(key);
}
@@ -130,4 +138,8 @@ private Segments readSegments(K key, @Nullable Long fileSize, Filter List readFromIterator(
throw new RuntimeException(e);
}
}
+
+ public ObjectsFile withCacheMetrics(CacheMetrics cacheMetrics) {
+ if (cache != null) {
+ cache.withCacheMetrics(cacheMetrics);
+ }
+ return this;
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java b/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java
index d5c2178c8b6a..0ef4b13a3122 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java
@@ -90,4 +90,14 @@ public static SegmentsCache create(
return new SegmentsCache<>(pageSize, maxMemorySize, maxElementSize);
}
+
+ public long getSegmentCacheSize() {
+ return cache.estimatedSize();
+ }
+
+ public long getSegmentCacheBytes() {
+ return cache.asMap().entrySet().stream()
+ .mapToLong(entry -> weigh(entry.getKey(), entry.getValue()))
+ .sum();
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/metrics/MetricRegistryImplTest.java b/paimon-core/src/test/java/org/apache/paimon/metrics/MetricGroupTest.java
similarity index 90%
rename from paimon-core/src/test/java/org/apache/paimon/metrics/MetricRegistryImplTest.java
rename to paimon-core/src/test/java/org/apache/paimon/metrics/MetricGroupTest.java
index 27f9e70ecba8..90a063e32415 100644
--- a/paimon-core/src/test/java/org/apache/paimon/metrics/MetricRegistryImplTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/metrics/MetricGroupTest.java
@@ -24,12 +24,12 @@
import static org.assertj.core.api.Assertions.assertThat;
-/** Tests for the {@link MetricRegistryImpl}. */
-public class MetricRegistryImplTest {
+/** Tests for the {@link MetricGroup}. */
+public class MetricGroupTest {
@Test
public void testGroupRegisterMetrics() {
- MetricRegistryImpl registry = new MetricRegistryImpl();
+ TestMetricRegistry registry = new TestMetricRegistry();
MetricGroup group = registry.tableMetricGroup("commit", "myTable");
// these will fail is the registration is propagated
@@ -50,7 +50,7 @@ public void testGroupRegisterMetrics() {
@Test
public void testTolerateMetricNameCollisions() {
final String name = "abctestname";
- MetricRegistryImpl registry = new MetricRegistryImpl();
+ TestMetricRegistry registry = new TestMetricRegistry();
MetricGroup group = registry.tableMetricGroup("commit", "myTable");
Counter counter = group.counter(name);
diff --git a/paimon-core/src/main/java/org/apache/paimon/metrics/MetricRegistryImpl.java b/paimon-core/src/test/java/org/apache/paimon/metrics/TestMetricRegistry.java
similarity index 89%
rename from paimon-core/src/main/java/org/apache/paimon/metrics/MetricRegistryImpl.java
rename to paimon-core/src/test/java/org/apache/paimon/metrics/TestMetricRegistry.java
index b815364a856d..4b2870041c11 100644
--- a/paimon-core/src/main/java/org/apache/paimon/metrics/MetricRegistryImpl.java
+++ b/paimon-core/src/test/java/org/apache/paimon/metrics/TestMetricRegistry.java
@@ -20,8 +20,8 @@
import java.util.Map;
-/** Default implementation of {@link MetricRegistry}. */
-public class MetricRegistryImpl extends MetricRegistry {
+/** Implementation of {@link MetricRegistry} for tests. */
+public class TestMetricRegistry extends MetricRegistry {
@Override
protected MetricGroup createMetricGroup(String groupName, Map variables) {
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CommitMetricsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CommitMetricsTest.java
index 476790a8c0dc..6a79a0ae5807 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CommitMetricsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CommitMetricsTest.java
@@ -23,7 +23,7 @@
import org.apache.paimon.metrics.Gauge;
import org.apache.paimon.metrics.Histogram;
import org.apache.paimon.metrics.Metric;
-import org.apache.paimon.metrics.MetricRegistryImpl;
+import org.apache.paimon.metrics.TestMetricRegistry;
import org.junit.jupiter.api.Test;
@@ -234,6 +234,6 @@ private void reportAgain(CommitMetrics commitMetrics) {
}
private CommitMetrics getCommitMetrics() {
- return new CommitMetrics(new MetricRegistryImpl(), TABLE_NAME);
+ return new CommitMetrics(new TestMetricRegistry(), TABLE_NAME);
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java
index 222b99769d06..d2f5e8963d99 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java
@@ -22,7 +22,7 @@
import org.apache.paimon.metrics.Counter;
import org.apache.paimon.metrics.Gauge;
import org.apache.paimon.metrics.Metric;
-import org.apache.paimon.metrics.MetricRegistryImpl;
+import org.apache.paimon.metrics.TestMetricRegistry;
import org.junit.jupiter.api.Test;
@@ -33,7 +33,7 @@ public class CompactionMetricsTest {
@Test
public void testReportMetrics() {
- CompactionMetrics metrics = new CompactionMetrics(new MetricRegistryImpl(), "myTable");
+ CompactionMetrics metrics = new CompactionMetrics(new TestMetricRegistry(), "myTable");
assertThat(getMetric(metrics, CompactionMetrics.MAX_LEVEL0_FILE_COUNT)).isEqualTo(-1L);
assertThat(getMetric(metrics, CompactionMetrics.AVG_LEVEL0_FILE_COUNT)).isEqualTo(-1.0);
assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_THREAD_BUSY)).isEqualTo(0.0);
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java
index a0427d95cab1..7ea86a2800d1 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java
@@ -22,7 +22,7 @@
import org.apache.paimon.metrics.Histogram;
import org.apache.paimon.metrics.Metric;
import org.apache.paimon.metrics.MetricGroup;
-import org.apache.paimon.metrics.MetricRegistryImpl;
+import org.apache.paimon.metrics.TestMetricRegistry;
import org.junit.jupiter.api.Test;
@@ -48,7 +48,9 @@ public void testGenericMetricsRegistration() {
ScanMetrics.SCAN_DURATION,
ScanMetrics.LAST_SCANNED_MANIFESTS,
ScanMetrics.LAST_SCAN_SKIPPED_TABLE_FILES,
- ScanMetrics.LAST_SCAN_RESULTED_TABLE_FILES);
+ ScanMetrics.LAST_SCAN_RESULTED_TABLE_FILES,
+ ScanMetrics.MANIFEST_HIT_CACHE,
+ ScanMetrics.MANIFEST_MISSED_CACHE);
}
/** Tests that the metrics are updated properly. */
@@ -124,6 +126,6 @@ private void reportAgain(ScanMetrics scanMetrics) {
}
private ScanMetrics getScanMetrics() {
- return new ScanMetrics(new MetricRegistryImpl(), TABLE_NAME);
+ return new ScanMetrics(new TestMetricRegistry(), TABLE_NAME);
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java
index 9d3275e3ab48..7e52814a820e 100644
--- a/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java
@@ -21,6 +21,8 @@
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.metrics.TestMetricRegistry;
+import org.apache.paimon.operation.metrics.ScanMetrics;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
@@ -40,7 +42,7 @@
public class ObjectsCacheTest {
@Test
- public void test() throws IOException {
+ public void testObjectsCacheAndMetrics() throws IOException {
Map> map = new HashMap<>();
ObjectsCache cache =
new ObjectsCache<>(
@@ -56,12 +58,15 @@ public void test() throws IOException {
.map(r -> (InternalRow) r)
.iterator()));
+ ScanMetrics scanMetrics = new ScanMetrics(new TestMetricRegistry(), "table");
+ cache.withCacheMetrics(scanMetrics.getCacheMetrics());
// test empty
map.put("k1", Collections.emptyList());
List values =
cache.read(
"k1", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue());
assertThat(values).isEmpty();
+ assertThat(scanMetrics.getCacheMetrics().getMissedObject()).hasValue(1);
// test values
List expect = Arrays.asList("v1", "v2", "v3");
@@ -70,12 +75,14 @@ public void test() throws IOException {
cache.read(
"k2", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue());
assertThat(values).containsExactlyElementsOf(expect);
+ assertThat(scanMetrics.getCacheMetrics().getMissedObject()).hasValue(2);
// test cache
values =
cache.read(
"k2", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue());
assertThat(values).containsExactlyElementsOf(expect);
+ assertThat(scanMetrics.getCacheMetrics().getHitObject()).hasValue(1);
// test filter
values =