diff --git a/docs/layouts/shortcodes/generated/catalog_configuration.html b/docs/layouts/shortcodes/generated/catalog_configuration.html index 6355c9558653..03efd178b8fc 100644 --- a/docs/layouts/shortcodes/generated/catalog_configuration.html +++ b/docs/layouts/shortcodes/generated/catalog_configuration.html @@ -30,7 +30,7 @@
cache-enabled
true Boolean - Controls whether the catalog will cache databases, tables and manifests. + Controls whether the catalog will cache databases, tables, manifests and partitions.
cache.expiration-interval
diff --git a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java index b22274e011fb..d0cfbeaf39ed 100644 --- a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java +++ b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java @@ -87,7 +87,7 @@ public class CatalogOptions { .booleanType() .defaultValue(true) .withDescription( - "Controls whether the catalog will cache databases, tables and manifests."); + "Controls whether the catalog will cache databases, tables, manifests and partitions."); public static final ConfigOption CACHE_EXPIRATION_INTERVAL_MS = key("cache.expiration-interval") diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java index 82d503b7a272..68dd54ecb744 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java @@ -305,6 +305,30 @@ public void invalidateTable(Identifier identifier) { } } + public CacheSizes estimatedCacheSizes() { + long databaseCacheSize = databaseCache.estimatedSize(); + long tableCacheSize = tableCache.estimatedSize(); + long manifestCacheSize = 0L; + long manifestCacheBytes = 0L; + if (manifestCache != null) { + manifestCacheSize = manifestCache.getSegmentCacheSize(); + manifestCacheBytes = manifestCache.getSegmentCacheBytes(); + } + long partitionCacheSize = 0L; + if (partitionCache != null) { + for (Map.Entry> entry : + partitionCache.asMap().entrySet()) { + partitionCacheSize += entry.getValue().size(); + } + } + return new CacheSizes( + databaseCacheSize, + tableCacheSize, + manifestCacheSize, + manifestCacheBytes, + partitionCacheSize); + } + // ================================== refresh ================================================ // following caches will affect the latency of table, so refresh method is provided for engine @@ -314,4 +338,46 @@ public void refreshPartitions(Identifier identifier) throws TableNotExistExcepti partitionCache.put(identifier, result); } } + + /** Cache sizes of a caching catalog. */ + public static class CacheSizes { + private final long databaseCacheSize; + private final long tableCacheSize; + private final long manifestCacheSize; + private final long manifestCacheBytes; + private final long partitionCacheSize; + + public CacheSizes( + long databaseCacheSize, + long tableCacheSize, + long manifestCacheSize, + long manifestCacheBytes, + long partitionCacheSize) { + this.databaseCacheSize = databaseCacheSize; + this.tableCacheSize = tableCacheSize; + this.manifestCacheSize = manifestCacheSize; + this.manifestCacheBytes = manifestCacheBytes; + this.partitionCacheSize = partitionCacheSize; + } + + public long databaseCacheSize() { + return databaseCacheSize; + } + + public long tableCacheSize() { + return tableCacheSize; + } + + public long manifestCacheSize() { + return manifestCacheSize; + } + + public long manifestCacheBytes() { + return manifestCacheBytes; + } + + public long partitionCacheSize() { + return partitionCacheSize; + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java index c73a92062b80..4f8a1f3264de 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java @@ -428,6 +428,8 @@ private List readManifest( List entries = manifestFileFactory .create() + .withCacheMetrics( + scanMetrics != null ? scanMetrics.getCacheMetrics() : null) .read( manifest.fileName(), manifest.fileSize(), diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CacheMetrics.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CacheMetrics.java new file mode 100644 index 000000000000..fdf9ce6921c1 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CacheMetrics.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation.metrics; + +import java.util.concurrent.atomic.AtomicLong; + +/** Metrics for manifest file cache of a caching catalog. */ +public class CacheMetrics { + + private final AtomicLong hitObject; + private final AtomicLong missedObject; + + public CacheMetrics() { + this.hitObject = new AtomicLong(0); + this.missedObject = new AtomicLong(0); + } + + public AtomicLong getHitObject() { + return hitObject; + } + + public void increaseHitObject() { + this.hitObject.incrementAndGet(); + } + + public AtomicLong getMissedObject() { + return missedObject; + } + + public void increaseMissedObject() { + this.missedObject.incrementAndGet(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java index 96f0aec1c0b2..fdc41baf280c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java @@ -30,9 +30,14 @@ public class ScanMetrics { public static final String GROUP_NAME = "scan"; private final MetricGroup metricGroup; + private Histogram durationHistogram; + + private ScanStats latestScan; + private CacheMetrics cacheMetrics; public ScanMetrics(MetricRegistry registry, String tableName) { this.metricGroup = registry.tableMetricGroup(GROUP_NAME, tableName); + this.cacheMetrics = new CacheMetrics(); registerGenericScanMetrics(); } @@ -41,17 +46,13 @@ public MetricGroup getMetricGroup() { return metricGroup; } - private Histogram durationHistogram; - - private ScanStats latestScan; - public static final String LAST_SCAN_DURATION = "lastScanDuration"; public static final String SCAN_DURATION = "scanDuration"; public static final String LAST_SCANNED_MANIFESTS = "lastScannedManifests"; - public static final String LAST_SCAN_SKIPPED_TABLE_FILES = "lastScanSkippedTableFiles"; - public static final String LAST_SCAN_RESULTED_TABLE_FILES = "lastScanResultedTableFiles"; + public static final String MANIFEST_HIT_CACHE = "manifestHitCache"; + public static final String MANIFEST_MISSED_CACHE = "manifestMissedCache"; private void registerGenericScanMetrics() { metricGroup.gauge( @@ -66,10 +67,20 @@ private void registerGenericScanMetrics() { metricGroup.gauge( LAST_SCAN_RESULTED_TABLE_FILES, () -> latestScan == null ? 0L : latestScan.getResultedTableFiles()); + metricGroup.gauge( + MANIFEST_HIT_CACHE, + () -> cacheMetrics == null ? 0L : cacheMetrics.getHitObject().get()); + metricGroup.gauge( + MANIFEST_MISSED_CACHE, + () -> cacheMetrics == null ? 0L : cacheMetrics.getMissedObject().get()); } public void reportScan(ScanStats scanStats) { latestScan = scanStats; durationHistogram.update(scanStats.getDuration()); } + + public CacheMetrics getCacheMetrics() { + return cacheMetrics; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java index 1c9d9664f22f..6f14c78107a2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java @@ -26,6 +26,7 @@ import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.memory.MemorySegment; import org.apache.paimon.memory.MemorySegmentSource; +import org.apache.paimon.operation.metrics.CacheMetrics; import org.apache.paimon.types.RowType; import javax.annotation.Nullable; @@ -47,6 +48,7 @@ public class ObjectsCache { private final ThreadLocal formatSerializer; private final FunctionWithIOException fileSizeFunction; private final BiFunctionWithIOE> reader; + private CacheMetrics cacheMetrics; public ObjectsCache( SegmentsCache cache, @@ -71,8 +73,14 @@ public List read( throws IOException { Segments segments = cache.getIfPresents(key); if (segments != null) { + if (cacheMetrics != null) { + cacheMetrics.increaseHitObject(); + } return readFromSegments(segments, readFilter, readVFilter); } else { + if (cacheMetrics != null) { + cacheMetrics.increaseMissedObject(); + } if (fileSize == null) { fileSize = fileSizeFunction.apply(key); } @@ -130,4 +138,8 @@ private Segments readSegments(K key, @Nullable Long fileSize, Filter List readFromIterator( throw new RuntimeException(e); } } + + public ObjectsFile withCacheMetrics(CacheMetrics cacheMetrics) { + if (cache != null) { + cache.withCacheMetrics(cacheMetrics); + } + return this; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java b/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java index d5c2178c8b6a..0ef4b13a3122 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java @@ -90,4 +90,14 @@ public static SegmentsCache create( return new SegmentsCache<>(pageSize, maxMemorySize, maxElementSize); } + + public long getSegmentCacheSize() { + return cache.estimatedSize(); + } + + public long getSegmentCacheBytes() { + return cache.asMap().entrySet().stream() + .mapToLong(entry -> weigh(entry.getKey(), entry.getValue())) + .sum(); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/metrics/MetricRegistryImplTest.java b/paimon-core/src/test/java/org/apache/paimon/metrics/MetricGroupTest.java similarity index 90% rename from paimon-core/src/test/java/org/apache/paimon/metrics/MetricRegistryImplTest.java rename to paimon-core/src/test/java/org/apache/paimon/metrics/MetricGroupTest.java index 27f9e70ecba8..90a063e32415 100644 --- a/paimon-core/src/test/java/org/apache/paimon/metrics/MetricRegistryImplTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/metrics/MetricGroupTest.java @@ -24,12 +24,12 @@ import static org.assertj.core.api.Assertions.assertThat; -/** Tests for the {@link MetricRegistryImpl}. */ -public class MetricRegistryImplTest { +/** Tests for the {@link MetricGroup}. */ +public class MetricGroupTest { @Test public void testGroupRegisterMetrics() { - MetricRegistryImpl registry = new MetricRegistryImpl(); + TestMetricRegistry registry = new TestMetricRegistry(); MetricGroup group = registry.tableMetricGroup("commit", "myTable"); // these will fail is the registration is propagated @@ -50,7 +50,7 @@ public void testGroupRegisterMetrics() { @Test public void testTolerateMetricNameCollisions() { final String name = "abctestname"; - MetricRegistryImpl registry = new MetricRegistryImpl(); + TestMetricRegistry registry = new TestMetricRegistry(); MetricGroup group = registry.tableMetricGroup("commit", "myTable"); Counter counter = group.counter(name); diff --git a/paimon-core/src/main/java/org/apache/paimon/metrics/MetricRegistryImpl.java b/paimon-core/src/test/java/org/apache/paimon/metrics/TestMetricRegistry.java similarity index 89% rename from paimon-core/src/main/java/org/apache/paimon/metrics/MetricRegistryImpl.java rename to paimon-core/src/test/java/org/apache/paimon/metrics/TestMetricRegistry.java index b815364a856d..4b2870041c11 100644 --- a/paimon-core/src/main/java/org/apache/paimon/metrics/MetricRegistryImpl.java +++ b/paimon-core/src/test/java/org/apache/paimon/metrics/TestMetricRegistry.java @@ -20,8 +20,8 @@ import java.util.Map; -/** Default implementation of {@link MetricRegistry}. */ -public class MetricRegistryImpl extends MetricRegistry { +/** Implementation of {@link MetricRegistry} for tests. */ +public class TestMetricRegistry extends MetricRegistry { @Override protected MetricGroup createMetricGroup(String groupName, Map variables) { diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CommitMetricsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CommitMetricsTest.java index 476790a8c0dc..6a79a0ae5807 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CommitMetricsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CommitMetricsTest.java @@ -23,7 +23,7 @@ import org.apache.paimon.metrics.Gauge; import org.apache.paimon.metrics.Histogram; import org.apache.paimon.metrics.Metric; -import org.apache.paimon.metrics.MetricRegistryImpl; +import org.apache.paimon.metrics.TestMetricRegistry; import org.junit.jupiter.api.Test; @@ -234,6 +234,6 @@ private void reportAgain(CommitMetrics commitMetrics) { } private CommitMetrics getCommitMetrics() { - return new CommitMetrics(new MetricRegistryImpl(), TABLE_NAME); + return new CommitMetrics(new TestMetricRegistry(), TABLE_NAME); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java index 222b99769d06..d2f5e8963d99 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java @@ -22,7 +22,7 @@ import org.apache.paimon.metrics.Counter; import org.apache.paimon.metrics.Gauge; import org.apache.paimon.metrics.Metric; -import org.apache.paimon.metrics.MetricRegistryImpl; +import org.apache.paimon.metrics.TestMetricRegistry; import org.junit.jupiter.api.Test; @@ -33,7 +33,7 @@ public class CompactionMetricsTest { @Test public void testReportMetrics() { - CompactionMetrics metrics = new CompactionMetrics(new MetricRegistryImpl(), "myTable"); + CompactionMetrics metrics = new CompactionMetrics(new TestMetricRegistry(), "myTable"); assertThat(getMetric(metrics, CompactionMetrics.MAX_LEVEL0_FILE_COUNT)).isEqualTo(-1L); assertThat(getMetric(metrics, CompactionMetrics.AVG_LEVEL0_FILE_COUNT)).isEqualTo(-1.0); assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_THREAD_BUSY)).isEqualTo(0.0); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java index a0427d95cab1..7ea86a2800d1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java @@ -22,7 +22,7 @@ import org.apache.paimon.metrics.Histogram; import org.apache.paimon.metrics.Metric; import org.apache.paimon.metrics.MetricGroup; -import org.apache.paimon.metrics.MetricRegistryImpl; +import org.apache.paimon.metrics.TestMetricRegistry; import org.junit.jupiter.api.Test; @@ -48,7 +48,9 @@ public void testGenericMetricsRegistration() { ScanMetrics.SCAN_DURATION, ScanMetrics.LAST_SCANNED_MANIFESTS, ScanMetrics.LAST_SCAN_SKIPPED_TABLE_FILES, - ScanMetrics.LAST_SCAN_RESULTED_TABLE_FILES); + ScanMetrics.LAST_SCAN_RESULTED_TABLE_FILES, + ScanMetrics.MANIFEST_HIT_CACHE, + ScanMetrics.MANIFEST_MISSED_CACHE); } /** Tests that the metrics are updated properly. */ @@ -124,6 +126,6 @@ private void reportAgain(ScanMetrics scanMetrics) { } private ScanMetrics getScanMetrics() { - return new ScanMetrics(new MetricRegistryImpl(), TABLE_NAME); + return new ScanMetrics(new TestMetricRegistry(), TABLE_NAME); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java index 9d3275e3ab48..7e52814a820e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java @@ -21,6 +21,8 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.metrics.TestMetricRegistry; +import org.apache.paimon.operation.metrics.ScanMetrics; import org.apache.paimon.options.MemorySize; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -40,7 +42,7 @@ public class ObjectsCacheTest { @Test - public void test() throws IOException { + public void testObjectsCacheAndMetrics() throws IOException { Map> map = new HashMap<>(); ObjectsCache cache = new ObjectsCache<>( @@ -56,12 +58,15 @@ public void test() throws IOException { .map(r -> (InternalRow) r) .iterator())); + ScanMetrics scanMetrics = new ScanMetrics(new TestMetricRegistry(), "table"); + cache.withCacheMetrics(scanMetrics.getCacheMetrics()); // test empty map.put("k1", Collections.emptyList()); List values = cache.read( "k1", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue()); assertThat(values).isEmpty(); + assertThat(scanMetrics.getCacheMetrics().getMissedObject()).hasValue(1); // test values List expect = Arrays.asList("v1", "v2", "v3"); @@ -70,12 +75,14 @@ public void test() throws IOException { cache.read( "k2", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue()); assertThat(values).containsExactlyElementsOf(expect); + assertThat(scanMetrics.getCacheMetrics().getMissedObject()).hasValue(2); // test cache values = cache.read( "k2", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue()); assertThat(values).containsExactlyElementsOf(expect); + assertThat(scanMetrics.getCacheMetrics().getHitObject()).hasValue(1); // test filter values =