From 95e89c327aa997a5c3e5d40b6a2618ee9b2f4e3c Mon Sep 17 00:00:00 2001 From: Daniel Pettersen <25056311+dtpett@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:16:35 +0100 Subject: [PATCH 1/4] Logic for registering metrics per index --- .../PrometheusMetricsCollector.java | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) diff --git a/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java b/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java index 4b21ab66..63aa8da8 100644 --- a/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java +++ b/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java @@ -1,8 +1,14 @@ package org.compuscene.metrics.prometheus; +import java.util.Map; + import io.prometheus.client.Summary; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.http.HttpStats; import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.indices.breaker.AllCircuitBreakerStats; @@ -342,6 +348,233 @@ private void updateIndicesMetrics(NodeIndicesStats idx) { } } + private void registerPerIndexMetrics() { + catalog.registerGauge("index_status", "Index status", "index"); + catalog.registerGauge("index_replicas_number", "Number of replicas", "index"); + catalog.registerGauge("index_shards_number", "Number of shards", "type", "index"); + + catalog.registerGauge("index_doc_number", "Total number of documents", "index", "context"); + catalog.registerGauge("index_doc_deleted_number", "Number of deleted documents", "index", "context"); + + catalog.registerGauge("index_store_size_bytes", "Store size of the indices in bytes", "index", "context"); + + catalog.registerGauge("index_indexing_delete_count", "Count of documents deleted", "index", "context"); + catalog.registerGauge("index_indexing_delete_current_number", "Current rate of documents deleted", "index", "context"); + catalog.registerGauge("index_indexing_delete_time_seconds", "Time spent while deleting documents", "index", "context"); + catalog.registerGauge("index_indexing_index_count", "Count of documents indexed", "index", "context"); + catalog.registerGauge("index_indexing_index_current_number", "Current rate of documents indexed", "index", "context"); + catalog.registerGauge("index_indexing_index_failed_count", "Count of failed to index documents", "index", "context"); + catalog.registerGauge("index_indexing_index_time_seconds", "Time spent while indexing documents", "index", "context"); + catalog.registerGauge("index_indexing_noop_update_count", "Count of noop document updates", "index", "context"); + catalog.registerGauge("index_indexing_is_throttled_bool", "Is indexing throttling ?", "index", "context"); + catalog.registerGauge("index_indexing_throttle_time_seconds", "Time spent while throttling", "index", "context"); + + catalog.registerGauge("index_get_count", "Count of get commands", "index", "context"); + catalog.registerGauge("index_get_time_seconds", "Time spent while get commands", "index", "context"); + catalog.registerGauge("index_get_exists_count", "Count of existing documents when get command", "index", "context"); + catalog.registerGauge("index_get_exists_time_seconds", "Time spent while existing documents get command", "index", "context"); + catalog.registerGauge("index_get_missing_count", "Count of missing documents when get command", "index", "context"); + catalog.registerGauge("index_get_missing_time_seconds", "Time spent while missing documents get command", "index", "context"); + catalog.registerGauge("index_get_current_number", "Current rate of get commands", "index", "context"); + + catalog.registerGauge("index_search_open_contexts_number", "Number of search open contexts", "index", "context"); + catalog.registerGauge("index_search_fetch_count", "Count of search fetches", "index", "context"); + catalog.registerGauge("index_search_fetch_current_number", "Current rate of search fetches", "index", "context"); + catalog.registerGauge("index_search_fetch_time_seconds", "Time spent while search fetches", "index", "context"); + catalog.registerGauge("index_search_query_count", "Count of search queries", "index", "context"); + catalog.registerGauge("index_search_query_current_number", "Current rate of search queries", "index", "context"); + catalog.registerGauge("index_search_query_time_seconds", "Time spent while search queries", "index", "context"); + catalog.registerGauge("index_search_scroll_count", "Count of search scrolls", "index", "context"); + catalog.registerGauge("index_search_scroll_current_number", "Current rate of search scrolls", "index", "context"); + catalog.registerGauge("index_search_scroll_time_seconds", "Time spent while search scrolls", "index", "context"); + + catalog.registerGauge("index_merges_current_number", "Current rate of merges", "index", "context"); + catalog.registerGauge("index_merges_current_docs_number", "Current rate of documents merged", "index", "context"); + catalog.registerGauge("index_merges_current_size_bytes", "Current rate of bytes merged", "index", "context"); + catalog.registerGauge("index_merges_total_number", "Count of merges", "index", "context"); + catalog.registerGauge("index_merges_total_time_seconds", "Time spent while merging", "index", "context"); + catalog.registerGauge("index_merges_total_docs_count", "Count of documents merged", "index", "context"); + catalog.registerGauge("index_merges_total_size_bytes", "Count of bytes of merged documents", "index", "context"); + catalog.registerGauge("index_merges_total_stopped_time_seconds", "Time spent while merge process stopped", "index", "context"); + catalog.registerGauge("index_merges_total_throttled_time_seconds", "Time spent while merging when throttling", "index", "context"); + catalog.registerGauge("index_merges_total_auto_throttle_bytes", "Bytes merged while throttling", "index", "context"); + + catalog.registerGauge("index_refresh_total_count", "Count of refreshes", "index", "context"); + catalog.registerGauge("index_refresh_total_time_seconds", "Time spent while refreshes", "index", "context"); + catalog.registerGauge("index_refresh_listeners_number", "Number of refresh listeners", "index", "context"); + + catalog.registerGauge("index_flush_total_count", "Count of flushes", "index", "context"); + catalog.registerGauge("index_flush_total_time_seconds", "Total time spent while flushes", "index", "context"); + + catalog.registerGauge("index_querycache_cache_count", "Count of queries in cache", "index", "context"); + catalog.registerGauge("index_querycache_cache_size_bytes", "Query cache size", "index", "context"); + catalog.registerGauge("index_querycache_evictions_count", "Count of evictions in query cache", "index", "context"); + catalog.registerGauge("index_querycache_hit_count", "Count of hits in query cache", "index", "context"); + catalog.registerGauge("index_querycache_memory_size_bytes", "Memory usage of query cache", "index", "context"); + catalog.registerGauge("index_querycache_miss_number", "Count of misses in query cache", "index", "context"); + catalog.registerGauge("index_querycache_total_number", "Count of usages of query cache", "index", "context"); + + catalog.registerGauge("index_fielddata_memory_size_bytes", "Memory usage of field date cache", "index", "context"); + catalog.registerGauge("index_fielddata_evictions_count", "Count of evictions in field data cache", "index", "context"); + + // Percolator cache was removed in ES 5.x + // See https://github.com/elastic/elasticsearch/commit/80fee8666ff5dd61ba29b175857cf42ce3b9eab9 + + catalog.registerGauge("index_completion_size_bytes", "Size of completion suggest statistics", "index", "context"); + + catalog.registerGauge("index_segments_number", "Current number of segments", "index", "context"); + catalog.registerGauge("index_segments_memory_bytes", "Memory used by segments", "type", "index", "context"); + + catalog.registerGauge("index_suggest_current_number", "Current rate of suggests", "index", "context"); + catalog.registerGauge("index_suggest_count", "Count of suggests", "index", "context"); + catalog.registerGauge("index_suggest_time_seconds", "Time spent while making suggests", "index", "context"); + + catalog.registerGauge("index_requestcache_memory_size_bytes", "Memory used for request cache", "index", "context"); + catalog.registerGauge("index_requestcache_hit_count", "Number of hits in request cache", "index", "context"); + catalog.registerGauge("index_requestcache_miss_count", "Number of misses in request cache", "index", "context"); + catalog.registerGauge("index_requestcache_evictions_count", "Number of evictions in request cache", "index", "context"); + + catalog.registerGauge("index_recovery_current_number", "Current number of recoveries", "type", "index", "context"); + catalog.registerGauge("index_recovery_throttle_time_seconds", "Time spent while throttling recoveries", "index", "context"); + + catalog.registerGauge("index_translog_operations_number", "Current number of translog operations", "index", "context"); + catalog.registerGauge("index_translog_size_bytes", "Translog size", "index", "context"); + catalog.registerGauge("index_translog_uncommitted_operations_number", "Current number of uncommitted translog operations", "index", "context"); + catalog.registerGauge("index_translog_uncommitted_size_bytes", "Translog uncommitted size", "index", "context"); + + catalog.registerGauge("index_warmer_current_number", "Current number of warmer", "index", "context"); + catalog.registerGauge("index_warmer_time_seconds", "Time spent during warmers", "index", "context"); + catalog.registerGauge("index_warmer_count", "Counter of warmers", "index", "context"); + } + + private void updatePerIndexMetrics(ClusterHealthResponse chr, IndicesStatsResponse isr) { + + if (chr != null && isr != null) { + for (Map.Entry entry : isr.getIndices().entrySet()) { + String indexName = entry.getKey(); + ClusterIndexHealth cih = chr.getIndices().get(indexName); + catalog.setGauge("index_status", cih.getStatus().value(), indexName); + catalog.setGauge("index_replicas_number", cih.getNumberOfReplicas(), indexName); + catalog.setGauge("index_shards_number", cih.getActiveShards(), "active", indexName); + catalog.setGauge("index_shards_number", cih.getNumberOfShards(), "shards", indexName); + catalog.setGauge("index_shards_number", cih.getActivePrimaryShards(), "active_primary", indexName); + catalog.setGauge("index_shards_number", cih.getInitializingShards(), "initializing", indexName); + catalog.setGauge("index_shards_number", cih.getRelocatingShards(), "relocating", indexName); + catalog.setGauge("index_shards_number", cih.getUnassignedShards(), "unassigned", indexName); + IndexStats indexStats = entry.getValue(); + updatePerIndexContextMetrics(indexName, "total", indexStats.getTotal()); + updatePerIndexContextMetrics(indexName, "primaries", indexStats.getPrimaries()); + } + } + } + + @SuppressWarnings("checkstyle:LineLength") + private void updatePerIndexContextMetrics(String indexName, String context, CommonStats idx) { + catalog.setGauge("index_doc_number", idx.getDocs().getCount(), indexName, context); + catalog.setGauge("index_doc_deleted_number", idx.getDocs().getDeleted(), indexName, context); + + catalog.setGauge("index_store_size_bytes", idx.getStore().getSizeInBytes(), indexName, context); + + catalog.setGauge("index_indexing_delete_count", idx.getIndexing().getTotal().getDeleteCount(), indexName, context); + catalog.setGauge("index_indexing_delete_current_number", idx.getIndexing().getTotal().getDeleteCurrent(), indexName, context); + catalog.setGauge("index_indexing_delete_time_seconds", idx.getIndexing().getTotal().getDeleteTime().seconds(), indexName, context); + catalog.setGauge("index_indexing_index_count", idx.getIndexing().getTotal().getIndexCount(), indexName, context); + catalog.setGauge("index_indexing_index_current_number", idx.getIndexing().getTotal().getIndexCurrent(), indexName, context); + catalog.setGauge("index_indexing_index_failed_count", idx.getIndexing().getTotal().getIndexFailedCount(), indexName, context); + catalog.setGauge("index_indexing_index_time_seconds", idx.getIndexing().getTotal().getIndexTime().seconds(), indexName, context); + catalog.setGauge("index_indexing_noop_update_count", idx.getIndexing().getTotal().getNoopUpdateCount(), indexName, context); + catalog.setGauge("index_indexing_is_throttled_bool", idx.getIndexing().getTotal().isThrottled() ? 1 : 0, indexName, context); + catalog.setGauge("index_indexing_throttle_time_seconds", idx.getIndexing().getTotal().getThrottleTime().seconds(), indexName, context); + + catalog.setGauge("index_get_count", idx.getGet().getCount(), indexName, context); + catalog.setGauge("index_get_time_seconds", idx.getGet().getTimeInMillis() / 1000.0, indexName, context); + catalog.setGauge("index_get_exists_count", idx.getGet().getExistsCount(), indexName, context); + catalog.setGauge("index_get_exists_time_seconds", idx.getGet().getExistsTimeInMillis() / 1000.0, indexName, context); + catalog.setGauge("index_get_missing_count", idx.getGet().getMissingCount(), indexName, context); + catalog.setGauge("index_get_missing_time_seconds", idx.getGet().getMissingTimeInMillis() / 1000.0, indexName, context); + catalog.setGauge("index_get_current_number", idx.getGet().current(), indexName, context); + + catalog.setGauge("index_search_open_contexts_number", idx.getSearch().getOpenContexts(), indexName, context); + catalog.setGauge("index_search_fetch_count", idx.getSearch().getTotal().getFetchCount(), indexName, context); + catalog.setGauge("index_search_fetch_current_number", idx.getSearch().getTotal().getFetchCurrent(), indexName, context); + catalog.setGauge("index_search_fetch_time_seconds", idx.getSearch().getTotal().getFetchTimeInMillis() / 1000.0, indexName, context); + catalog.setGauge("index_search_query_count", idx.getSearch().getTotal().getQueryCount(), indexName, context); + catalog.setGauge("index_search_query_current_number", idx.getSearch().getTotal().getQueryCurrent(), indexName, context); + catalog.setGauge("index_search_query_time_seconds", idx.getSearch().getTotal().getQueryTimeInMillis() / 1000.0, indexName, context); + catalog.setGauge("index_search_scroll_count", idx.getSearch().getTotal().getScrollCount(), indexName, context); + catalog.setGauge("index_search_scroll_current_number", idx.getSearch().getTotal().getScrollCurrent(), indexName, context); + catalog.setGauge("index_search_scroll_time_seconds", idx.getSearch().getTotal().getScrollTimeInMillis() / 1000.0, indexName, context); + + catalog.setGauge("index_merges_current_number", idx.getMerge().getCurrent(), indexName, context); + catalog.setGauge("index_merges_current_docs_number", idx.getMerge().getCurrentNumDocs(), indexName, context); + catalog.setGauge("index_merges_current_size_bytes", idx.getMerge().getCurrentSizeInBytes(), indexName, context); + catalog.setGauge("index_merges_total_number", idx.getMerge().getTotal(), indexName, context); + catalog.setGauge("index_merges_total_time_seconds", idx.getMerge().getTotalTimeInMillis() / 1000.0, indexName, context); + catalog.setGauge("index_merges_total_docs_count", idx.getMerge().getTotalNumDocs(), indexName, context); + catalog.setGauge("index_merges_total_size_bytes", idx.getMerge().getTotalSizeInBytes(), indexName, context); + catalog.setGauge("index_merges_total_stopped_time_seconds", idx.getMerge().getTotalStoppedTimeInMillis() / 1000.0, indexName, context); + catalog.setGauge("index_merges_total_throttled_time_seconds", idx.getMerge().getTotalThrottledTimeInMillis() / 1000.0, indexName, context); + catalog.setGauge("index_merges_total_auto_throttle_bytes", idx.getMerge().getTotalBytesPerSecAutoThrottle(), indexName, context); + + catalog.setGauge("index_refresh_total_count", idx.getRefresh().getTotal(), indexName, context); + catalog.setGauge("index_refresh_total_time_seconds", idx.getRefresh().getTotalTimeInMillis() / 1000.0, indexName, context); + catalog.setGauge("index_refresh_listeners_number", idx.getRefresh().getListeners(), indexName, context); + + catalog.setGauge("index_flush_total_count", idx.getFlush().getTotal(), indexName, context); + catalog.setGauge("index_flush_total_time_seconds", idx.getFlush().getTotalTimeInMillis() / 1000.0, indexName, context); + + catalog.setGauge("index_querycache_cache_count", idx.getQueryCache().getCacheCount(), indexName, context); + catalog.setGauge("index_querycache_cache_size_bytes", idx.getQueryCache().getCacheSize(), indexName, context); + catalog.setGauge("index_querycache_evictions_count", idx.getQueryCache().getEvictions(), indexName, context); + catalog.setGauge("index_querycache_hit_count", idx.getQueryCache().getHitCount(), indexName, context); + catalog.setGauge("index_querycache_memory_size_bytes", idx.getQueryCache().getMemorySizeInBytes(), indexName, context); + catalog.setGauge("index_querycache_miss_number", idx.getQueryCache().getMissCount(), indexName, context); + catalog.setGauge("index_querycache_total_number", idx.getQueryCache().getTotalCount(), indexName, context); + + catalog.setGauge("index_fielddata_memory_size_bytes", idx.getFieldData().getMemorySizeInBytes(), indexName, context); + catalog.setGauge("index_fielddata_evictions_count", idx.getFieldData().getEvictions(), indexName, context); + + // Percolator cache was removed in ES 5.x + // See https://github.com/elastic/elasticsearch/commit/80fee8666ff5dd61ba29b175857cf42ce3b9eab9 + + catalog.setGauge("index_completion_size_bytes", idx.getCompletion().getSizeInBytes(), indexName, context); + + catalog.setGauge("index_segments_number", idx.getSegments().getCount(), indexName, context); + catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getMemoryInBytes(), "all", indexName, context); + catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getBitsetMemoryInBytes(), "bitset", indexName, context); + catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getDocValuesMemoryInBytes(), "docvalues", indexName, context); + catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getIndexWriterMemoryInBytes(), "indexwriter", indexName, context); + catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getNormsMemoryInBytes(), "norms", indexName, context); + catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getStoredFieldsMemoryInBytes(), "storefields", indexName, context); + catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getTermsMemoryInBytes(), "terms", indexName, context); + catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getTermVectorsMemoryInBytes(), "termvectors", indexName, context); + catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getVersionMapMemoryInBytes(), "versionmap", indexName, context); + catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getPointsMemoryInBytes(), "points", indexName, context); + + catalog.setGauge("index_suggest_current_number", idx.getSearch().getTotal().getSuggestCurrent(), indexName, context); + catalog.setGauge("index_suggest_count", idx.getSearch().getTotal().getSuggestCount(), indexName, context); + catalog.setGauge("index_suggest_time_seconds", idx.getSearch().getTotal().getSuggestTimeInMillis() / 1000.0, indexName, context); + + catalog.setGauge("index_requestcache_memory_size_bytes", idx.getRequestCache().getMemorySizeInBytes(), indexName, context); + catalog.setGauge("index_requestcache_hit_count", idx.getRequestCache().getHitCount(), indexName, context); + catalog.setGauge("index_requestcache_miss_count", idx.getRequestCache().getMissCount(), indexName, context); + catalog.setGauge("index_requestcache_evictions_count", idx.getRequestCache().getEvictions(), indexName, context); + + catalog.setGauge("index_recovery_current_number", idx.getRecoveryStats().currentAsSource(), "source", indexName, context); + catalog.setGauge("index_recovery_current_number", idx.getRecoveryStats().currentAsTarget(), "target", indexName, context); + catalog.setGauge("index_recovery_throttle_time_seconds", idx.getRecoveryStats().throttleTime().getSeconds(), indexName, context); + + catalog.setGauge("index_translog_operations_number", idx.getTranslog().estimatedNumberOfOperations(), indexName, context); + catalog.setGauge("index_translog_size_bytes", idx.getTranslog().getTranslogSizeInBytes(), indexName, context); + catalog.setGauge("index_translog_uncommitted_operations_number", idx.getTranslog().getUncommittedOperations(), indexName, context); + catalog.setGauge("index_translog_uncommitted_size_bytes", idx.getTranslog().getUncommittedSizeInBytes(), indexName, context); + + catalog.setGauge("index_warmer_current_number", idx.getWarmer().current(), indexName, context); + catalog.setGauge("index_warmer_time_seconds", idx.getWarmer().totalTimeInMillis(), indexName, context); + catalog.setGauge("index_warmer_count", idx.getWarmer().total(), indexName, context); + } + + private void registerTransportMetrics() { catalog.registerGauge("transport_server_open_number", "Opened server connections", "node", "nodeId"); catalog.registerCounter("transport_rx_packets_count", "Received packets", "node", "nodeId"); From 5c9d4e3f72d5a86255c5ab18777e6dad5cbe5ed6 Mon Sep 17 00:00:00 2001 From: Daniel Pettersen <25056311+dtpett@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:18:45 +0100 Subject: [PATCH 2/4] Remove unsupported metrics for ES2.x --- .../prometheus/PrometheusMetricsCollector.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java b/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java index 63aa8da8..91f8056f 100644 --- a/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java +++ b/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java @@ -401,7 +401,6 @@ private void registerPerIndexMetrics() { catalog.registerGauge("index_refresh_total_count", "Count of refreshes", "index", "context"); catalog.registerGauge("index_refresh_total_time_seconds", "Time spent while refreshes", "index", "context"); - catalog.registerGauge("index_refresh_listeners_number", "Number of refresh listeners", "index", "context"); catalog.registerGauge("index_flush_total_count", "Count of flushes", "index", "context"); catalog.registerGauge("index_flush_total_time_seconds", "Total time spent while flushes", "index", "context"); @@ -425,10 +424,6 @@ private void registerPerIndexMetrics() { catalog.registerGauge("index_segments_number", "Current number of segments", "index", "context"); catalog.registerGauge("index_segments_memory_bytes", "Memory used by segments", "type", "index", "context"); - catalog.registerGauge("index_suggest_current_number", "Current rate of suggests", "index", "context"); - catalog.registerGauge("index_suggest_count", "Count of suggests", "index", "context"); - catalog.registerGauge("index_suggest_time_seconds", "Time spent while making suggests", "index", "context"); - catalog.registerGauge("index_requestcache_memory_size_bytes", "Memory used for request cache", "index", "context"); catalog.registerGauge("index_requestcache_hit_count", "Number of hits in request cache", "index", "context"); catalog.registerGauge("index_requestcache_miss_count", "Number of misses in request cache", "index", "context"); @@ -439,8 +434,6 @@ private void registerPerIndexMetrics() { catalog.registerGauge("index_translog_operations_number", "Current number of translog operations", "index", "context"); catalog.registerGauge("index_translog_size_bytes", "Translog size", "index", "context"); - catalog.registerGauge("index_translog_uncommitted_operations_number", "Current number of uncommitted translog operations", "index", "context"); - catalog.registerGauge("index_translog_uncommitted_size_bytes", "Translog uncommitted size", "index", "context"); catalog.registerGauge("index_warmer_current_number", "Current number of warmer", "index", "context"); catalog.registerGauge("index_warmer_time_seconds", "Time spent during warmers", "index", "context"); @@ -518,7 +511,6 @@ private void updatePerIndexContextMetrics(String indexName, String context, Comm catalog.setGauge("index_refresh_total_count", idx.getRefresh().getTotal(), indexName, context); catalog.setGauge("index_refresh_total_time_seconds", idx.getRefresh().getTotalTimeInMillis() / 1000.0, indexName, context); - catalog.setGauge("index_refresh_listeners_number", idx.getRefresh().getListeners(), indexName, context); catalog.setGauge("index_flush_total_count", idx.getFlush().getTotal(), indexName, context); catalog.setGauge("index_flush_total_time_seconds", idx.getFlush().getTotalTimeInMillis() / 1000.0, indexName, context); @@ -549,11 +541,6 @@ private void updatePerIndexContextMetrics(String indexName, String context, Comm catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getTermsMemoryInBytes(), "terms", indexName, context); catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getTermVectorsMemoryInBytes(), "termvectors", indexName, context); catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getVersionMapMemoryInBytes(), "versionmap", indexName, context); - catalog.setGauge("index_segments_memory_bytes", idx.getSegments().getPointsMemoryInBytes(), "points", indexName, context); - - catalog.setGauge("index_suggest_current_number", idx.getSearch().getTotal().getSuggestCurrent(), indexName, context); - catalog.setGauge("index_suggest_count", idx.getSearch().getTotal().getSuggestCount(), indexName, context); - catalog.setGauge("index_suggest_time_seconds", idx.getSearch().getTotal().getSuggestTimeInMillis() / 1000.0, indexName, context); catalog.setGauge("index_requestcache_memory_size_bytes", idx.getRequestCache().getMemorySizeInBytes(), indexName, context); catalog.setGauge("index_requestcache_hit_count", idx.getRequestCache().getHitCount(), indexName, context); @@ -566,8 +553,6 @@ private void updatePerIndexContextMetrics(String indexName, String context, Comm catalog.setGauge("index_translog_operations_number", idx.getTranslog().estimatedNumberOfOperations(), indexName, context); catalog.setGauge("index_translog_size_bytes", idx.getTranslog().getTranslogSizeInBytes(), indexName, context); - catalog.setGauge("index_translog_uncommitted_operations_number", idx.getTranslog().getUncommittedOperations(), indexName, context); - catalog.setGauge("index_translog_uncommitted_size_bytes", idx.getTranslog().getUncommittedSizeInBytes(), indexName, context); catalog.setGauge("index_warmer_current_number", idx.getWarmer().current(), indexName, context); catalog.setGauge("index_warmer_time_seconds", idx.getWarmer().totalTimeInMillis(), indexName, context); From b3c5880874bdd534dc563e83e4802c842dd6fc16 Mon Sep 17 00:00:00 2001 From: Daniel Pettersen <25056311+dtpett@users.noreply.github.com> Date: Mon, 8 Jan 2024 18:27:09 +0100 Subject: [PATCH 3/4] Backport functionality for status per index From ES 5.x til 2.x branch --- .../PrometheusMetricsCollector.java | 4 +- .../action/NodePrometheusMetricsResponse.java | 40 ++++++++++++++--- .../TransportNodePrometheusMetricsAction.java | 24 ++++++++-- .../indices/stats/PackageAccessHelper.java | 44 +++++++++++++++++++ .../RestPrometheusMetricsAction.java | 2 +- 5 files changed, 104 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/elasticsearch/action/admin/indices/stats/PackageAccessHelper.java diff --git a/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java b/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java index 91f8056f..0111cd94 100644 --- a/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java +++ b/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java @@ -43,6 +43,7 @@ private void registerMetrics() { registerClusterMetrics(); registerJVMMetrics(); registerIndicesMetrics(); + registerPerIndexMetrics(); registerTransportMetrics(); registerHTTPMetrics(); registerScriptMetrics(); @@ -720,13 +721,14 @@ private void updateFsMetrics(FsInfo fs) { } } - public void updateMetrics(ClusterHealthResponse clusterHealthResponse, NodeStats nodeStats) { + public void updateMetrics(ClusterHealthResponse clusterHealthResponse, NodeStats nodeStats, IndicesStatsResponse indicesStats) { Summary.Timer timer = catalog.startSummaryTimer("metrics_generate_time_seconds", node, nodeId); updateClusterMetrics(clusterHealthResponse); updateJVMMetrics(nodeStats.getJvm()); updateIndicesMetrics(nodeStats.getIndices()); + updatePerIndexMetrics(clusterHealthResponse, indicesStats); updateTransportMetrics(nodeStats.getTransport()); updateHTTPMetrics(nodeStats.getHttp()); updateScriptMetrics(nodeStats.getScriptStats()); diff --git a/src/main/java/org/elasticsearch/action/NodePrometheusMetricsResponse.java b/src/main/java/org/elasticsearch/action/NodePrometheusMetricsResponse.java index 6265786a..f6651ca6 100644 --- a/src/main/java/org/elasticsearch/action/NodePrometheusMetricsResponse.java +++ b/src/main/java/org/elasticsearch/action/NodePrometheusMetricsResponse.java @@ -2,24 +2,39 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.PackageAccessHelper; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.support.broadcast.BroadcastResponse; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; +import java.util.Arrays; public class NodePrometheusMetricsResponse extends ActionResponse { private ClusterHealthResponse clusterHealth; private NodeStats nodeStats; + @Nullable + private IndicesStatsResponse indicesStats; public NodePrometheusMetricsResponse() { } - public NodePrometheusMetricsResponse(ClusterHealthResponse clusterHealth, NodeStats nodesStats) { + public NodePrometheusMetricsResponse(ClusterHealthResponse clusterHealth, NodeStats nodesStats, + @Nullable IndicesStatsResponse indicesStats) { this.clusterHealth = clusterHealth; this.nodeStats = nodesStats; + this.indicesStats = indicesStats; } + public NodePrometheusMetricsResponse readNodePrometheusMetrics(StreamInput in) throws IOException { + NodePrometheusMetricsResponse metrics = new NodePrometheusMetricsResponse(); + metrics.readFrom(in); + return metrics; + } public ClusterHealthResponse getClusterHealth() { return this.clusterHealth; } @@ -28,10 +43,9 @@ public NodeStats getNodeStats() { return this.nodeStats; } - public static NodePrometheusMetricsResponse readNodePrometheusMetrics(StreamInput in) throws IOException { - NodePrometheusMetricsResponse metrics = new NodePrometheusMetricsResponse(); - metrics.readFrom(in); - return metrics; + @Nullable + public IndicesStatsResponse getIndicesStats() { + return this.indicesStats; } @Override @@ -39,6 +53,17 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); clusterHealth = ClusterHealthResponse.readResponseFrom(in); nodeStats = NodeStats.readNodeStats(in); + BroadcastResponse br = new BroadcastResponse(); + br.readFrom(in); + int size = in.readInt(); + ShardStats[] ss = new ShardStats[size]; + for (int i = 0; i < size; i ++) { + ss[i] = ShardStats.readShardStats(in); + } + indicesStats = PackageAccessHelper.createIndicesStatsResponse( + ss, br.getTotalShards(), br.getSuccessfulShards(), br.getFailedShards(), + Arrays.asList(br.getShardFailures()) + ); } @Override @@ -46,5 +71,10 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); clusterHealth.writeTo(out); nodeStats.writeTo(out); + if (indicesStats != null) { + //indicesStats.writeTo(out); + ((BroadcastResponse) indicesStats).writeTo(out); + out.writeGenericValue(indicesStats.getShards()); + } } } diff --git a/src/main/java/org/elasticsearch/action/TransportNodePrometheusMetricsAction.java b/src/main/java/org/elasticsearch/action/TransportNodePrometheusMetricsAction.java index a0a20ba7..3bf37b19 100644 --- a/src/main/java/org/elasticsearch/action/TransportNodePrometheusMetricsAction.java +++ b/src/main/java/org/elasticsearch/action/TransportNodePrometheusMetricsAction.java @@ -5,10 +5,13 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; @@ -37,6 +40,7 @@ private class AsyncAction { private final ClusterHealthRequest healthRequest; private final NodesStatsRequest nodesStatsRequest; private ClusterHealthResponse clusterHealthResponse; + private NodesStatsResponse nodesStatsResponse; private AsyncAction(NodePrometheusMetricsRequest request, ActionListener listener) { this.listener = listener; @@ -49,10 +53,23 @@ private void start() { client.admin().cluster().health(healthRequest, clusterHealthResponseActionListener); } + private ActionListener indicesStatsResponseActionListener = + new ActionListener() { + @Override + public void onResponse(IndicesStatsResponse indicesStatsResponse) { + listener.onResponse(buildResponse(clusterHealthResponse, nodesStatsResponse, indicesStatsResponse)); + } + @Override + public void onFailure(Throwable e) { + listener.onFailure(new ElasticsearchException("Indices stats request failed", e)); + } + }; + private ActionListener nodesStatsResponseActionListener = new ActionListener() { @Override public void onResponse(NodesStatsResponse nodeStats) { - listener.onResponse(buildResponse(clusterHealthResponse, nodeStats)); + nodesStatsResponse = nodeStats; + client.admin().indices().stats(new IndicesStatsRequest(), indicesStatsResponseActionListener); } @Override @@ -74,8 +91,9 @@ public void onFailure(Throwable throwable) { } }; - protected NodePrometheusMetricsResponse buildResponse(ClusterHealthResponse clusterHealth, NodesStatsResponse nodesStats) { - NodePrometheusMetricsResponse response = new NodePrometheusMetricsResponse(clusterHealth, nodesStats.getAt(0)); + protected NodePrometheusMetricsResponse buildResponse(ClusterHealthResponse clusterHealth, NodesStatsResponse nodesStats, @Nullable + IndicesStatsResponse indicesStats) { + NodePrometheusMetricsResponse response = new NodePrometheusMetricsResponse(clusterHealth, nodesStats.getAt(0), indicesStats); if (logger.isTraceEnabled()) { logger.trace("Return response: [{}]", response); } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/PackageAccessHelper.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/PackageAccessHelper.java new file mode 100644 index 00000000..866497d9 --- /dev/null +++ b/src/main/java/org/elasticsearch/action/admin/indices/stats/PackageAccessHelper.java @@ -0,0 +1,44 @@ +/* + * Copyright [2016] [Vincent VAN HOLLEBEKE] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.elasticsearch.action.admin.indices.stats; + +import org.elasticsearch.action.ShardOperationFailedException; +import org.elasticsearch.action.support.DefaultShardOperationFailedException; + +import java.util.List; + +/** + * Utility methods. + */ +public class PackageAccessHelper { + + /** + * Shortcut to IndicesStatsResponse constructor which has package access restriction. + * + * @param shards The shards stats. + * @param totalShards The total shards this request ran against. + * @param successfulShards The successful shards this request was executed on. + * @param failedShards The failed shards this request was executed on. + * @param shardFailures The list of shard failures exception. + * @return new instance of IndicesStatsResponse. + */ + public static IndicesStatsResponse createIndicesStatsResponse(ShardStats[] shards, int totalShards, + int successfulShards, int failedShards, + List shardFailures) { + return new IndicesStatsResponse(shards, totalShards, successfulShards, failedShards, shardFailures); + } +} diff --git a/src/main/java/org/elasticsearch/rest/action/prometheus/RestPrometheusMetricsAction.java b/src/main/java/org/elasticsearch/rest/action/prometheus/RestPrometheusMetricsAction.java index 6acdc64d..230e9f07 100644 --- a/src/main/java/org/elasticsearch/rest/action/prometheus/RestPrometheusMetricsAction.java +++ b/src/main/java/org/elasticsearch/rest/action/prometheus/RestPrometheusMetricsAction.java @@ -43,7 +43,7 @@ public RestResponse buildResponse(NodePrometheusMetricsResponse response) throws logger.trace("Prepare new Prometheus metric collector for: [{}], [{}], [{}]", clusterName, nodeId, nodeName); } PrometheusMetricsCollector collector = new PrometheusMetricsCollector(clusterName, nodeName, nodeId); - collector.updateMetrics(response.getClusterHealth(), response.getNodeStats()); + collector.updateMetrics(response.getClusterHealth(), response.getNodeStats(), response.getIndicesStats()); return new BytesRestResponse(RestStatus.OK, collector.getCatalog().toTextFormat()); } From 8850a326280150f19e18e0fbf808fa928f859043 Mon Sep 17 00:00:00 2001 From: Daniel Pettersen <25056311+dtpett@users.noreply.github.com> Date: Tue, 16 Jan 2024 10:18:33 +0100 Subject: [PATCH 4/4] Reformatting: reduce arrow code --- .../PrometheusMetricsCollector.java | 439 +++++++++--------- 1 file changed, 228 insertions(+), 211 deletions(-) diff --git a/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java b/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java index 0111cd94..9676e5b2 100644 --- a/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java +++ b/src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java @@ -119,45 +119,48 @@ private void registerJVMMetrics() { } private void updateJVMMetrics(JvmStats jvm) { - if (jvm != null) { - catalog.setCounter("jvm_uptime_seconds", jvm.getUptime().getSeconds(), node, nodeId); - - catalog.setGauge("jvm_mem_heap_max_bytes", jvm.getMem().getHeapMax().bytes(), node, nodeId); - catalog.setGauge("jvm_mem_heap_used_bytes", jvm.getMem().getHeapUsed().bytes(), node, nodeId); - catalog.setGauge("jvm_mem_heap_used_percent", jvm.getMem().getHeapUsedPercent(), node, nodeId); - catalog.setGauge("jvm_mem_nonheap_used_bytes", jvm.getMem().getNonHeapUsed().bytes(), node, nodeId); - catalog.setGauge("jvm_mem_heap_committed_bytes", jvm.getMem().getHeapCommitted().bytes(), node, nodeId); - catalog.setGauge("jvm_mem_nonheap_committed_bytes", jvm.getMem().getNonHeapCommitted().bytes(), node, nodeId); - - for (JvmStats.MemoryPool mp : jvm.getMem()) { - String name = mp.getName(); - catalog.setGauge("jvm_mem_pool_max_bytes", mp.getMax().bytes(), node, nodeId, name); - catalog.setGauge("jvm_mem_pool_peak_max_bytes", mp.getPeakMax().bytes(), node, nodeId, name); - catalog.setGauge("jvm_mem_pool_used_bytes", mp.getUsed().bytes(), node, nodeId, name); - catalog.setGauge("jvm_mem_pool_peak_used_bytes", mp.getPeakUsed().bytes(), node, nodeId, name); - } + if (jvm == null) { + return; + } + catalog.setCounter("jvm_uptime_seconds", jvm.getUptime().getSeconds(), node, nodeId); + + catalog.setGauge("jvm_mem_heap_max_bytes", jvm.getMem().getHeapMax().bytes(), node, nodeId); + catalog.setGauge("jvm_mem_heap_used_bytes", jvm.getMem().getHeapUsed().bytes(), node, nodeId); + catalog.setGauge("jvm_mem_heap_used_percent", jvm.getMem().getHeapUsedPercent(), node, nodeId); + catalog.setGauge("jvm_mem_nonheap_used_bytes", jvm.getMem().getNonHeapUsed().bytes(), node, nodeId); + catalog.setGauge("jvm_mem_heap_committed_bytes", jvm.getMem().getHeapCommitted().bytes(), node, nodeId); + catalog.setGauge("jvm_mem_nonheap_committed_bytes", jvm.getMem().getNonHeapCommitted().bytes(), node, nodeId); + + for (JvmStats.MemoryPool mp : jvm.getMem()) { + String name = mp.getName(); + catalog.setGauge("jvm_mem_pool_max_bytes", mp.getMax().bytes(), node, nodeId, name); + catalog.setGauge("jvm_mem_pool_peak_max_bytes", mp.getPeakMax().bytes(), node, nodeId, name); + catalog.setGauge("jvm_mem_pool_used_bytes", mp.getUsed().bytes(), node, nodeId, name); + catalog.setGauge("jvm_mem_pool_peak_used_bytes", mp.getPeakUsed().bytes(), node, nodeId, name); + } - catalog.setGauge("jvm_threads_number", jvm.getThreads().getCount(), node, nodeId); - catalog.setGauge("jvm_threads_peak_number", jvm.getThreads().getPeakCount(), node, nodeId); + catalog.setGauge("jvm_threads_number", jvm.getThreads().getCount(), node, nodeId); + catalog.setGauge("jvm_threads_peak_number", jvm.getThreads().getPeakCount(), node, nodeId); - for (JvmStats.GarbageCollector gc : jvm.getGc().getCollectors()) { - String name = gc.getName(); - catalog.setCounter("jvm_gc_collection_count", gc.getCollectionCount(), node, nodeId, name); - catalog.setCounter("jvm_gc_collection_time_seconds", gc.getCollectionTime().getSeconds(), node, nodeId, name); - } + for (JvmStats.GarbageCollector gc : jvm.getGc().getCollectors()) { + String name = gc.getName(); + catalog.setCounter("jvm_gc_collection_count", gc.getCollectionCount(), node, nodeId, name); + catalog.setCounter("jvm_gc_collection_time_seconds", gc.getCollectionTime().getSeconds(), node, nodeId, + name); + } - for (JvmStats.BufferPool bp : jvm.getBufferPools()) { - String name = bp.getName(); - catalog.setGauge("jvm_bufferpool_number", bp.getCount(), node, nodeId, name); - catalog.setGauge("jvm_bufferpool_total_capacity_bytes", bp.getTotalCapacity().bytes(), node, nodeId, name); - catalog.setGauge("jvm_bufferpool_used_bytes", bp.getUsed().bytes(), node, nodeId, name); - } - if (jvm.getClasses() != null) { - catalog.setGauge("jvm_classes_loaded_number", jvm.getClasses().getLoadedClassCount(), node, nodeId); - catalog.setGauge("jvm_classes_total_loaded_number", jvm.getClasses().getTotalLoadedClassCount(), node, nodeId); - catalog.setGauge("jvm_classes_unloaded_number", jvm.getClasses().getUnloadedClassCount(), node, nodeId); - } + for (JvmStats.BufferPool bp : jvm.getBufferPools()) { + String name = bp.getName(); + catalog.setGauge("jvm_bufferpool_number", bp.getCount(), node, nodeId, name); + catalog.setGauge("jvm_bufferpool_total_capacity_bytes", bp.getTotalCapacity().bytes(), node, nodeId, name); + catalog.setGauge("jvm_bufferpool_used_bytes", bp.getUsed().bytes(), node, nodeId, name); } + if (jvm.getClasses() == null) { + return; + } + catalog.setGauge("jvm_classes_loaded_number", jvm.getClasses().getLoadedClassCount(), node, nodeId); + catalog.setGauge("jvm_classes_total_loaded_number", jvm.getClasses().getTotalLoadedClassCount(), node, nodeId); + catalog.setGauge("jvm_classes_unloaded_number", jvm.getClasses().getUnloadedClassCount(), node, nodeId); } private void registerIndicesMetrics() { @@ -250,103 +253,104 @@ private void registerIndicesMetrics() { } private void updateIndicesMetrics(NodeIndicesStats idx) { - if (idx != null) { - catalog.setGauge("indices_doc_number", idx.getDocs().getCount(), node, nodeId); - catalog.setGauge("indices_doc_deleted_number", idx.getDocs().getDeleted(), node, nodeId); - - catalog.setGauge("indices_store_size_bytes", idx.getStore().getSizeInBytes(), node, nodeId); - catalog.setCounter("indices_store_throttle_time_seconds", idx.getStore().getThrottleTime().millis() / 1000.0, node, nodeId); - - catalog.setCounter("indices_indexing_delete_count", idx.getIndexing().getTotal().getDeleteCount(), node, nodeId); - catalog.setGauge("indices_indexing_delete_current_number", idx.getIndexing().getTotal().getDeleteCurrent(), node, nodeId); - catalog.setCounter("indices_indexing_delete_time_seconds", idx.getIndexing().getTotal().getDeleteTimeInMillis() / 1000.0, node, nodeId); - catalog.setCounter("indices_indexing_index_count", idx.getIndexing().getTotal().getIndexCount(), node, nodeId); - catalog.setGauge("indices_indexing_index_current_number", idx.getIndexing().getTotal().getIndexCurrent(), node, nodeId); - catalog.setCounter("indices_indexing_index_failed_count", idx.getIndexing().getTotal().getIndexFailedCount(), node, nodeId); - catalog.setCounter("indices_indexing_index_time_seconds", idx.getIndexing().getTotal().getIndexTimeInMillis() / 1000.0, node, nodeId); - catalog.setCounter("indices_indexing_noop_update_count", idx.getIndexing().getTotal().getNoopUpdateCount(), node, nodeId); - catalog.setGauge("indices_indexing_is_throttled_bool", idx.getIndexing().getTotal().isThrottled() ? 1 : 0, node, nodeId); - catalog.setCounter("indices_indexing_throttle_time_seconds", idx.getIndexing().getTotal().getThrottleTimeInMillis() / 1000.0, node, nodeId); - - catalog.setCounter("indices_get_count", idx.getGet().getCount(), node, nodeId); - catalog.setCounter("indices_get_time_seconds", idx.getGet().getTimeInMillis() / 1000.0, node, nodeId); - catalog.setCounter("indices_get_exists_count", idx.getGet().getExistsCount(), node, nodeId); - catalog.setCounter("indices_get_exists_time_seconds", idx.getGet().getExistsTimeInMillis() / 1000.0, node, nodeId); - catalog.setCounter("indices_get_missing_count", idx.getGet().getMissingCount(), node, nodeId); - catalog.setCounter("indices_get_missing_time_seconds", idx.getGet().getMissingTimeInMillis() / 1000.0, node, nodeId); - - catalog.setGauge("indices_search_open_contexts_number", idx.getSearch().getOpenContexts(), node, nodeId); - catalog.setCounter("indices_search_fetch_count", idx.getSearch().getTotal().getFetchCount(), node, nodeId); - catalog.setGauge("indices_search_fetch_current_number", idx.getSearch().getTotal().getFetchCurrent(), node, nodeId); - catalog.setCounter("indices_search_fetch_time_seconds", idx.getSearch().getTotal().getFetchTimeInMillis() / 1000.0, node, nodeId); - catalog.setCounter("indices_search_query_count", idx.getSearch().getTotal().getQueryCount(), node, nodeId); - catalog.setGauge("indices_search_query_current_number", idx.getSearch().getTotal().getQueryCurrent(), node, nodeId); - catalog.setCounter("indices_search_query_time_seconds", idx.getSearch().getTotal().getQueryTimeInMillis() / 1000.0, node, nodeId); - catalog.setCounter("indices_search_scroll_count", idx.getSearch().getTotal().getScrollCount(), node, nodeId); - catalog.setGauge("indices_search_scroll_current_number", idx.getSearch().getTotal().getScrollCurrent(), node, nodeId); - catalog.setCounter("indices_search_scroll_time_seconds", idx.getSearch().getTotal().getScrollTimeInMillis() / 1000.0, node, nodeId); - - catalog.setGauge("indices_merges_current_number", idx.getMerge().getCurrent(), node, nodeId); - catalog.setGauge("indices_merges_current_docs_number", idx.getMerge().getCurrentNumDocs(), node, nodeId); - catalog.setGauge("indices_merges_current_size_bytes", idx.getMerge().getCurrentSizeInBytes(), node, nodeId); - catalog.setCounter("indices_merges_total_number", idx.getMerge().getTotal(), node, nodeId); - catalog.setCounter("indices_merges_total_time_seconds", idx.getMerge().getTotalTimeInMillis() / 1000.0, node, nodeId); - catalog.setCounter("indices_merges_total_docs_count", idx.getMerge().getTotalNumDocs(), node, nodeId); - catalog.setCounter("indices_merges_total_size_bytes", idx.getMerge().getTotalSizeInBytes(), node, nodeId); - catalog.setCounter("indices_merges_total_stopped_time_seconds", idx.getMerge().getTotalStoppedTimeInMillis() / 1000.0, node, nodeId); - catalog.setCounter("indices_merges_total_throttled_time_seconds", idx.getMerge().getTotalThrottledTimeInMillis() / 1000.0, node, nodeId); - catalog.setGauge("indices_merges_total_auto_throttle_bytes", idx.getMerge().getTotalBytesPerSecAutoThrottle(), node, nodeId); - - catalog.setCounter("indices_refresh_total_count", idx.getRefresh().getTotal(), node, nodeId); - catalog.setCounter("indices_refresh_total_time_seconds", idx.getRefresh().getTotalTimeInMillis() / 1000.0, node, nodeId); - - catalog.setCounter("indices_flush_total_count", idx.getFlush().getTotal(), node, nodeId); - catalog.setCounter("indices_flush_total_time_seconds", idx.getFlush().getTotalTimeInMillis() / 1000.0, node, nodeId); - - catalog.setCounter("indices_querycache_cache_count", idx.getQueryCache().getCacheCount(), node, nodeId); - catalog.setGauge("indices_querycache_cache_size_bytes", idx.getQueryCache().getCacheSize(), node, nodeId); - catalog.setCounter("indices_querycache_evictions_count", idx.getQueryCache().getEvictions(), node, nodeId); - catalog.setCounter("indices_querycache_hit_count", idx.getQueryCache().getHitCount(), node, nodeId); - catalog.setGauge("indices_querycache_memory_size_bytes", idx.getQueryCache().getMemorySizeInBytes(), node, nodeId); - catalog.setGauge("indices_querycache_miss_number", idx.getQueryCache().getMissCount(), node, nodeId); - catalog.setGauge("indices_querycache_total_number", idx.getQueryCache().getTotalCount(), node, nodeId); - - catalog.setGauge("indices_fielddata_memory_size_bytes", idx.getFieldData().getMemorySizeInBytes(), node, nodeId); - catalog.setCounter("indices_fielddata_evictions_count", idx.getFieldData().getEvictions(), node, nodeId); - - catalog.setCounter("indices_percolate_count", idx.getPercolate().getCount(), node, nodeId); - catalog.setGauge("indices_percolate_current_number", idx.getPercolate().getCurrent(), node, nodeId); - catalog.setGauge("indices_percolate_memory_size_bytes", idx.getPercolate().getMemorySizeInBytes(), node, nodeId); - catalog.setCounter("indices_percolate_queries_count", idx.getPercolate().getNumQueries(), node, nodeId); - catalog.setCounter("indices_percolate_time_seconds", idx.getPercolate().getTimeInMillis() / 1000.0, node, nodeId); - - catalog.setGauge("indices_completion_size_bytes", idx.getCompletion().getSizeInBytes(), node, nodeId); - - catalog.setGauge("indices_segments_number", idx.getSegments().getCount(), node, nodeId); - catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getMemoryInBytes(), node, nodeId, "all"); - catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getBitsetMemoryInBytes(), node, nodeId, "bitset"); - catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getDocValuesMemoryInBytes(), node, nodeId, "docvalues"); - catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getIndexWriterMaxMemoryInBytes(), node, nodeId, "indexwriter_max"); - catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getIndexWriterMemoryInBytes(), node, nodeId, "indexwriter"); - catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getNormsMemoryInBytes(), node, nodeId, "norms"); - catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getStoredFieldsMemoryInBytes(), node, nodeId, "storefields"); - catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getTermsMemoryInBytes(), node, nodeId, "terms"); - catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getTermVectorsMemoryInBytes(), node, nodeId, "termvectors"); - catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getVersionMapMemoryInBytes(), node, nodeId, "versionmap"); - - catalog.setGauge("indices_suggest_current_number", idx.getSuggest().getCurrent(), node, nodeId); - catalog.setCounter("indices_suggest_count", idx.getSuggest().getCount(), node, nodeId); - catalog.setCounter("indices_suggest_time_seconds", idx.getSuggest().getTimeInMillis() / 1000.0, node, nodeId); - - catalog.setGauge("indices_requestcache_memory_size_bytes", idx.getRequestCache().getMemorySizeInBytes(), node, nodeId); - catalog.setCounter("indices_requestcache_hit_count", idx.getRequestCache().getHitCount(), node, nodeId); - catalog.setCounter("indices_requestcache_miss_count", idx.getRequestCache().getMissCount(), node, nodeId); - catalog.setCounter("indices_requestcache_evictions_count", idx.getRequestCache().getEvictions(), node, nodeId); - - catalog.setGauge("indices_recovery_current_number", idx.getRecoveryStats().currentAsSource(), node, nodeId, "source"); - catalog.setGauge("indices_recovery_current_number", idx.getRecoveryStats().currentAsTarget(), node, nodeId, "target"); - catalog.setCounter("indices_recovery_throttle_time_seconds", idx.getRecoveryStats().throttleTime().getSeconds(), node, nodeId); + if (idx == null) { + return; } + catalog.setGauge("indices_doc_number", idx.getDocs().getCount(), node, nodeId); + catalog.setGauge("indices_doc_deleted_number", idx.getDocs().getDeleted(), node, nodeId); + + catalog.setGauge("indices_store_size_bytes", idx.getStore().getSizeInBytes(), node, nodeId); + catalog.setCounter("indices_store_throttle_time_seconds", idx.getStore().getThrottleTime().millis() / 1000.0, node, nodeId); + + catalog.setCounter("indices_indexing_delete_count", idx.getIndexing().getTotal().getDeleteCount(), node, nodeId); + catalog.setGauge("indices_indexing_delete_current_number", idx.getIndexing().getTotal().getDeleteCurrent(), node, nodeId); + catalog.setCounter("indices_indexing_delete_time_seconds", idx.getIndexing().getTotal().getDeleteTimeInMillis() / 1000.0, node, nodeId); + catalog.setCounter("indices_indexing_index_count", idx.getIndexing().getTotal().getIndexCount(), node, nodeId); + catalog.setGauge("indices_indexing_index_current_number", idx.getIndexing().getTotal().getIndexCurrent(), node, nodeId); + catalog.setCounter("indices_indexing_index_failed_count", idx.getIndexing().getTotal().getIndexFailedCount(), node, nodeId); + catalog.setCounter("indices_indexing_index_time_seconds", idx.getIndexing().getTotal().getIndexTimeInMillis() / 1000.0, node, nodeId); + catalog.setCounter("indices_indexing_noop_update_count", idx.getIndexing().getTotal().getNoopUpdateCount(), node, nodeId); + catalog.setGauge("indices_indexing_is_throttled_bool", idx.getIndexing().getTotal().isThrottled() ? 1 : 0, node, nodeId); + catalog.setCounter("indices_indexing_throttle_time_seconds", idx.getIndexing().getTotal().getThrottleTimeInMillis() / 1000.0, node, nodeId); + + catalog.setCounter("indices_get_count", idx.getGet().getCount(), node, nodeId); + catalog.setCounter("indices_get_time_seconds", idx.getGet().getTimeInMillis() / 1000.0, node, nodeId); + catalog.setCounter("indices_get_exists_count", idx.getGet().getExistsCount(), node, nodeId); + catalog.setCounter("indices_get_exists_time_seconds", idx.getGet().getExistsTimeInMillis() / 1000.0, node, nodeId); + catalog.setCounter("indices_get_missing_count", idx.getGet().getMissingCount(), node, nodeId); + catalog.setCounter("indices_get_missing_time_seconds", idx.getGet().getMissingTimeInMillis() / 1000.0, node, nodeId); + + catalog.setGauge("indices_search_open_contexts_number", idx.getSearch().getOpenContexts(), node, nodeId); + catalog.setCounter("indices_search_fetch_count", idx.getSearch().getTotal().getFetchCount(), node, nodeId); + catalog.setGauge("indices_search_fetch_current_number", idx.getSearch().getTotal().getFetchCurrent(), node, nodeId); + catalog.setCounter("indices_search_fetch_time_seconds", idx.getSearch().getTotal().getFetchTimeInMillis() / 1000.0, node, nodeId); + catalog.setCounter("indices_search_query_count", idx.getSearch().getTotal().getQueryCount(), node, nodeId); + catalog.setGauge("indices_search_query_current_number", idx.getSearch().getTotal().getQueryCurrent(), node, nodeId); + catalog.setCounter("indices_search_query_time_seconds", idx.getSearch().getTotal().getQueryTimeInMillis() / 1000.0, node, nodeId); + catalog.setCounter("indices_search_scroll_count", idx.getSearch().getTotal().getScrollCount(), node, nodeId); + catalog.setGauge("indices_search_scroll_current_number", idx.getSearch().getTotal().getScrollCurrent(), node, nodeId); + catalog.setCounter("indices_search_scroll_time_seconds", idx.getSearch().getTotal().getScrollTimeInMillis() / 1000.0, node, nodeId); + + catalog.setGauge("indices_merges_current_number", idx.getMerge().getCurrent(), node, nodeId); + catalog.setGauge("indices_merges_current_docs_number", idx.getMerge().getCurrentNumDocs(), node, nodeId); + catalog.setGauge("indices_merges_current_size_bytes", idx.getMerge().getCurrentSizeInBytes(), node, nodeId); + catalog.setCounter("indices_merges_total_number", idx.getMerge().getTotal(), node, nodeId); + catalog.setCounter("indices_merges_total_time_seconds", idx.getMerge().getTotalTimeInMillis() / 1000.0, node, nodeId); + catalog.setCounter("indices_merges_total_docs_count", idx.getMerge().getTotalNumDocs(), node, nodeId); + catalog.setCounter("indices_merges_total_size_bytes", idx.getMerge().getTotalSizeInBytes(), node, nodeId); + catalog.setCounter("indices_merges_total_stopped_time_seconds", idx.getMerge().getTotalStoppedTimeInMillis() / 1000.0, node, nodeId); + catalog.setCounter("indices_merges_total_throttled_time_seconds", idx.getMerge().getTotalThrottledTimeInMillis() / 1000.0, node, nodeId); + catalog.setGauge("indices_merges_total_auto_throttle_bytes", idx.getMerge().getTotalBytesPerSecAutoThrottle(), node, nodeId); + + catalog.setCounter("indices_refresh_total_count", idx.getRefresh().getTotal(), node, nodeId); + catalog.setCounter("indices_refresh_total_time_seconds", idx.getRefresh().getTotalTimeInMillis() / 1000.0, node, nodeId); + + catalog.setCounter("indices_flush_total_count", idx.getFlush().getTotal(), node, nodeId); + catalog.setCounter("indices_flush_total_time_seconds", idx.getFlush().getTotalTimeInMillis() / 1000.0, node, nodeId); + + catalog.setCounter("indices_querycache_cache_count", idx.getQueryCache().getCacheCount(), node, nodeId); + catalog.setGauge("indices_querycache_cache_size_bytes", idx.getQueryCache().getCacheSize(), node, nodeId); + catalog.setCounter("indices_querycache_evictions_count", idx.getQueryCache().getEvictions(), node, nodeId); + catalog.setCounter("indices_querycache_hit_count", idx.getQueryCache().getHitCount(), node, nodeId); + catalog.setGauge("indices_querycache_memory_size_bytes", idx.getQueryCache().getMemorySizeInBytes(), node, nodeId); + catalog.setGauge("indices_querycache_miss_number", idx.getQueryCache().getMissCount(), node, nodeId); + catalog.setGauge("indices_querycache_total_number", idx.getQueryCache().getTotalCount(), node, nodeId); + + catalog.setGauge("indices_fielddata_memory_size_bytes", idx.getFieldData().getMemorySizeInBytes(), node, nodeId); + catalog.setCounter("indices_fielddata_evictions_count", idx.getFieldData().getEvictions(), node, nodeId); + + catalog.setCounter("indices_percolate_count", idx.getPercolate().getCount(), node, nodeId); + catalog.setGauge("indices_percolate_current_number", idx.getPercolate().getCurrent(), node, nodeId); + catalog.setGauge("indices_percolate_memory_size_bytes", idx.getPercolate().getMemorySizeInBytes(), node, nodeId); + catalog.setCounter("indices_percolate_queries_count", idx.getPercolate().getNumQueries(), node, nodeId); + catalog.setCounter("indices_percolate_time_seconds", idx.getPercolate().getTimeInMillis() / 1000.0, node, nodeId); + + catalog.setGauge("indices_completion_size_bytes", idx.getCompletion().getSizeInBytes(), node, nodeId); + + catalog.setGauge("indices_segments_number", idx.getSegments().getCount(), node, nodeId); + catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getMemoryInBytes(), node, nodeId, "all"); + catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getBitsetMemoryInBytes(), node, nodeId, "bitset"); + catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getDocValuesMemoryInBytes(), node, nodeId, "docvalues"); + catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getIndexWriterMaxMemoryInBytes(), node, nodeId, "indexwriter_max"); + catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getIndexWriterMemoryInBytes(), node, nodeId, "indexwriter"); + catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getNormsMemoryInBytes(), node, nodeId, "norms"); + catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getStoredFieldsMemoryInBytes(), node, nodeId, "storefields"); + catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getTermsMemoryInBytes(), node, nodeId, "terms"); + catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getTermVectorsMemoryInBytes(), node, nodeId, "termvectors"); + catalog.setGauge("indices_segments_memory_bytes", idx.getSegments().getVersionMapMemoryInBytes(), node, nodeId, "versionmap"); + + catalog.setGauge("indices_suggest_current_number", idx.getSuggest().getCurrent(), node, nodeId); + catalog.setCounter("indices_suggest_count", idx.getSuggest().getCount(), node, nodeId); + catalog.setCounter("indices_suggest_time_seconds", idx.getSuggest().getTimeInMillis() / 1000.0, node, nodeId); + + catalog.setGauge("indices_requestcache_memory_size_bytes", idx.getRequestCache().getMemorySizeInBytes(), node, nodeId); + catalog.setCounter("indices_requestcache_hit_count", idx.getRequestCache().getHitCount(), node, nodeId); + catalog.setCounter("indices_requestcache_miss_count", idx.getRequestCache().getMissCount(), node, nodeId); + catalog.setCounter("indices_requestcache_evictions_count", idx.getRequestCache().getEvictions(), node, nodeId); + + catalog.setGauge("indices_recovery_current_number", idx.getRecoveryStats().currentAsSource(), node, nodeId, "source"); + catalog.setGauge("indices_recovery_current_number", idx.getRecoveryStats().currentAsTarget(), node, nodeId, "target"); + catalog.setCounter("indices_recovery_throttle_time_seconds", idx.getRecoveryStats().throttleTime().getSeconds(), node, nodeId); } private void registerPerIndexMetrics() { @@ -442,23 +446,23 @@ private void registerPerIndexMetrics() { } private void updatePerIndexMetrics(ClusterHealthResponse chr, IndicesStatsResponse isr) { - - if (chr != null && isr != null) { - for (Map.Entry entry : isr.getIndices().entrySet()) { - String indexName = entry.getKey(); - ClusterIndexHealth cih = chr.getIndices().get(indexName); - catalog.setGauge("index_status", cih.getStatus().value(), indexName); - catalog.setGauge("index_replicas_number", cih.getNumberOfReplicas(), indexName); - catalog.setGauge("index_shards_number", cih.getActiveShards(), "active", indexName); - catalog.setGauge("index_shards_number", cih.getNumberOfShards(), "shards", indexName); - catalog.setGauge("index_shards_number", cih.getActivePrimaryShards(), "active_primary", indexName); - catalog.setGauge("index_shards_number", cih.getInitializingShards(), "initializing", indexName); - catalog.setGauge("index_shards_number", cih.getRelocatingShards(), "relocating", indexName); - catalog.setGauge("index_shards_number", cih.getUnassignedShards(), "unassigned", indexName); - IndexStats indexStats = entry.getValue(); - updatePerIndexContextMetrics(indexName, "total", indexStats.getTotal()); - updatePerIndexContextMetrics(indexName, "primaries", indexStats.getPrimaries()); - } + if (chr == null || isr == null) { + return; + } + for (Map.Entry entry : isr.getIndices().entrySet()) { + String indexName = entry.getKey(); + ClusterIndexHealth cih = chr.getIndices().get(indexName); + catalog.setGauge("index_status", cih.getStatus().value(), indexName); + catalog.setGauge("index_replicas_number", cih.getNumberOfReplicas(), indexName); + catalog.setGauge("index_shards_number", cih.getActiveShards(), "active", indexName); + catalog.setGauge("index_shards_number", cih.getNumberOfShards(), "shards", indexName); + catalog.setGauge("index_shards_number", cih.getActivePrimaryShards(), "active_primary", indexName); + catalog.setGauge("index_shards_number", cih.getInitializingShards(), "initializing", indexName); + catalog.setGauge("index_shards_number", cih.getRelocatingShards(), "relocating", indexName); + catalog.setGauge("index_shards_number", cih.getUnassignedShards(), "unassigned", indexName); + IndexStats indexStats = entry.getValue(); + updatePerIndexContextMetrics(indexName, "total", indexStats.getTotal()); + updatePerIndexContextMetrics(indexName, "primaries", indexStats.getPrimaries()); } } @@ -570,13 +574,14 @@ private void registerTransportMetrics() { } private void updateTransportMetrics(TransportStats ts) { - if (ts != null) { - catalog.setGauge("transport_server_open_number", ts.getServerOpen(), node, nodeId); - catalog.setCounter("transport_rx_packets_count", ts.getRxCount(), node, nodeId); - catalog.setCounter("transport_tx_packets_count", ts.getTxCount(), node, nodeId); - catalog.setCounter("transport_rx_bytes_count", ts.getRxSize().bytes(), node, nodeId); - catalog.setCounter("transport_tx_bytes_count", ts.getTxSize().bytes(), node, nodeId); + if (ts == null) { + return; } + catalog.setGauge("transport_server_open_number", ts.getServerOpen(), node, nodeId); + catalog.setCounter("transport_rx_packets_count", ts.getRxCount(), node, nodeId); + catalog.setCounter("transport_tx_packets_count", ts.getTxCount(), node, nodeId); + catalog.setCounter("transport_rx_bytes_count", ts.getRxSize().bytes(), node, nodeId); + catalog.setCounter("transport_tx_bytes_count", ts.getTxSize().bytes(), node, nodeId); } private void registerHTTPMetrics() { @@ -585,10 +590,11 @@ private void registerHTTPMetrics() { } private void updateHTTPMetrics(HttpStats http) { - if (http != null) { - catalog.setGauge("http_open_server_number", http.getServerOpen(), node, nodeId); - catalog.setCounter("http_open_total_count", http.getTotalOpen(), node, nodeId); + if (http == null) { + return; } + catalog.setGauge("http_open_server_number", http.getServerOpen(), node, nodeId); + catalog.setCounter("http_open_total_count", http.getTotalOpen(), node, nodeId); } private void registerScriptMetrics() { @@ -597,10 +603,11 @@ private void registerScriptMetrics() { } private void updateScriptMetrics(ScriptStats sc) { - if (sc != null) { - catalog.setCounter("script_cache_evictions_count", sc.getCacheEvictions(), node, nodeId); - catalog.setCounter("script_compilations_count", sc.getCompilations(), node, nodeId); + if (sc == null) { + return; } + catalog.setCounter("script_cache_evictions_count", sc.getCacheEvictions(), node, nodeId); + catalog.setCounter("script_compilations_count", sc.getCompilations(), node, nodeId); } private void registerProcessMetrics() { @@ -612,13 +619,14 @@ private void registerProcessMetrics() { } private void updateProcessMetrics(ProcessStats ps) { - if (ps != null) { - catalog.setGauge("process_cpu_percent", ps.getCpu().getPercent(), node, nodeId); - catalog.setGauge("process_cpu_time_seconds", ps.getCpu().getTotal().getSeconds(), node, nodeId); - catalog.setGauge("process_mem_total_virtual_bytes", ps.getMem().getTotalVirtual().bytes(), node, nodeId); - catalog.setGauge("process_file_descriptors_open_number", ps.getOpenFileDescriptors(), node, nodeId); - catalog.setGauge("process_file_descriptors_max_number", ps.getMaxFileDescriptors(), node, nodeId); + if (ps == null) { + return; } + catalog.setGauge("process_cpu_percent", ps.getCpu().getPercent(), node, nodeId); + catalog.setGauge("process_cpu_time_seconds", ps.getCpu().getTotal().getSeconds(), node, nodeId); + catalog.setGauge("process_mem_total_virtual_bytes", ps.getMem().getTotalVirtual().bytes(), node, nodeId); + catalog.setGauge("process_file_descriptors_open_number", ps.getOpenFileDescriptors(), node, nodeId); + catalog.setGauge("process_file_descriptors_max_number", ps.getMaxFileDescriptors(), node, nodeId); } private void registerOsMetrics() { @@ -635,18 +643,19 @@ private void registerOsMetrics() { } private void updateOsMetrics(OsStats os) { - if (os != null) { - catalog.setGauge("os_cpu_percent", os.getCpuPercent(), node, nodeId); - catalog.setGauge("os_load_average", os.getLoadAverage(), node, nodeId); - catalog.setGauge("os_mem_free_bytes", os.getMem().getFree().bytes(), node, nodeId); - catalog.setGauge("os_mem_free_percent", os.getMem().getFreePercent(), node, nodeId); - catalog.setGauge("os_mem_used_bytes", os.getMem().getUsed().bytes(), node, nodeId); - catalog.setGauge("os_mem_used_percent", os.getMem().getUsedPercent(), node, nodeId); - catalog.setGauge("os_mem_total_bytes", os.getMem().getTotal().bytes(), node, nodeId); - catalog.setGauge("os_swap_free_bytes", os.getSwap().getFree().bytes(), node, nodeId); - catalog.setGauge("os_swap_used_bytes", os.getSwap().getUsed().bytes(), node, nodeId); - catalog.setGauge("os_swap_total_bytes", os.getSwap().getTotal().bytes(), node, nodeId); + if (os == null) { + return; } + catalog.setGauge("os_cpu_percent", os.getCpuPercent(), node, nodeId); + catalog.setGauge("os_load_average", os.getLoadAverage(), node, nodeId); + catalog.setGauge("os_mem_free_bytes", os.getMem().getFree().bytes(), node, nodeId); + catalog.setGauge("os_mem_free_percent", os.getMem().getFreePercent(), node, nodeId); + catalog.setGauge("os_mem_used_bytes", os.getMem().getUsed().bytes(), node, nodeId); + catalog.setGauge("os_mem_used_percent", os.getMem().getUsedPercent(), node, nodeId); + catalog.setGauge("os_mem_total_bytes", os.getMem().getTotal().bytes(), node, nodeId); + catalog.setGauge("os_swap_free_bytes", os.getSwap().getFree().bytes(), node, nodeId); + catalog.setGauge("os_swap_used_bytes", os.getSwap().getUsed().bytes(), node, nodeId); + catalog.setGauge("os_swap_total_bytes", os.getSwap().getTotal().bytes(), node, nodeId); } private void registerCircuitBreakerMetrics() { @@ -657,14 +666,15 @@ private void registerCircuitBreakerMetrics() { } private void updateCircuitBreakersMetrics(AllCircuitBreakerStats acbs) { - if (acbs != null) { - for (CircuitBreakerStats cbs : acbs.getAllStats()) { - String name = cbs.getName(); - catalog.setGauge("circuitbreaker_estimated_bytes", cbs.getEstimated(), node, nodeId, name); - catalog.setGauge("circuitbreaker_limit_bytes", cbs.getLimit(), node, nodeId, name); - catalog.setGauge("circuitbreaker_overhead_ratio", cbs.getOverhead(), node, nodeId, name); - catalog.setCounter("circuitbreaker_tripped_count", cbs.getTrippedCount(), node, nodeId, name); - } + if (acbs == null) { + return; + } + for (CircuitBreakerStats cbs : acbs.getAllStats()) { + String name = cbs.getName(); + catalog.setGauge("circuitbreaker_estimated_bytes", cbs.getEstimated(), node, nodeId, name); + catalog.setGauge("circuitbreaker_limit_bytes", cbs.getLimit(), node, nodeId, name); + catalog.setGauge("circuitbreaker_overhead_ratio", cbs.getOverhead(), node, nodeId, name); + catalog.setCounter("circuitbreaker_tripped_count", cbs.getTrippedCount(), node, nodeId, name); } } @@ -675,16 +685,17 @@ private void registerThreadPoolMetrics() { } private void updateThreadPoolMetrics(ThreadPoolStats tps) { - if (tps != null) { - for (ThreadPoolStats.Stats st : tps) { - String name = st.getName(); - catalog.setGauge("threadpool_threads_number", st.getThreads(), node, nodeId, name, "threads"); - catalog.setGauge("threadpool_threads_number", st.getActive(), node, nodeId, name, "active"); - catalog.setGauge("threadpool_threads_number", st.getLargest(), node, nodeId, name, "largest"); - catalog.setCounter("threadpool_threads_count", st.getCompleted(), node, nodeId, name, "completed"); - catalog.setCounter("threadpool_threads_count", st.getRejected(), node, nodeId, name, "rejected"); - catalog.setGauge("threadpool_tasks_number", st.getQueue(), node, nodeId, name, "queue"); - } + if (tps == null) { + return; + } + for (ThreadPoolStats.Stats st : tps) { + String name = st.getName(); + catalog.setGauge("threadpool_threads_number", st.getThreads(), node, nodeId, name, "threads"); + catalog.setGauge("threadpool_threads_number", st.getActive(), node, nodeId, name, "active"); + catalog.setGauge("threadpool_threads_number", st.getLargest(), node, nodeId, name, "largest"); + catalog.setCounter("threadpool_threads_count", st.getCompleted(), node, nodeId, name, "completed"); + catalog.setCounter("threadpool_threads_count", st.getRejected(), node, nodeId, name, "rejected"); + catalog.setGauge("threadpool_tasks_number", st.getQueue(), node, nodeId, name, "queue"); } } @@ -701,23 +712,29 @@ private void registerFsMetrics() { } private void updateFsMetrics(FsInfo fs) { - if (fs != null) { - catalog.setGauge("fs_total_total_bytes", fs.getTotal().getTotal().bytes(), node, nodeId); - catalog.setGauge("fs_total_available_bytes", fs.getTotal().getAvailable().bytes(), node, nodeId); - catalog.setGauge("fs_total_free_bytes", fs.getTotal().getFree().bytes(), node, nodeId); - if (fs.getTotal() != null && fs.getTotal().getSpins() != null) - catalog.setGauge("fs_total_is_spinning_bool", fs.getTotal().getSpins() ? 1 : 0, node, nodeId); - - for (FsInfo.Path fspath : fs) { - String path = fspath.getPath(); - String mount = fspath.getMount(); - String type = fspath.getType(); - catalog.setGauge("fs_path_total_bytes", fspath.getTotal().bytes(), node, nodeId, path, mount, type); - catalog.setGauge("fs_path_available_bytes", fspath.getAvailable().bytes(), node, nodeId, path, mount, type); - catalog.setGauge("fs_path_free_bytes", fspath.getFree().bytes(), node, nodeId, path, mount, type); - if (fspath.getSpins() != null) - catalog.setGauge("fs_path_is_spinning_bool", fspath.getSpins() ? 1 : 0, node, nodeId, path, mount, type); + if (fs == null) { + return; + } + catalog.setGauge("fs_total_total_bytes", fs.getTotal().getTotal().bytes(), node, nodeId); + catalog.setGauge("fs_total_available_bytes", fs.getTotal().getAvailable().bytes(), node, nodeId); + catalog.setGauge("fs_total_free_bytes", fs.getTotal().getFree().bytes(), node, nodeId); + if (fs.getTotal() == null || fs.getTotal().getSpins() == null) { + return; + } + catalog.setGauge("fs_total_is_spinning_bool", fs.getTotal().getSpins() ? 1 : 0, node, nodeId); + + for (FsInfo.Path fspath : fs) { + String path = fspath.getPath(); + String mount = fspath.getMount(); + String type = fspath.getType(); + catalog.setGauge("fs_path_total_bytes", fspath.getTotal().bytes(), node, nodeId, path, mount, type); + catalog.setGauge("fs_path_available_bytes", fspath.getAvailable().bytes(), node, nodeId, path, mount, type); + catalog.setGauge("fs_path_free_bytes", fspath.getFree().bytes(), node, nodeId, path, mount, type); + if (fspath.getSpins() == null) { + return; } + catalog.setGauge("fs_path_is_spinning_bool", fspath.getSpins() ? 1 : 0, node, nodeId, path, mount, + type); } }