[improve](metrics)Display garbage collector type (apache#27408)

seawinde · Nov 28, 2023 · 6e28a99 · 6e28a99
1 parent 7733dea
commit 6e28a99
Show file tree

Hide file tree

Showing 9 changed files with 365 additions and 338 deletions.
diff --git a/docs/en/docs/admin-manual/maint-monitor/monitor-alert.md b/docs/en/docs/admin-manual/maint-monitor/monitor-alert.md
@@ -84,14 +84,10 @@ jvm_young_size_bytes{type="max"} 907345920
 jvm_old_size_bytes{type="used"} 114633448
 jvm_old_size_bytes{type="peak_used"} 114633448
 jvm_old_size_bytes{type="max"} 7455834112
-# HELP  jvm_young_gc jvm young gc stat
-# TYPE  jvm_young_gc gauge
-jvm_young_gc{type="count"} 247
-jvm_young_gc{type="time"} 860
-# HELP  jvm_old_gc jvm old gc stat
-# TYPE  jvm_old_gc gauge
-jvm_old_gc{type="count"} 3
-jvm_old_gc{type="time"} 211
+# HELP  jvm_gc jvm gc stat
+# TYPE  jvm_gc gauge
+<GarbageCollector>{type="count"} 247
+<GarbageCollector>{type="time"} 860
 # HELP  jvm_thread jvm thread stat
 # TYPE  jvm_thread gauge
 jvm_thread{type="count"} 162

diff --git a/docs/zh-CN/docs/admin-manual/maint-monitor/monitor-alert.md b/docs/zh-CN/docs/admin-manual/maint-monitor/monitor-alert.md
@@ -84,14 +84,10 @@ jvm_young_size_bytes{type="max"} 907345920
 jvm_old_size_bytes{type="used"} 114633448
 jvm_old_size_bytes{type="peak_used"} 114633448
 jvm_old_size_bytes{type="max"} 7455834112
-# HELP  jvm_young_gc jvm young gc stat
-# TYPE  jvm_young_gc gauge
-jvm_young_gc{type="count"} 247
-jvm_young_gc{type="time"} 860
-# HELP  jvm_old_gc jvm old gc stat
-# TYPE  jvm_old_gc gauge
-jvm_old_gc{type="count"} 3
-jvm_old_gc{type="time"} 211
+# HELP  jvm_gc jvm gc stat
+# TYPE  jvm_gc gauge
+<GarbageCollector>{type="count"} 247
+<GarbageCollector>{type="time"} 860
 # HELP  jvm_thread jvm thread stat
 # TYPE  jvm_thread gauge
 jvm_thread{type="count"} 162

diff --git a/docs/zh-CN/docs/admin-manual/maint-monitor/monitor-metrics/metrics.md b/docs/zh-CN/docs/admin-manual/maint-monitor/monitor-metrics/metrics.md
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/JsonMetricVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/metric/JsonMetricVisitor.java
@@ -40,10 +40,10 @@ public class JsonMetricVisitor extends MetricVisitor {
     private static final String JVM_NON_HEAP_SIZE_BYTES = "jvm_non_heap_size_bytes";
     private static final String JVM_YOUNG_SIZE_BYTES = "jvm_young_size_bytes";
     private static final String JVM_OLD_SIZE_BYTES = "jvm_old_size_bytes";
-    private static final String JVM_YOUNG_GC = "jvm_young_gc";
-    private static final String JVM_OLD_GC = "jvm_old_gc";
     private static final String JVM_THREAD = "jvm_thread";
 
+    private static final String JVM_GC = "jvm_gc";
+
     public JsonMetricVisitor() {
         super();
         sb.append("[\n");
@@ -52,59 +52,59 @@ public JsonMetricVisitor() {
     @Override
     public void visitJvm(JvmStats jvmStats) {
         // heap
-        setJvmJsonMetric(sb, JVM_HEAP_SIZE_BYTES, "max", "bytes", jvmStats.getMem().getHeapMax().getBytes());
-        setJvmJsonMetric(sb, JVM_HEAP_SIZE_BYTES, "committed", "bytes",
+        setJvmJsonMetric(sb, JVM_HEAP_SIZE_BYTES, null, "max", "bytes", jvmStats.getMem().getHeapMax().getBytes());
+        setJvmJsonMetric(sb, JVM_HEAP_SIZE_BYTES, null, "committed", "bytes",
                 jvmStats.getMem().getHeapCommitted().getBytes());
-        setJvmJsonMetric(sb, JVM_HEAP_SIZE_BYTES, "used", "bytes", jvmStats.getMem().getHeapUsed().getBytes());
+        setJvmJsonMetric(sb, JVM_HEAP_SIZE_BYTES, null, "used", "bytes", jvmStats.getMem().getHeapUsed().getBytes());
 
         // non heap
-        setJvmJsonMetric(sb, JVM_NON_HEAP_SIZE_BYTES, "committed", "bytes",
+        setJvmJsonMetric(sb, JVM_NON_HEAP_SIZE_BYTES, null, "committed", "bytes",
                 jvmStats.getMem().getNonHeapCommitted().getBytes());
-        setJvmJsonMetric(sb, JVM_NON_HEAP_SIZE_BYTES, "used", "bytes", jvmStats.getMem().getNonHeapUsed().getBytes());
+        setJvmJsonMetric(sb, JVM_NON_HEAP_SIZE_BYTES, null, "used", "bytes",
+                jvmStats.getMem().getNonHeapUsed().getBytes());
 
         // mem pool
         Iterator<MemoryPool> memIter = jvmStats.getMem().iterator();
         while (memIter.hasNext()) {
             MemoryPool memPool = memIter.next();
             if (memPool.getName().equalsIgnoreCase("young")) {
-                setJvmJsonMetric(sb, JVM_YOUNG_SIZE_BYTES, "used", "bytes", memPool.getUsed().getBytes());
-                setJvmJsonMetric(sb, JVM_YOUNG_SIZE_BYTES, "peak_used", "bytes", memPool.getPeakUsed().getBytes());
-                setJvmJsonMetric(sb, JVM_YOUNG_SIZE_BYTES, "max", "bytes", memPool.getMax().getBytes());
+                setJvmJsonMetric(sb, JVM_YOUNG_SIZE_BYTES, null, "used", "bytes", memPool.getUsed().getBytes());
+                setJvmJsonMetric(sb, JVM_YOUNG_SIZE_BYTES, null, "peak_used", "bytes",
+                        memPool.getPeakUsed().getBytes());
+                setJvmJsonMetric(sb, JVM_YOUNG_SIZE_BYTES, null, "max", "bytes", memPool.getMax().getBytes());
             } else if (memPool.getName().equalsIgnoreCase("old")) {
-                setJvmJsonMetric(sb, JVM_OLD_SIZE_BYTES, "used", "bytes", memPool.getUsed().getBytes());
-                setJvmJsonMetric(sb, JVM_OLD_SIZE_BYTES, "peak_used", "bytes", memPool.getPeakUsed().getBytes());
-                setJvmJsonMetric(sb, JVM_OLD_SIZE_BYTES, "max", "bytes", memPool.getMax().getBytes());
+                setJvmJsonMetric(sb, JVM_OLD_SIZE_BYTES, null, "used", "bytes", memPool.getUsed().getBytes());
+                setJvmJsonMetric(sb, JVM_OLD_SIZE_BYTES, null, "peak_used", "bytes", memPool.getPeakUsed().getBytes());
+                setJvmJsonMetric(sb, JVM_OLD_SIZE_BYTES, null, "max", "bytes", memPool.getMax().getBytes());
             }
         }
 
         // gc
-        Iterator<GarbageCollector> gcIter = jvmStats.getGc().iterator();
-        while (gcIter.hasNext()) {
-            GarbageCollector gc = gcIter.next();
-            if (gc.getName().equalsIgnoreCase("young")) {
-                setJvmJsonMetric(sb, JVM_YOUNG_GC, "count", "nounit", gc.getCollectionCount());
-                setJvmJsonMetric(sb, JVM_YOUNG_GC, "time", "milliseconds", gc.getCollectionTime().getMillis());
-            } else if (gc.getName().equalsIgnoreCase("old")) {
-                setJvmJsonMetric(sb, JVM_OLD_GC, "count", "nounit", gc.getCollectionCount());
-                setJvmJsonMetric(sb, JVM_OLD_GC, "time", "milliseconds", gc.getCollectionTime().getMillis());
-            }
+        for (GarbageCollector gc : jvmStats.getGc()) {
+            setJvmJsonMetric(sb, JVM_GC, gc.getName() + " Count", "count", "nounit", gc.getCollectionCount());
+            setJvmJsonMetric(sb, JVM_GC, gc.getName() + " Time", "time", "milliseconds",
+                    gc.getCollectionTime().getMillis());
         }
 
         // threads
         Threads threads = jvmStats.getThreads();
-        setJvmJsonMetric(sb, JVM_THREAD, "count", "nounit", threads.getCount());
-        setJvmJsonMetric(sb, JVM_THREAD, "peak_count", "nounit", threads.getPeakCount());
-        setJvmJsonMetric(sb, JVM_THREAD, "new_count", "nounit", threads.getThreadsNewCount());
-        setJvmJsonMetric(sb, JVM_THREAD, "runnable_count", "nounit", threads.getThreadsRunnableCount());
-        setJvmJsonMetric(sb, JVM_THREAD, "blocked_count", "nounit", threads.getThreadsBlockedCount());
-        setJvmJsonMetric(sb, JVM_THREAD, "waiting_count", "nounit", threads.getThreadsWaitingCount());
-        setJvmJsonMetric(sb, JVM_THREAD, "timed_waiting_count", "nounit", threads.getThreadsTimedWaitingCount());
-        setJvmJsonMetric(sb, JVM_THREAD, "terminated_count", "nounit", threads.getThreadsTerminatedCount());
+        setJvmJsonMetric(sb, JVM_THREAD, null, "count", "nounit", threads.getCount());
+        setJvmJsonMetric(sb, JVM_THREAD, null, "peak_count", "nounit", threads.getPeakCount());
+        setJvmJsonMetric(sb, JVM_THREAD, null, "new_count", "nounit", threads.getThreadsNewCount());
+        setJvmJsonMetric(sb, JVM_THREAD, null,  "runnable_count", "nounit", threads.getThreadsRunnableCount());
+        setJvmJsonMetric(sb, JVM_THREAD, null,  "blocked_count", "nounit", threads.getThreadsBlockedCount());
+        setJvmJsonMetric(sb, JVM_THREAD, null,  "waiting_count", "nounit", threads.getThreadsWaitingCount());
+        setJvmJsonMetric(sb, JVM_THREAD, null, "timed_waiting_count", "nounit", threads.getThreadsTimedWaitingCount());
+        setJvmJsonMetric(sb, JVM_THREAD, null, "terminated_count", "nounit", threads.getThreadsTerminatedCount());
     }
 
-    private void setJvmJsonMetric(StringBuilder sb, String metric, String type, String unit, long value) {
+    private void setJvmJsonMetric(StringBuilder sb, String metric, String name, String type, String unit, long value) {
         sb.append("{\n\t\"tags\":\n\t{\n");
         sb.append("\t\t\"metric\":\"").append(metric).append("\"");
+        if (name != null) {
+            sb.append(",\n");
+            sb.append("\t\t\"name\":\"").append(name).append("\"\n");
+        }
         if (type != null) {
             sb.append(",\n");
             sb.append("\t\t\"type\":\"").append(type).append("\"\n");

diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
@@ -46,10 +46,10 @@ public class PrometheusMetricVisitor extends MetricVisitor {
     private static final String JVM_NON_HEAP_SIZE_BYTES = "jvm_non_heap_size_bytes";
     private static final String JVM_YOUNG_SIZE_BYTES = "jvm_young_size_bytes";
     private static final String JVM_OLD_SIZE_BYTES = "jvm_old_size_bytes";
-    private static final String JVM_YOUNG_GC = "jvm_young_gc";
-    private static final String JVM_OLD_GC = "jvm_old_gc";
     private static final String JVM_THREAD = "jvm_thread";
 
+    private static final String JVM_GC = "jvm_gc";
+
     private static final String HELP = "# HELP ";
     private static final String TYPE = "# TYPE ";
 
@@ -104,22 +104,15 @@ public void visitJvm(JvmStats jvmStats) {
         }
 
         // gc
-        Iterator<GarbageCollector> gcIter = jvmStats.getGc().iterator();
-        while (gcIter.hasNext()) {
-            GarbageCollector gc = gcIter.next();
-            if (gc.getName().equalsIgnoreCase("young")) {
-                sb.append(Joiner.on(" ").join(HELP, JVM_YOUNG_GC, "jvm young gc stat\n"));
-                sb.append(Joiner.on(" ").join(TYPE, JVM_YOUNG_GC, "gauge\n"));
-                sb.append(JVM_YOUNG_GC).append("{type=\"count\"} ").append(gc.getCollectionCount()).append("\n");
-                sb.append(JVM_YOUNG_GC).append("{type=\"time\"} ")
-                        .append(gc.getCollectionTime().getMillis()).append("\n");
-            } else if (gc.getName().equalsIgnoreCase("old")) {
-                sb.append(Joiner.on(" ").join(HELP, JVM_OLD_GC, "jvm old gc stat\n"));
-                sb.append(Joiner.on(" ").join(TYPE, JVM_OLD_GC, "gauge\n"));
-                sb.append(JVM_OLD_GC).append("{type=\"count\"} ").append(gc.getCollectionCount()).append("\n");
-                sb.append(JVM_OLD_GC).append("{type=\"time\"} ")
-                        .append(gc.getCollectionTime().getMillis()).append("\n");
-            }
+        sb.append(Joiner.on(" ").join(HELP, JVM_GC, "jvm gc stat\n"));
+        sb.append(Joiner.on(" ").join(TYPE, JVM_GC, "\n"));
+        for (GarbageCollector gc : jvmStats.getGc()) {
+            sb.append(JVM_GC).append("{");
+            sb.append("name=\"").append(gc.getName()).append(" Count").append("\", ").append("type=\"count\"} ")
+                    .append(gc.getCollectionCount()).append("\n");
+            sb.append(JVM_GC).append("{");
+            sb.append("name=\"").append(gc.getName()).append(" Time").append("\", ").append("type=\"time\"} ")
+                    .append(gc.getCollectionTime().getMillis()).append("\n");
         }
 
         // threads

diff --git a/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/GcNames.java b/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/GcNames.java
@@ -46,16 +46,4 @@ public static String getByMemoryPoolName(String poolName, String defaultName) {
         }
         return defaultName;
     }
-
-    public static String getByGcName(String gcName, String defaultName) {
-        if ("Copy".equals(gcName) || "PS Scavenge".equals(gcName) || "ParNew".equals(gcName)
-                || "G1 Young Generation".equals(gcName)) {
-            return YOUNG;
-        }
-        if ("MarkSweepCompact".equals(gcName) || "PS MarkSweep".equals(gcName)
-                || "ConcurrentMarkSweep".equals(gcName) || "G1 Old Generation".equals(gcName)) {
-            return OLD;
-        }
-        return defaultName;
-    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/JvmStats.java b/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/JvmStats.java
@@ -125,7 +125,7 @@ public static JvmStats jvmStats() {
         GarbageCollector[] collectors = new GarbageCollector[gcMxBeans.size()];
         for (int i = 0; i < collectors.length; i++) {
             GarbageCollectorMXBean gcMxBean = gcMxBeans.get(i);
-            collectors[i] = new GarbageCollector(GcNames.getByGcName(gcMxBean.getName(), gcMxBean.getName()),
+            collectors[i] = new GarbageCollector(gcMxBean.getName(),
                     gcMxBean.getCollectionCount(), gcMxBean.getCollectionTime());
         }
         GarbageCollectors garbageCollectors = new GarbageCollectors(collectors);

diff --git a/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java b/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
@@ -18,16 +18,25 @@
 package org.apache.doris.metric;
 
 import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.util.JsonUtil;
+import org.apache.doris.monitor.jvm.JvmService;
+import org.apache.doris.monitor.jvm.JvmStats;
 
 import com.codahale.metrics.Histogram;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.extern.slf4j.Slf4j;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
 import java.util.List;
 import java.util.Map;
 import java.util.SortedMap;
+import java.util.concurrent.atomic.AtomicInteger;
 
+@Slf4j
 public class MetricsTest {
 
     @BeforeClass
@@ -78,4 +87,39 @@ public void testUserQueryMetrics() {
         Assert.assertTrue(metricResult.contains("doris_fe_query_latency_ms{quantile=\"0.999\",user=\"test_user\"} 10.0"));
 
     }
+
+    @Test
+    public void testGc() {
+        PrometheusMetricVisitor visitor = new PrometheusMetricVisitor();
+        JvmService jvmService = new JvmService();
+        JvmStats jvmStats = jvmService.stats();
+        visitor.visitJvm(jvmStats);
+        String metric = MetricRepo.getMetric(visitor);
+        List<GarbageCollectorMXBean> gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans();
+        String finalMetricPrometheus = metric;
+        gcMxBeans.forEach(gcMxBean -> {
+            String name = gcMxBean.getName();
+            Assert.assertTrue(finalMetricPrometheus.contains("jvm_gc{name=\"" + name + " Count\", type=\"count\"} "));
+            Assert.assertTrue(finalMetricPrometheus.contains("jvm_gc{name=\"" + name + " Time\", type=\"time\"} "));
+        });
+
+        JsonMetricVisitor jsonMetricVisitor = new JsonMetricVisitor();
+        jsonMetricVisitor.visitJvm(jvmStats);
+        metric = MetricRepo.getMetric(jsonMetricVisitor);
+        String finalMetricJson = metric;
+        AtomicInteger size = new AtomicInteger(JsonUtil.parseArray(finalMetricJson).size());
+        gcMxBeans.forEach(gcMxBean -> JsonUtil.parseArray(finalMetricJson).forEach(json -> {
+            ObjectNode jsonObject = JsonUtil.parseObject(json.toString());
+            String name = gcMxBean.getName();
+            if (jsonObject.findValue("tags").findValue("metric").asText().equals("jvm_gc")
+                    && jsonObject.findValue("tags").findValue("name").asText().contains(name + " Count")) {
+                size.getAndDecrement();
+                Assert.assertTrue(jsonObject.findValue("tags").findValue("name").asText().contains(name + " Count")
+                        || jsonObject.findValue("tags").findValue("name").asText().contains(name + " Time"));
+            }
+
+        }));
+        Assert.assertTrue(size.get() < JsonUtil.parseArray(finalMetricJson).size());
+
+    }
 }
diff --git a/regression-test/suites/metrics_p0/test_fe_metrics.groovy b/regression-test/suites/metrics_p0/test_fe_metrics.groovy
@@ -24,6 +24,19 @@ suite("test_fe_metrics") {
             logger.debug("code:${code} body:${body}");
             assertEquals(200, code)
             assertTrue(body.contains("jvm_heap_size_bytes"))
+            assertTrue(body.contains("jvm_gc"))
         }
     }
-}
+
+    httpTest {
+        endpoint context.config.feHttpAddress
+        uri "/metrics?type=json"
+        op "get"
+        check { code, body ->
+            logger.debug("code:${code} body:${body}");
+            assertEquals(200, code)
+            assertTrue(body.contains("jvm_heap_size_bytes"))
+            assertTrue(body.contains("jvm_gc"))
+        }
+    }
+}