Allow dots in metric names and label names

Signed-off-by: Fabian Stäber <[email protected]>
prometheus · Jul 27, 2023 · 0179958 · 0179958
1 parent fd37345
commit 0179958
Show file tree

Hide file tree

Showing 35 changed files with 1,096 additions and 317 deletions.
diff --git a/...src/main/java/io/prometheus/metrics/examples/otel_exemplars/greeting/GreetingServlet.java b/...src/main/java/io/prometheus/metrics/examples/otel_exemplars/greeting/GreetingServlet.java
@@ -27,7 +27,7 @@ public GreetingServlet() {
             .withUnit(Unit.SECONDS)
             .withLabelNames("http_status")
             .register();
-        histogram.withLabelValues("200");
+        histogram.initLabelValues("200");
     }
 
     @Override

diff --git a/...pp/src/main/java/io/prometheus/metrics/examples/otel_exemplars/app/HelloWorldServlet.java b/...pp/src/main/java/io/prometheus/metrics/examples/otel_exemplars/app/HelloWorldServlet.java
@@ -34,7 +34,7 @@ public HelloWorldServlet() {
                 .withUnit(Unit.SECONDS)
                 .withLabelNames("http_status")
                 .register();
-        histogram.withLabelValues("200");
+        histogram.initLabelValues("200");
     }
 
     @Override

diff --git a/...ervlet/src/main/java/io/prometheus/metrics/examples/tomcat_servlet/HelloWorldServlet.java b/...ervlet/src/main/java/io/prometheus/metrics/examples/tomcat_servlet/HelloWorldServlet.java
@@ -32,6 +32,11 @@ public class HelloWorldServlet extends HttpServlet {
             .withLabelNames("http_status")
             .register();
 
+    public HelloWorldServlet() {
+        counter.initLabelValues("200");
+        histogram.initLabelValues("200");
+    }
+
     @Override
     protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
         long start = System.nanoTime();

diff --git a/...theus-metrics-config/src/main/java/io/prometheus/metrics/config/PrometheusProperties.java b/...theus-metrics-config/src/main/java/io/prometheus/metrics/config/PrometheusProperties.java
@@ -58,7 +58,7 @@ public MetricsProperties getDefaultMetricProperties() {
      * May return {@code null} if no metric-specific properties are configured for a metric name.
      */
     public MetricsProperties getMetricProperties(String metricName) {
-        return metricProperties.get(metricName);
+        return metricProperties.get(metricName.replace(".", "_"));
     }
 
     public ExemplarsProperties getExemplarProperties() {

diff --git a/...metrics-config/src/main/java/io/prometheus/metrics/config/PrometheusPropertiesLoader.java b/...metrics-config/src/main/java/io/prometheus/metrics/config/PrometheusPropertiesLoader.java
@@ -39,6 +39,8 @@ public static PrometheusProperties load() throws PrometheusPropertiesException {
     // This will remove entries from properties when they are processed.
     private static Map<String, MetricsProperties> loadMetricsConfigs(Map<Object, Object> properties) {
         Map<String, MetricsProperties> result = new HashMap<>();
+        // Note that the metric name in the properties file must be as exposed in the Prometheus exposition formats,
+        // i.e. all dots replaced with underscores.
         Pattern pattern = Pattern.compile("io\\.prometheus\\.metrics\\.([^.]+)\\.");
         // Create a copy of the keySet() for iterating. We cannot iterate directly over keySet()
         // because entries are removed when MetricsConfig.load(...) is called.
@@ -49,7 +51,7 @@ private static Map<String, MetricsProperties> loadMetricsConfigs(Map<Object, Obj
         for (String propertyName : propertyNames) {
             Matcher matcher = pattern.matcher(propertyName);
             if (matcher.find()) {
-                String metricName = matcher.group(1);
+                String metricName = matcher.group(1).replace(".", "_");
                 if (!result.containsKey(metricName)) {
                     result.put(metricName, MetricsProperties.load("io.prometheus.metrics." + metricName, properties));
                 }

diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Counter.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Counter.java
@@ -9,13 +9,16 @@
 import io.prometheus.metrics.model.snapshots.CounterSnapshot;
 import io.prometheus.metrics.model.snapshots.Exemplar;
 import io.prometheus.metrics.model.snapshots.Labels;
+import io.prometheus.metrics.model.snapshots.PrometheusNaming;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.atomic.DoubleAdder;
 import java.util.concurrent.atomic.LongAdder;
 
+import static io.prometheus.metrics.model.snapshots.PrometheusNaming.prometheusName;
+
 /**
  * Counter metric.
  * <p>
@@ -109,8 +112,8 @@ protected CounterSnapshot collect(List<Labels> labels, List<DataPoint> metricDat
         return new CounterSnapshot(getMetadata(), data);
     }
 
-    static String normalizeName(String name) {
-        if (name != null && name.endsWith("_total")) {
+    static String stripTotalSuffix(String name) {
+        if (name != null && (name.endsWith("_total") || name.endsWith(".total"))) {
             name = name.substring(0, name.length() - 6);
         }
         return name;
@@ -231,12 +234,12 @@ private Builder(PrometheusProperties properties) {
          * In the example above both {@code c1} and {@code c2} would be named {@code "events_total"} in Prometheus.
          * <p>
          * Throws an {@link IllegalArgumentException} if
-         * {@link io.prometheus.metrics.model.snapshots.MetricMetadata#isValidMetricName(String) MetricMetadata.isValidMetricName(name)}
+         * {@link io.prometheus.metrics.model.snapshots.PrometheusNaming#isValidMetricName(String) MetricMetadata.isValidMetricName(name)}
          * is {@code false}.
          */
         @Override
         public Builder withName(String name) {
-            return super.withName(normalizeName(name));
+            return super.withName(stripTotalSuffix(name));
         }
 
         @Override

diff --git a/...us-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/CounterWithCallback.java b/...us-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/CounterWithCallback.java
@@ -80,12 +80,12 @@ private Builder(PrometheusProperties properties) {
          * In the example above both {@code c1} and {@code c2} would be named {@code "events_total"} in Prometheus.
          * <p>
          * Throws an {@link IllegalArgumentException} if
-         * {@link io.prometheus.metrics.model.snapshots.MetricMetadata#isValidMetricName(String) MetricMetadata.isValidMetricName(name)}
+         * {@link io.prometheus.metrics.model.snapshots.PrometheusNaming#isValidMetricName(String) MetricMetadata.isValidMetricName(name)}
          * is {@code false}.
          */
         @Override
         public Builder withName(String name) {
-            return super.withName(Counter.normalizeName(name));
+            return super.withName(Counter.stripTotalSuffix(name));
         }
 
         @Override

diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Info.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/Info.java
@@ -2,16 +2,16 @@
 
 import io.prometheus.metrics.config.PrometheusProperties;
 import io.prometheus.metrics.model.snapshots.InfoSnapshot;
-import io.prometheus.metrics.model.snapshots.Label;
 import io.prometheus.metrics.model.snapshots.Labels;
 import io.prometheus.metrics.model.snapshots.Unit;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 
+import static io.prometheus.metrics.model.snapshots.PrometheusNaming.prometheusName;
+
 /**
  * Info metric. Example:
  * <pre>{@code
@@ -90,15 +90,12 @@ private Builder(PrometheusProperties config) {
          * In the example above both {@code info1} and {@code info2} will be named {@code "runtime_info"} in Prometheus.
          * <p>
          * Throws an {@link IllegalArgumentException} if
-         * {@link io.prometheus.metrics.model.snapshots.MetricMetadata#isValidMetricName(String) MetricMetadata.isValidMetricName(name)}
+         * {@link io.prometheus.metrics.model.snapshots.PrometheusNaming#isValidMetricName(String) MetricMetadata.isValidMetricName(name)}
          * is {@code false}.
          */
         @Override
         public Builder withName(String name) {
-            if (name != null && name.endsWith("_info")) {
-                name = name.substring(0, name.length() - 5);
-            }
-            return super.withName(name);
+            return super.withName(stripInfoSuffix(name));
         }
 
         /**
@@ -112,16 +109,16 @@ public Builder withUnit(Unit unit) {
             return this;
         }
 
-        private static String normalizeName(String name) {
-            if (name != null && name.endsWith("_info")) {
+        private static String stripInfoSuffix(String name) {
+            if (name != null && (name.endsWith("_info") || name.endsWith(".info"))) {
                 name = name.substring(0, name.length() - 5);
             }
             return name;
         }
 
         @Override
         public Info build() {
-            return new Info(withName(normalizeName(name)));
+            return new Info(this);
         }
 
         @Override

diff --git a/...etrics-core/src/main/java/io/prometheus/metrics/core/metrics/MetricWithFixedMetadata.java b/...etrics-core/src/main/java/io/prometheus/metrics/core/metrics/MetricWithFixedMetadata.java
@@ -3,11 +3,14 @@
 import io.prometheus.metrics.config.PrometheusProperties;
 import io.prometheus.metrics.model.snapshots.Labels;
 import io.prometheus.metrics.model.snapshots.MetricMetadata;
+import io.prometheus.metrics.model.snapshots.PrometheusNaming;
 import io.prometheus.metrics.model.snapshots.Unit;
 
 import java.util.Arrays;
 import java.util.List;
 
+import static io.prometheus.metrics.model.snapshots.PrometheusNaming.prometheusName;
+
 /**
  * Almost all metrics have fixed metadata, i.e. the metric name is known when the metric is created.
  * <p>
@@ -31,9 +34,8 @@ protected MetricMetadata getMetadata() {
 
     private String makeName(String name, Unit unit) {
         if (unit != null) {
-            String suffix = "_" + unit;
-            if (!name.endsWith(suffix)) {
-                name = name + suffix;
+            if (!name.endsWith(unit.toString())) {
+                name = name + "_" + unit;
             }
         }
         return name;
@@ -51,7 +53,7 @@ protected Builder(List<String> illegalLabelNames, PrometheusProperties propertie
         }
 
         public B withName(String name) {
-            if (!MetricMetadata.isValidMetricName(name)) {
+            if (!PrometheusNaming.isValidMetricName(name)) {
                 throw new IllegalArgumentException("'" + name + "': Illegal metric name.");
             }
             this.name = name;
@@ -70,7 +72,7 @@ public B withHelp(String help) {
 
         public B withLabelNames(String... labelNames) {
             for (String labelName : labelNames) {
-                if (!Labels.isValidLabelName(labelName)) {
+                if (!PrometheusNaming.isValidLabelName(labelName)) {
                     throw new IllegalArgumentException(labelName + ": illegal label name");
                 }
                 if (illegalLabelNames.contains(labelName)) {

diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/StateSet.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/StateSet.java
@@ -3,6 +3,7 @@
 import io.prometheus.metrics.config.MetricsProperties;
 import io.prometheus.metrics.config.PrometheusProperties;
 import io.prometheus.metrics.model.snapshots.Labels;
+import io.prometheus.metrics.model.snapshots.MetricMetadata;
 import io.prometheus.metrics.model.snapshots.StateSetSnapshot;
 import io.prometheus.metrics.core.datapoints.StateSetDataPoint;
 
@@ -11,6 +12,8 @@
 import java.util.List;
 import java.util.stream.Stream;
 
+import static io.prometheus.metrics.model.snapshots.PrometheusNaming.prometheusName;
+
 /**
  * StateSet metric. Example:
  * <pre>{@code
@@ -58,7 +61,7 @@ private StateSet(Builder builder, PrometheusProperties prometheusProperties) {
         exemplarsEnabled = getConfigProperty(properties, MetricsProperties::getExemplarsEnabled);
         this.names = builder.names; // builder.names is already a validated copy
         for (String name : names) {
-            if (this.getMetadata().getName().equals(name)) {
+            if (this.getMetadata().getPrometheusName().equals(prometheusName(name))) {
                 throw new IllegalArgumentException("Label name " + name + " is illegal (can't use the metric name as label name in state set metrics)");
             }
         }

diff --git a/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/StatefulMetric.java b/prometheus-metrics-core/src/main/java/io/prometheus/metrics/core/metrics/StatefulMetric.java
@@ -62,6 +62,34 @@ public MetricSnapshot collect() {
         return collect(labels, metricData);
     }
 
+    /**
+     * Initialize label values.
+     * <p>
+     * Example: Imagine you have a counter for payments as follows
+     * <pre>
+     * payment_transactions_total{payment_type="credit card"} 7.0
+     * payment_transactions_total{payment_type="paypal"} 3.0
+     * </pre>
+     * Now, the data points for the {@code payment_type} label values get initialized when they are
+     * first used, i.e. the first time you call
+     * <pre>{@code
+     * counter.withLabelValues("paypal").inc();
+     * }</pre>
+     * the data point with label {@code payment_type="paypal"} will go from non-existent to having value {@code 1.0}.
+     * <p>
+     * In some cases this is confusing, and you want to have data points initialized on application start
+     * with an initial value of {@code 0.0}:
+     * <pre>
+     * payment_transactions_total{payment_type="credit card"} 0.0
+     * payment_transactions_total{payment_type="paypal"} 0.0
+     * </pre>
+     * {@code initLabelValues(...)} can be used to initialize label value, so that the data points
+     * show up in the exposition format with an initial value of zero.
+     */
+    public void initLabelValues(String... labelValues) {
+        withLabelValues(labelValues);
+    }
+
     public D withLabelValues(String... labelValues) {
         if (labelValues.length != labelNames.length) {
             if (labelValues.length == 0) {
@@ -73,7 +101,6 @@ public D withLabelValues(String... labelValues) {
         return data.computeIfAbsent(Arrays.asList(labelValues), l -> newDataPoint());
     }
 
-    // TODO: Remove automatically if label values have not been used in a while?
     public void remove(String... labelValues) {
         data.remove(Arrays.asList(labelValues));
     }

diff --git a/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/CounterTest.java b/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/CounterTest.java
@@ -109,18 +109,18 @@ public void testLabels() {
 
     @Test
     public void testTotalStrippedFromName() {
-        Counter counter = Counter.newBuilder()
-                .withName("my_counter_total")
-                .withUnit(Unit.SECONDS)
-                .build();
-        Metrics.MetricFamily protobufData = new PrometheusProtobufWriter().convert(counter.collect());
-        assertEquals("name: \"my_counter_seconds_total\" type: COUNTER metric { counter { value: 0.0 } }", TextFormat.printer().shortDebugString(protobufData));
-
-        counter = Counter.newBuilder()
-                .withName("my_counter")
-                .build();
-        protobufData = new PrometheusProtobufWriter().convert(counter.collect());
-        assertEquals("name: \"my_counter_total\" type: COUNTER metric { counter { value: 0.0 } }", TextFormat.printer().shortDebugString(protobufData));
+        for (String name : new String[]{
+                "my_counter_total", "my.counter.total",
+                "my_counter_seconds_total", "my.counter.seconds.total",
+                "my_counter", "my.counter",
+                "my_counter_seconds", "my.counter.seconds"}) {
+            Counter counter = Counter.newBuilder()
+                    .withName(name)
+                    .withUnit(Unit.SECONDS)
+                    .build();
+            Metrics.MetricFamily protobufData = new PrometheusProtobufWriter().convert(counter.collect());
+            assertEquals("name: \"my_counter_seconds_total\" type: COUNTER metric { counter { value: 0.0 } }", TextFormat.printer().shortDebugString(protobufData));
+        }
     }
 
     @Test

diff --git a/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/HistogramTest.java b/prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/HistogramTest.java
@@ -1,19 +1,19 @@
 package io.prometheus.metrics.core.metrics;
 
 import io.prometheus.metrics.com_google_protobuf_3_21_7.TextFormat;
+import io.prometheus.metrics.core.datapoints.DistributionDataPoint;
+import io.prometheus.metrics.core.exemplars.ExemplarSamplerConfigTestUtil;
 import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter;
 import io.prometheus.metrics.expositionformats.PrometheusProtobufWriter;
 import io.prometheus.metrics.expositionformats.generated.com_google_protobuf_3_21_7.Metrics;
-import io.prometheus.metrics.core.exemplars.ExemplarSamplerConfigTestUtil;
-import io.prometheus.metrics.model.snapshots.MetricSnapshots;
-import io.prometheus.metrics.tracer.common.SpanContext;
-import io.prometheus.metrics.tracer.initializer.SpanContextSupplier;
 import io.prometheus.metrics.model.snapshots.ClassicHistogramBucket;
 import io.prometheus.metrics.model.snapshots.Exemplar;
 import io.prometheus.metrics.model.snapshots.Exemplars;
 import io.prometheus.metrics.model.snapshots.HistogramSnapshot;
 import io.prometheus.metrics.model.snapshots.Labels;
-import io.prometheus.metrics.core.datapoints.DistributionDataPoint;
+import io.prometheus.metrics.model.snapshots.MetricSnapshots;
+import io.prometheus.metrics.tracer.common.SpanContext;
+import io.prometheus.metrics.tracer.initializer.SpanContextSupplier;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -1043,19 +1043,9 @@ public void testIllegalLabelNamePrefix() {
                 .withLabelNames("__hello");
     }
 
-    @Test(expected = IllegalArgumentException.class)
-    public void testIllegalLabelNameDot() {
-        // The Prometheus team are investigating to allow dots in future Prometheus versions, but for now it's invalid.
-        // The reason is that you cannot use illegal label names in the Prometheus query language.
-        Histogram.newBuilder()
-                .withName("test")
-                .withLabelNames("http.status");
-    }
-
     @Test(expected = IllegalArgumentException.class)
     public void testIllegalName() {
-        Histogram.newBuilder()
-                .withName("server.durations");
+        Histogram.newBuilder().withName("my_namespace/server.durations");
     }
 
     @Test(expected = IllegalArgumentException.class)