[AMORO-2494]: Promethues metric reporter (apache#2445)

* event api refactor * metric managers. * plugin manage && metric api * Refactoring of the plugin mechanism * metric name tests * Unit test for base plugin manager * unit test reporter * fix init logical * Adjust metrics api for fetch all metrics * Metric api adjust * prometheus demo * delete useless class * add prometheus modules * refactor codes to prometheus modules * refactor codes to prometheus modules * add docs * spotless apply * add liences * add comments * fix compile error * fix reviewers * merge from master and fix review comments * merge from master and fix review comments --------- Co-authored-by: ZhouJinsong <[email protected]>
huyuanfeng2018 · Mar 6, 2024 · b4af7da · b4af7da
1 parent 77163f0
commit b4af7da
Show file tree

Hide file tree

Showing 12 changed files with 472 additions and 4 deletions.
diff --git a/ams/api/src/test/java/com/netease/arctic/ams/api/metrics/MetricDefineTest.java b/ams/api/src/test/java/com/netease/arctic/ams/api/metrics/MetricDefineTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netease.arctic.ams.api.metrics;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.stream.Stream;
+
+public class MetricDefineTest {
+  private MetricDefine source =
+      new MetricDefine(
+          "test-define", Arrays.asList("tag1", "tag2"), MetricType.Counter, "description");
+
+  @ParameterizedTest
+  @MethodSource("provideMetricNamesForEquality")
+  public void testEquals(MetricDefine target, boolean expectedEquality) {
+    // MetricDefine is equally if(name, tag set, type) are qually
+    if (expectedEquality) {
+      assertEquals(source, target, "MetricNames should be equal");
+      assertEquals(source.hashCode(), target.hashCode(), "MetricNames hash code should be equal");
+    } else {
+      assertNotEquals(source, target, "MetricNames should not be equal");
+      assertNotEquals(
+          Objects.hash(source), Objects.hash(target), "MetricNames hash code should not be equal");
+    }
+  }
+
+  public static Stream<Arguments> provideMetricNamesForEquality() {
+
+    return Stream.of(
+        // same <name, tags, type> should be true
+        Arguments.of(
+            new MetricDefine(
+                "test-define", Arrays.asList("tag1", "tag2"), MetricType.Counter, "description"),
+            true),
+        // different name should be false
+        Arguments.of(
+            new MetricDefine(
+                "different-name", Arrays.asList("tag1", "tag2"), MetricType.Counter, "description"),
+            false),
+        // different order of tags should be true
+        Arguments.of(
+            new MetricDefine(
+                "test-define", Arrays.asList("tag2", "tag1"), MetricType.Counter, "description"),
+            true),
+        // different tags should be false
+        Arguments.of(
+            new MetricDefine(
+                "test-define", Arrays.asList("tag3", "tag4"), MetricType.Counter, "description"),
+            false),
+        // different MetricType should be false
+        Arguments.of(
+            new MetricDefine(
+                "test-define", Arrays.asList("tag1", "tag2"), MetricType.Gauge, "description"),
+            false),
+        // even description is different，should be true
+        Arguments.of(
+            new MetricDefine(
+                "test-define",
+                Arrays.asList("tag1", "tag2"),
+                MetricType.Counter,
+                "different description"),
+            true));
+  }
+}
diff --git a/ams/dist/pom.xml b/ams/dist/pom.xml
@@ -58,6 +58,12 @@
             <version>${project.version}</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>com.netease.amoro</groupId>
+            <artifactId>prometheus-reporter</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/ams/dist/src/main/arctic-bin/conf/plugins/metric-reporters.yaml b/ams/dist/src/main/arctic-bin/conf/plugins/metric-reporters.yaml
@@ -19,5 +19,7 @@
 # configurations of metric reporters
 
 #metric-reporters:
-#  - name: prometheus-exporter
-#    enabled: false
+#  - name: prometheus-exporter            # configs for prometheus exporter
+#    enabled: false
+#    properties:
+#       port: 7001
diff --git a/ams/dist/src/main/assemblies/bin.xml b/ams/dist/src/main/assemblies/bin.xml
@@ -36,6 +36,14 @@
             <destName>optimizer-job.jar</destName>
             <fileMode>0644</fileMode>
         </file>
+        <file>
+            <source>
+                ../metric-reporter/prometheus-reporter/target/prometheus-reporter-${project.version}-jar-with-dependencies.jar
+            </source>
+            <outputDirectory>plugin/metrics-reporter</outputDirectory>
+            <destName>prometheus-reporter.jar</destName>
+            <fileMode>0644</fileMode>
+        </file>
         <file>
             <source>
                 ../optimizer/spark-optimizer/target/spark-optimizer-${project.version}.jar
@@ -89,4 +97,4 @@
             <fileMode>0644</fileMode>
         </fileSet>
     </fileSets>
-</assembly>
+</assembly>
diff --git a/ams/metric-reporter/pom.xml b/ams/metric-reporter/pom.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.netease.amoro</groupId>
+        <artifactId>amoro-ams</artifactId>
+        <version>0.7.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>amoro-metric-reporter</artifactId>
+    <packaging>pom</packaging>
+    <name>Amoro Project AMS Metric Reporter Parent</name>
+    <url>https://amoro.netease.com</url>
+
+    <modules>
+        <module>prometheus-reporter</module>
+    </modules>
+
+</project>
diff --git a/ams/metric-reporter/prometheus-reporter/pom.xml b/ams/metric-reporter/prometheus-reporter/pom.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>amoro-metric-reporter</artifactId>
+        <groupId>com.netease.amoro</groupId>
+        <version>0.7.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>prometheus-reporter</artifactId>
+    <name>Amoro Project AMS Prometheus Reporter</name>
+    <url>https://amoro.netease.com</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.netease.amoro</groupId>
+            <artifactId>amoro-ams-api</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_httpserver</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>${maven-assembly-plugin.version}</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/...eporter/src/main/java/com.netease.arctic.metrics.reporter.promethues/MetricCollector.java b/...eporter/src/main/java/com.netease.arctic.metrics.reporter.promethues/MetricCollector.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netease.arctic.metrics.reporter.promethues;
+
+import com.netease.arctic.ams.api.metrics.Counter;
+import com.netease.arctic.ams.api.metrics.Gauge;
+import com.netease.arctic.ams.api.metrics.Metric;
+import com.netease.arctic.ams.api.metrics.MetricDefine;
+import com.netease.arctic.ams.api.metrics.MetricKey;
+import com.netease.arctic.ams.api.metrics.MetricSet;
+import com.netease.arctic.ams.api.metrics.MetricType;
+import io.prometheus.client.Collector;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/** Metric type converter for prometheus api */
+public class MetricCollector extends Collector {
+  private static final Logger LOGGER = LoggerFactory.getLogger(MetricCollector.class);
+  private static final String PREFIX = "amoro_";
+  private static final Pattern NAME_PATTERN = Pattern.compile("[a-zA-Z_:][a-zA-Z0-9_:]*");
+  private static final Pattern LABEL_PATTERN = Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*");
+  MetricSet metrics;
+
+  public MetricCollector(MetricSet metrics) {
+    this.metrics = metrics;
+  }
+
+  @Override
+  public List<MetricFamilySamples> collect() {
+    Map<MetricKey, Metric> registeredMetrics = metrics.getMetrics();
+
+    Map<MetricDefine, List<MetricKey>> metricDefineMap =
+        registeredMetrics.keySet().stream()
+            .collect(
+                Collectors.groupingBy(
+                    MetricKey::getDefine,
+                    Collectors.mapping(Function.identity(), Collectors.toList())));
+    return metricDefineMap.entrySet().stream()
+        .filter(entry -> isValidMetric(entry.getKey()))
+        .map(entry -> createFamilySample(entry.getKey(), entry.getValue(), registeredMetrics))
+        .collect(Collectors.toList());
+  }
+
+  private boolean isValidMetric(MetricDefine define) {
+    boolean nameIsValid = NAME_PATTERN.matcher(define.getName()).matches();
+    boolean labelIsValid = true;
+    for (String tag : define.getTags()) {
+      if (!LABEL_PATTERN.matcher(tag).matches()) {
+        labelIsValid = false;
+        break;
+      }
+    }
+    boolean valid = nameIsValid && labelIsValid;
+    if (!valid) {
+      LOGGER.warn("Metric {} is not a valid prometheus metric.", define);
+    }
+    return valid;
+  }
+
+  private MetricFamilySamples createFamilySample(
+      MetricDefine define, List<MetricKey> keys, Map<MetricKey, Metric> registeredMetrics) {
+
+    List<MetricFamilySamples.Sample> samples = Lists.newArrayList();
+    for (MetricKey key : keys) {
+      Metric metric = registeredMetrics.get(key);
+
+      MetricFamilySamples.Sample sample =
+          new MetricFamilySamples.Sample(
+              PREFIX + define.getName(), define.getTags(), key.valueOfTags(), covertValue(metric));
+      samples.add(sample);
+    }
+
+    return new MetricFamilySamples(
+        PREFIX + define.getName(), covertType(define.getType()), define.getDescription(), samples);
+  }
+
+  private Type covertType(MetricType metricType) {
+    switch (metricType) {
+      case Counter:
+        return Type.COUNTER;
+      case Gauge:
+        return Type.GAUGE;
+      default:
+        throw new IllegalStateException("unknown type:" + metricType);
+    }
+  }
+
+  private double covertValue(Metric metric) {
+    if (metric instanceof Counter) {
+      return ((Counter) metric).getCount();
+    } else if (metric instanceof Gauge) {
+      return ((Gauge<?>) metric).getValue().doubleValue();
+    } else {
+      throw new IllegalStateException(
+          "unknown metric implement class:" + metric.getClass().getName());
+    }
+  }
+}